Skip to content

Commit

Permalink
run a spicedbcluster by picking a version from an update channel
Browse files Browse the repository at this point in the history 
this changes some of the default behavior for spicedbclusters:

- if an image with a tag or digest is provided, that image is used
- if an image without a tag or digest is provide, that is used as a base
  image for updates from the update graph
- images, tags, and digests are no longer validated against an allowlist
  - instead, all installs come from an update graph with an explicit tag
    or digest (unless explicitly overridden)
- update graphs are configured for the operator, for now only using a
  local config file
- when a version is specified, the operator will walk the update graph
  up to that version and then stop.
- when no version is specified, the operator will walk to the head of
  the channel, and will continue to do so as the update graph is updated
- when no channel is specified, the operator attempts to pick one based
  on the datastore chosen. in the future, other heuristics may be used.
  • Loading branch information
@ecordell
ecordell committed Nov 14, 2022
1 parent 964c0be commit c9b07b3
Show file tree
Hide file tree
Showing 13 changed files with 708 additions and 649 deletions.
3 changes: 0 additions & 3 deletions .golangci.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,6 @@ linters:
enable:
- "bidichk"
- "bodyclose"
- "deadcode"
- "errcheck"
- "errname"
- "errorlint"
Expand All @@ -29,12 +28,10 @@ linters:
- "revive"
- "rowserrcheck"
- "staticcheck"
- "structcheck"
- "stylecheck"
- "tenv"
- "typecheck"
- "unconvert"
- "unused"
- "varcheck"
- "wastedassign"
- "whitespace"
48 changes: 48 additions & 0 deletions config/crds/authzed.com_spicedbclusters.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,14 @@ spec:
spec:
description: ClusterSpec holds the desired state of the cluster.
properties:
channel:
description: Channel is the name of a series of updates that operator
should follow. The operator is configured with a datasource that
configures available channels and update paths. If `version` is
not specified, then the operator will keep SpiceDB up-to-date with
the current head of the channel. If `version` is specified, then
the operator will write available updates in the status.
type: string
config:
description: Config values to be passed to the cluster
type: object
Expand All @@ -46,10 +54,36 @@ spec:
that holds secret config for the cluster like passwords, credentials,
etc. If the secret is omitted, one will be generated
type: string
version:
description: Version is the name of the version of SpiceDB that will
be run. The version is usually a simple version string like `v1.13.0`,
but the operator is configured with a data source that tells it
what versions are allowed, and they may have other names. If omitted,
the newest version in the head of the channel will be used. Note
that the `config.image` field will take precedence over version/channel,
if it is specified
type: string
type: object
status:
description: ClusterStatus communicates the observed state of the cluster.
properties:
availableVersions:
description: AvailableVersions is a list of versions that the currently
running version can be updated to. Only applies if using an update
channel.
items:
properties:
channel:
type: string
description:
type: string
name:
type: string
required:
- channel
- name
type: object
type: array
conditions:
description: Conditions for the current state of the Stack.
items:
Expand Down Expand Up @@ -147,6 +181,20 @@ spec:
description: TargetMigrationHash is a hash of the desired migration
target and config
type: string
version:
description: CurrentVersion is a description of the currently selected
version from the channel, if an update channel is being used.
properties:
channel:
type: string
description:
type: string
name:
type: string
required:
- channel
- name
type: object
type: object
type: object
served: true
Expand Down
124 changes: 51 additions & 73 deletions e2e/cluster_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,7 @@ import (
"github.com/authzed/spicedb-operator/pkg/apis/authzed/v1alpha1"
"github.com/authzed/spicedb-operator/pkg/config"
"github.com/authzed/spicedb-operator/pkg/metadata"
"github.com/authzed/spicedb-operator/pkg/updates"
)

var (
Expand Down Expand Up @@ -306,6 +307,7 @@ var _ = Describe("SpiceDBClusters", func() {
defer watchCancel()
Watch(watchCtx, client, v1alpha1ClusterGVR, ktypes.NamespacedName{Name: name, Namespace: namespace}, "0", func(c *v1alpha1.SpiceDBCluster) bool {
condition = c.FindStatusCondition("Migrating")
GinkgoWriter.Println(c.Status)
return condition == nil
})
g.Expect(condition).To(EqualCondition(v1alpha1.NewMigratingCondition(datastoreEngine, migration)))
Expand Down Expand Up @@ -440,7 +442,7 @@ var _ = Describe("SpiceDBClusters", func() {
condition := c.FindStatusCondition("ValidatingFailed")
if condition != nil {
foundValidationFailed = true
Expect(condition).To(EqualCondition(v1alpha1.NewInvalidConfigCondition("", fmt.Errorf("[datastoreEngine is a required field, secret must be provided]"))))
Expect(condition).To(EqualCondition(v1alpha1.NewInvalidConfigCondition("", fmt.Errorf("[datastoreEngine is a required field, couldn't find channel for datastore \"\": no channel found for datastore \"\", no update found in channel, secret must be provided]"))))
break
}
}
Expand Down Expand Up @@ -642,6 +644,7 @@ var _ = Describe("SpiceDBClusters", func() {
config := map[string]any{
"datastoreEngine": dsDef.datastoreEngine,
"envPrefix": spicedbEnvPrefix,
"image": "spicedb:dev",
"cmd": spicedbCmd,
}
for k, v := range dsDef.passthroughConfig {
Expand Down Expand Up @@ -723,6 +726,7 @@ var _ = Describe("SpiceDBClusters", func() {
config, err := json.Marshal(map[string]any{
"skipMigrations": true,
"datastoreEngine": dsDef.datastoreEngine,
"image": "spicedb:dev",
"envPrefix": spicedbEnvPrefix,
"cmd": spicedbCmd,
})
Expand Down Expand Up @@ -772,6 +776,7 @@ var _ = Describe("SpiceDBClusters", func() {
"datastoreEngine": dsDef.datastoreEngine,
"envPrefix": spicedbEnvPrefix,
"cmd": spicedbCmd,
"image": "spicedb:dev",
"tlsSecretName": "spicedb-grpc-tls",
"dispatchUpstreamCASecretName": "spicedb-grpc-tls",
}
Expand Down Expand Up @@ -853,47 +858,6 @@ var _ = Describe("SpiceDBClusters", func() {
return spiceCluster.Namespace, spiceCluster.Name
})
})

When("the image name/tag are updated", Ordered, func() {
var imageName string

BeforeAll(func() {
newConfig := config.OperatorConfig{
ImageTag: "updated",
ImageName: "spicedb",
}
imageName = strings.Join([]string{newConfig.ImageName, newConfig.ImageTag}, ":")
WriteConfig(newConfig)
})

AfterAll(func() {
newConfig := config.OperatorConfig{
ImageName: "spicedb",
ImageTag: "dev",
}
imageName = strings.Join([]string{newConfig.ImageName, newConfig.ImageTag}, ":")
WriteConfig(newConfig)
})

It("migrates to the latest version", func() {
AssertMigrationsCompleted(imageName, "", "",
func() (string, string, string) {
return spiceCluster.Namespace, spiceCluster.Name, dsDef.datastoreEngine
})
})

It("updates the deployment", func() {
AssertHealthySpiceDBCluster(imageName, func() (string, string) {
return spiceCluster.Namespace, spiceCluster.Name
}, Not(ContainSubstring("ERROR: kuberesolver")))
})

It("deletes the migration job", func() {
AssertMigrationJobCleanup(func() (string, string) {
return spiceCluster.Namespace, spiceCluster.Name
})
})
})
})
})
})
Expand Down Expand Up @@ -953,23 +917,36 @@ var _ = Describe("SpiceDBClusters", func() {
DeferCleanup(cancel)

newConfig := config.OperatorConfig{
AllowedTags: []string{"v1.13.0", "dev", "updated"},
AllowedImages: []string{
"ghcr.io/authzed/spicedb",
"spicedb",
ImageName: "ghcr.io/authzed/spicedb",
UpdateGraph: updates.UpdateGraph{
Channels: []updates.Channel{
{
Name: "postgres",
Metadata: map[string]string{"datastore": "postgres"},
Nodes: []updates.State{
{ID: "v1.14.1", Tag: "v1.14.1", Migration: "drop-id-constraints"},
{ID: "v1.14.0", Tag: "v1.14.0", Migration: "drop-id-constraints"},
{ID: "v1.14.0-phase2", Tag: "v1.14.0", Migration: "add-xid-constraints", Phase: "write-both-read-new"},
{ID: "v1.14.0-phase1", Tag: "v1.14.0", Migration: "add-xid-columns", Phase: "write-both-read-old"},
{ID: "v1.13.0", Tag: "v1.13.0", Migration: "add-ns-config-id"},
},
Edges: map[string][]string{
"v1.13.0": {"v1.14.0-phase1"},
"v1.14.0-phase1": {"v1.14.0-phase2"},
"v1.14.0-phase2": {"v1.14.0"},
"v1.14.0": {"v1.14.1"},
},
},
},
},
UpdateGraph: config.NewUpdateGraph(),
}
WriteConfig(newConfig)

classConfig := map[string]any{
"logLevel": "debug",
"datastoreEngine": "postgres",
"envPrefix": spicedbEnvPrefix,
"cmd": spicedbCmd,
"tlsSecretName": "spicedb4-grpc-tls",
"dispatchUpstreamCASecretName": "spicedb4-grpc-tls",
"image": "spicedb:v1.13.0",
}
jsonConfig, err := json.Marshal(classConfig)
Expect(err).To(BeNil())
Expand All @@ -983,6 +960,7 @@ var _ = Describe("SpiceDBClusters", func() {
Namespace: testNamespace,
},
Spec: v1alpha1.ClusterSpec{
Version: "v1.13.0",
Config: jsonConfig,
SecretRef: "spicedb4",
},
Expand Down Expand Up @@ -1016,7 +994,7 @@ var _ = Describe("SpiceDBClusters", func() {
_, err = client.Resource(v1alpha1ClusterGVR).Namespace(spiceCluster.Namespace).Create(ctx, &unstructured.Unstructured{Object: u}, metav1.CreateOptions{})
Expect(err).To(Succeed())

AssertMigrationsCompleted("spicedb:v1.13.0", "head", "",
AssertMigrationsCompleted("ghcr.io/authzed/spicedb:v1.13.0", "add-ns-config-id", "",
func() (string, string, string) {
return spiceCluster.Namespace, spiceCluster.Name, "postgres"
})
Expand All @@ -1029,19 +1007,20 @@ var _ = Describe("SpiceDBClusters", func() {
g.Expect(len(fetched.Status.Conditions)).To(BeZero())
}).Should(Succeed())

// once the cluster is running at the initial version,
init := config.SpiceDBMigrationState{Tag: "v1.13.0", Migration: "add-ns-config-id"}
phase1 := config.SpiceDBMigrationState{Tag: "dev", Migration: "add-xid-columns", Phase: "write-both-read-old"}
phase2 := config.SpiceDBMigrationState{Tag: "dev", Migration: "add-xid-constraints", Phase: "write-both-read-new"}
phase3 := config.SpiceDBMigrationState{Tag: "dev", Migration: "drop-id-constraints"}
phase4 := config.SpiceDBMigrationState{Tag: "updated"}

newConfig.AddHeadMigration(config.SpiceDBDatastoreState{Tag: "v1.13.0", Datastore: "postgres"}, "add-ns-config-id")
newConfig.AddEdge(init, phase1)
newConfig.AddEdge(phase1, phase2)
newConfig.AddEdge(phase2, phase3)
newConfig.AddEdge(phase3, phase4)
WriteConfig(newConfig)
// once the cluster is running at the initial version, update the target version
Eventually(func(g Gomega) {
clusterUnst, err := client.Resource(v1alpha1ClusterGVR).Namespace(spiceCluster.Namespace).Get(ctx, spiceCluster.Name, metav1.GetOptions{})
g.Expect(err).To(Succeed())
fetched, err := typed.UnstructuredObjToTypedObj[*v1alpha1.SpiceDBCluster](clusterUnst)
g.Expect(err).To(Succeed())
g.Expect(len(fetched.Status.Conditions)).To(BeZero())

fetched.Spec.Version = "v1.14.1"
u, err = runtime.DefaultUnstructuredConverter.ToUnstructured(fetched)
Expect(err).To(Succeed())
_, err = client.Resource(v1alpha1ClusterGVR).Namespace(spiceCluster.Namespace).Update(ctx, &unstructured.Unstructured{Object: u}, metav1.UpdateOptions{})
Expect(err).To(Succeed())
}).Should(Succeed())
})

AfterAll(func() {
Expand All @@ -1050,7 +1029,6 @@ var _ = Describe("SpiceDBClusters", func() {

newConfig := config.OperatorConfig{
ImageName: "spicedb",
ImageTag: "dev",
}
WriteConfig(newConfig)

Expand All @@ -1061,31 +1039,31 @@ var _ = Describe("SpiceDBClusters", func() {

When("there is a series of required migrations", Ordered, func() {
It("migrates to phase1", func() {
AssertMigrationsCompleted("spicedb:dev", "add-xid-columns", "write-both-read-old",
AssertMigrationsCompleted("ghcr.io/authzed/spicedb:v1.14.0", "add-xid-columns", "write-both-read-old",
func() (string, string, string) {
return spiceCluster.Namespace, spiceCluster.Name, "postgres"
})
})

It("migrates to phase2", func() {
AssertMigrationsCompleted("spicedb:dev", "add-xid-constraints", "write-both-read-new",
AssertMigrationsCompleted("ghcr.io/authzed/spicedb:v1.14.0", "add-xid-constraints", "write-both-read-new",
func() (string, string, string) {
return spiceCluster.Namespace, spiceCluster.Name, "postgres"
})
})

It("migrates to phase3", func() {
AssertMigrationsCompleted("spicedb:dev", "drop-id-constraints", "",
AssertMigrationsCompleted("ghcr.io/authzed/spicedb:v1.14.0", "drop-id-constraints", "",
func() (string, string, string) {
return spiceCluster.Namespace, spiceCluster.Name, "postgres"
})
})

It("migrates to phase4", func() {
AssertMigrationsCompleted("spicedb:updated", "", "",
func() (string, string, string) {
return spiceCluster.Namespace, spiceCluster.Name, "postgres"
})
It("updates to the next version", func() {
AssertHealthySpiceDBCluster("ghcr.io/authzed/spicedb:v1.14.1",
func() (string, string) {
return spiceCluster.Namespace, spiceCluster.Name
}, Not(ContainSubstring("ERROR: kuberesolver")))
})
})
})
Expand Down
1 change: 0 additions & 1 deletion e2e/e2e_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -136,7 +136,6 @@ func StartOperator() {

opconfig := config.OperatorConfig{
ImageName: "spicedb",
ImageTag: "dev",
}

testRestConfig := rest.CopyConfig(restConfig)
Expand Down
32 changes: 32 additions & 0 deletions pkg/apis/authzed/v1alpha1/types.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,24 @@ func (c *SpiceDBCluster) WithAnnotations(entries map[string]string) *SpiceDBClus

// ClusterSpec holds the desired state of the cluster.
type ClusterSpec struct {
// Version is the name of the version of SpiceDB that will be run.
// The version is usually a simple version string like `v1.13.0`, but the
// operator is configured with a data source that tells it what versions
// are allowed, and they may have other names.
// If omitted, the newest version in the head of the channel will be used.
// Note that the `config.image` field will take precedence over
// version/channel, if it is specified
Version string `json:"version,omitempty"`

// Channel is the name of a series of updates that operator should follow.
// The operator is configured with a datasource that configures available
// channels and update paths.
// If `version` is not specified, then the operator will keep SpiceDB
// up-to-date with the current head of the channel.
// If `version` is specified, then the operator will write available updates
// in the status.
Channel string `json:"channel,omitempty"`

// Config values to be passed to the cluster
// +optional
// +kubebuilder:validation:Schemaless
Expand Down Expand Up @@ -89,11 +107,25 @@ type ClusterStatus struct {
// Phase is the currently running phase (used for phased migrations)
Phase string `json:"phase,omitempty"`

// CurrentVersion is a description of the currently selected version from
// the channel, if an update channel is being used.
CurrentVersion *SpiceDBVersion `json:"version,omitempty"`

// AvailableVersions is a list of versions that the currently running
// version can be updated to. Only applies if using an update channel.
AvailableVersions []SpiceDBVersion `json:"availableVersions,omitempty"`

// Conditions for the current state of the Stack.
// +optional
Conditions []metav1.Condition `json:"conditions,omitempty" patchStrategy:"merge" patchMergeKey:"type" protobuf:"bytes,1,rep,name=conditions"`
}

type SpiceDBVersion struct {
Name string `json:"name"`
Channel string `json:"channel"`
Description string `json:"description,omitempty"`
}

// SpiceDBClusterList is a list of SpiceDBCluster resources
//
// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object
Expand Down
Loading

0 comments on commit c9b07b3

Please sign in to comment.