Add snapshot migration tool (#326)

Add a CLI tool to assist in migrating v1alpha1 snapshots to v1beta1. The
tool works by converting v1alpha1 representations into v1beta1 and
writing them out as YAML that can then be applied on a target cluster.

CHANGELOG.md

@@ -1,5 +1,7 @@
 ## unreleased
 
+* Add snapshot migration tool
+  [[GH-325]](https://github.com/digitalocean/csi-digitalocean/pull/325)
 * Update csi-resizer sidecar to v0.5.0
   [[GH-324]](https://github.com/digitalocean/csi-digitalocean/pull/324)
 * Support v1beta1 snapshots

README.md

@@ -77,7 +77,9 @@ Snapshots can be created and restored through `VolumeSnapshot` objects.
 ---
 **Note:**
 
-Since version 2, the CSI plugin support v1beta1 Volume Snapshots only. Support for the v1alpha1 has been dropped. Users that intend to migrate need to remove all v1alpha1 Volume Snapshot CRDs before installing the v1beta1 CRDs.
+Since version 2, the CSI plugin support v1beta1 Volume Snapshots only. Support for the v1alpha1 has been dropped.
+
+Users that want to migrate their v1alpha1 Volume Snapshots into a v1beta1 cluster can leverage [this migration tool](/cmd/migrate-snapshots). (For DOKS customers, the migration will be applied automatically during cluster upgrades.)
 
 ---
 

cmd/migrate-snapshots/.gitignore

@@ -0,0 +1 @@
+migrate-snapshots

cmd/migrate-snapshots/README.md

@@ -0,0 +1,30 @@
+# migrate-snapshots
+
+_tl;dr: for DigitalOcean Kubernetes Service (DOKS) users: you can ignore the instructions and the tool below -- DigitalOcean takes care of migrating your snapshots when upgrading your clusters from 1.17 to 1.18._
+
+`migrate-snapshots` is a small tool that allows to migrate existing snapshots from the v1alpha1 to the v1beta1 API. This is necessary because no official migration path exists by the Kubernetes project.
+
+The tool is focused on DigitalOcean's Block Storage snapshot driver only: snapshot objects belonging to a different `VolumeSnapshotClass` will be ignored.
+
+`migrate-snapshots` is meant to be run against a cluster that still runs on v1alpha1 snapshots. After connecting to the cluster, it will iterate over all `VolumeSnapshotContent` and `VolumeSnapshot` objects and convert them to their respective v1beta1 structures. The converted objects are stored as YAML manifest files below a directory of choice.
+
+It is assumed that the snapshots in the DigitalOcean storage system won't be deleted even if the snapshots objects in the Kubernetes cluster are, which can be achieved by changing the `deletionPolicy` on all `VolumeSnapshotContent` objects to `Retain`. Afterwards, the snapshots can be reimported by applying the YAML manifests to the upgraded or new cluster.
+
+By default, `migrate-snapshots` reads out the `$HOME/.kube/config` file if it exists. A custom kubeconfig can be specified through the `KUBECONFIG` environment variable. The Kubernetes host can also be specified directly through the `-server` argument.
+
+## Steps
+
+Here are the proposed steps:
+
+1. Run `migrate-snapshots` without the `-directory` parameter to see which objects would be persisted.
+1. Run `migrate-snapshots -directory snapshot_objects` to store all snapshot-related objects below the `snapshot_objects` directory.
+1. (optional) Change the deletion policy of the DigitalOcean Block Storage `VolumeSnapshotClass` to `Retain` to ensure that newly created snapshots are not removed: `kubectl patch volumesnapshotclass do-block-storage --patch 'deletionPolicy: Retain' --type merge`
+1. Set all existing `VolumeSnapshotContent` objects to retain the snapshots on deletion: `kubectl get volumesnapshotcontent -o name --no-headers | xargs -n 1 kubectl patch --patch '{ "spec": { "deletionPolicy": "Retain" } }'`
+1. Upgrade / re-create the cluster.
+1. Ensure that all v1alpha1 snapshot CRDs and associated snapshot objects are removed. (It is not possible to run them concurrently with the v1beta1 objects.)
+1. Ensure that the v1beta1 CRDs and DigitalOcean Block Storage `VolumeSnapshotClass` exist.
+1. Re-import the previous snapshots: `kubectl apply -f snapshot_objects`.
+
+## Limitations
+
+No object metadata other than the name and namespace will be transferred over from v1alpha1 to v1beta1.

cmd/migrate-snapshots/main.go

@@ -0,0 +1,345 @@
+package main
+
+import (
+	"context"
+	"flag"
+	"fmt"
+	"os"
+	"path/filepath"
+
+	v1beta1snapshot "github.com/kubernetes-csi/external-snapshotter/v2/pkg/apis/volumesnapshot/v1beta1"
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
+	"k8s.io/apimachinery/pkg/runtime/schema"
+	"k8s.io/client-go/dynamic"
+	"k8s.io/client-go/tools/clientcmd"
+)
+
+const (
+	dobsDriverNewStyle = "dobs.csi.digitalocean.com"
+	dobsDriverOldStyle = "com.digitalocean.csi.dobs"
+)
+
+var (
+	alphaSnapClassRes   = schema.GroupVersionResource{Group: "snapshot.storage.k8s.io", Version: "v1alpha1", Resource: "volumesnapshotclasses"}
+	alphaSnapContentRes = schema.GroupVersionResource{Group: "snapshot.storage.k8s.io", Version: "v1alpha1", Resource: "volumesnapshotcontents"}
+	alphaSnapshotRes    = schema.GroupVersionResource{Group: "snapshot.storage.k8s.io", Version: "v1alpha1", Resource: "volumesnapshots"}
+)
+
+var errNoVolumeSnapshotClassFound = fmt.Errorf("could not find v1alpha1/VolumeStorageClass for driver %q", dobsDriverNewStyle)
+
+type snapshotClassDefinition struct {
+	dobsDriver         string
+	dobsSnapClassName  string
+	dobsIsDefaultClass bool
+}
+
+type params struct {
+	kubeconfig string
+	server     string
+	directory  string
+}
+
+var p params
+
+func errExit(err error) {
+	fmt.Fprintf(os.Stderr, "Failed to migrate snapshots: %s\n", err)
+	os.Exit(1)
+}
+
+func main() {
+	var defaultKubeConfigPath string
+	home, err := os.UserHomeDir()
+	if err == nil {
+		defaultKubeConfigPath = filepath.Join(home, ".kube", "config")
+	}
+
+	flag.StringVar(&p.kubeconfig, "kubeconfig", defaultKubeConfigPath, "(optional) absolute path to the kubeconfig file")
+	flag.StringVar(&p.server, "server", "", "(optional) address and port of the Kubernetes API server")
+	flag.StringVar(&p.directory, "directory", "", "the top-level directory to write YAML-marshaled objects into, or stdout if omitted")
+
+	flag.Parse()
+
+	if envVar := os.Getenv("KUBECONFIG"); envVar != "" {
+		p.kubeconfig = envVar
+	}
+
+	if p.server != "" {
+		p.kubeconfig = ""
+	}
+
+	if err := run(p); err != nil {
+		errExit(err)
+	}
+}
+
+func run(p params) error {
+	config, err := clientcmd.BuildConfigFromFlags(p.server, p.kubeconfig)
+	if err != nil {
+		return fmt.Errorf("failed to create config from flags: %s", err)
+	}
+
+	dynClient, err := dynamic.NewForConfig(config)
+	if err != nil {
+		return fmt.Errorf("failed to create dynamic client for v1alpha1 snapshots: %s", err)
+	}
+
+	volWriter, err := newVolumeSnapshotWriter(p.directory)
+	if err != nil {
+		return fmt.Errorf("failed to create volume snapshot writer: %s", err)
+	}
+
+	err = writeVolumeSnapshotObjects(context.TODO(), dynClient, volWriter)
+	if err != nil {
+		return fmt.Errorf("failed to write volume snapshot objects: %s", err)
+	}
+
+	return nil
+}
+
+func writeVolumeSnapshotObjects(ctx context.Context, client dynamic.Interface, volWriter *volumeSnapshotWriter) error {
+	snapClassDef, err := analyzeVolumeSnapshotClass(ctx, client.Resource(alphaSnapClassRes))
+	if err != nil {
+		return err
+	}
+	fmt.Printf("Found VolumeSnapshotClass for DigitalOcean Block Storage (driver=%s, name=%s, default class? %t)\n", snapClassDef.dobsDriver, snapClassDef.dobsSnapClassName, snapClassDef.dobsIsDefaultClass)
+
+	num, err := writeVolumeSnapshotContents(ctx, client.Resource(alphaSnapContentRes), snapClassDef, volWriter)
+	if err != nil {
+		return err
+	}
+	fmt.Printf("Converted %d VolumeSnapshotContent object(s)\n", num)
+
+	num, err = writeVolumeSnapshots(ctx, client.Resource(alphaSnapshotRes), snapClassDef, volWriter)
+	if err != nil {
+		return err
+	}
+
+	fmt.Printf("Converted %d VolumeSnapshot object(s)\n", num)
+	return nil
+}
+
+// analyzeVolumeSnapshotClass extracts metadata from the DigitalOcean Block
+// Storage (DOBS) VolumeSnapshotClass needed to write other volume snapshot
+// objects.
+func analyzeVolumeSnapshotClass(ctx context.Context, client dynamic.NamespaceableResourceInterface) (*snapshotClassDefinition, error) {
+	alphaSnapClasses, err := client.List(ctx, metav1.ListOptions{})
+	if err != nil {
+		return nil, fmt.Errorf("failed to list v1alpha1/VolumeSnapshotClass: %s", err)
+	}
+
+	for _, alphaSnapClass := range alphaSnapClasses.Items {
+		name, _, err := unstructured.NestedString(alphaSnapClass.Object, "metadata", "name")
+		if err != nil {
+			return nil, fmt.Errorf("failed to get v1alpha1/VolumeSnapshotClass.metadata.name: %s", err)
+		}
+
+		snapshotter, found, err := unstructured.NestedString(alphaSnapClass.Object, "snapshotter")
+		if err != nil {
+			return nil, fmt.Errorf("failed to get v1alpha1/VolumeSnapshotClass.snapshotter for %s: %s", name, err)
+		}
+		if !found {
+			return nil, fmt.Errorf("v1alpha1 VolumeSnapshotClass.snapshotter does not exist for %s", name)
+		}
+		if snapshotter == dobsDriverNewStyle || snapshotter == dobsDriverOldStyle {
+			dobsDriver := snapshotter
+			dobsSnapClassName := name
+
+			isDefaultClass, _, err := unstructured.NestedString(alphaSnapClass.Object, "metadata", "annotations", "snapshot.storage.kubernetes.io/is-default-class")
+			if err != nil {
+				return nil, fmt.Errorf("failed to get v1alpha1/VolumeSnapshotClass.metadataannotation.'snapshot.storage.kubernetes.io/is-default-class' for %s: %s", name, err)
+			}
+			dobsIsDefaultClass := isDefaultClass == "true"
+
+			return &snapshotClassDefinition{dobsDriver, dobsSnapClassName, dobsIsDefaultClass}, nil
+		}
+	}
+
+	return nil, errNoVolumeSnapshotClassFound
+}
+
+// writeVolumeSnapshotContents converts v1alpha1 VolumeSnapshotContent objects
+// into their v1beta1 equivalents and writes them out.
+func writeVolumeSnapshotContents(ctx context.Context, client dynamic.NamespaceableResourceInterface, snapClassDef *snapshotClassDefinition, volWriter *volumeSnapshotWriter) (num int, err error) {
+	alphaSnapContents, err := client.List(ctx, metav1.ListOptions{})
+	if err != nil {
+		return 0, fmt.Errorf("failed to list v1alpha1/VolumeSnapshotContents: %s", err)
+	}
+
+	for _, alphaSnapContent := range alphaSnapContents.Items {
+		name, _, err := unstructured.NestedString(alphaSnapContent.Object, "metadata", "name")
+		if err != nil {
+			return 0, fmt.Errorf("failed get v1alpha1/VolumeSnapshotContent.metadata.name: %s", err)
+		}
+
+		volSnapClassNameStr, found, err := unstructured.NestedString(alphaSnapContent.Object, "spec", "snapshotClassName")
+		if err != nil {
+			return 0, fmt.Errorf("failed get v1alpha1/VolumeSnapshotContent.spec.snapshotClassName for %s: %s", name, err)
+		}
+		if !found && !snapClassDef.dobsIsDefaultClass {
+			fmt.Printf("Skipping v1alpha1/VolumeSnapshotContent %q because spec.snapshotClassName is missing and DOBS is not the default driver\n", name)
+			continue
+		}
+
+		var volSnapClassName *string
+		if found {
+			if volSnapClassNameStr != snapClassDef.dobsSnapClassName {
+				fmt.Printf("Skipping v1alpha1/VolumeSnapshotContent %q because object is managed by non-DOBS driver %q\n", name, volSnapClassNameStr)
+				continue
+			}
+
+			volSnapClassName = &volSnapClassNameStr
+		}
+
+		volSnapRefName, _, err := unstructured.NestedString(alphaSnapContent.Object, "spec", "volumeSnapshotRef", "name")
+		if err != nil {
+			return 0, fmt.Errorf("failed get v1alpha1/VolumeSnapshotContent.spec.volumeSnapshotRef.name for %s: %s", name, err)
+		}
+		volSnapRefNamespace, _, err := unstructured.NestedString(alphaSnapContent.Object, "spec", "volumeSnapshotRef", "namespace")
+		if err != nil {
+			return 0, fmt.Errorf("failed get v1alpha1/VolumeSnapshotContent.spec.volumeSnapshotRef.namespace for %s: %s", name, err)
+		}
+
+		deletionPolicy, _, err := unstructured.NestedString(alphaSnapContent.Object, "spec", "deletionPolicy")
+		if err != nil {
+			return 0, fmt.Errorf("failed get v1alpha1/VolumeSnapshotContent.spec.deletionPolicy for %s: %s", name, err)
+		}
+
+		csiVolSnapSourceSnapHandleStr, found, err := unstructured.NestedString(alphaSnapContent.Object, "spec", "csiVolumeSnapshotSource", "snapshotHandle")
+		if err != nil {
+			return 0, fmt.Errorf("failed get v1alpha1/VolumeSnapshotContent.spec.csiVolumeSnapshotSource.snapshotHandle for %s: %s", name, err)
+		}
+		var csiVolSnapSourceSnapHandle *string
+		if found {
+			csiVolSnapSourceSnapHandle = &csiVolSnapSourceSnapHandleStr
+		}
+
+		betaSnapContent := v1beta1snapshot.VolumeSnapshotContent{
+			TypeMeta: metav1.TypeMeta{
+				Kind:       "VolumeSnapshotContent",
+				APIVersion: v1beta1snapshot.SchemeGroupVersion.String(),
+			},
+			ObjectMeta: metav1.ObjectMeta{
+				Name: name,
+			},
+			Spec: v1beta1snapshot.VolumeSnapshotContentSpec{
+				VolumeSnapshotRef: corev1.ObjectReference{
+					Name:      volSnapRefName,
+					Namespace: volSnapRefNamespace,
+				},
+				DeletionPolicy:          v1beta1snapshot.DeletionPolicy(deletionPolicy),
+				Driver:                  snapClassDef.dobsDriver,
+				VolumeSnapshotClassName: volSnapClassName,
+				Source: v1beta1snapshot.VolumeSnapshotContentSource{
+					SnapshotHandle: csiVolSnapSourceSnapHandle,
+				},
+			},
+		}
+
+		fmt.Printf("--- Writing VolumeSnapshotContent %q\n", name)
+		err = volWriter.writeVolumeSnapshotContent(betaSnapContent)
+		if err != nil {
+			return 0, fmt.Errorf("failed to write VolumeSnapshotContent %q: %s", name, err)
+		}
+
+		num++
+	}
+
+	return num, nil
+}
+
+// writeVolumeSnapshots converts v1alpha1 VolumeSnapshot objects into their
+// v1beta1 equivalents and writes them out.
+func writeVolumeSnapshots(ctx context.Context, client dynamic.NamespaceableResourceInterface, snapClassDef *snapshotClassDefinition, volWriter *volumeSnapshotWriter) (num int, err error) {
+	alphaSnapshots, err := client.List(ctx, metav1.ListOptions{})
+	if err != nil {
+		return 0, fmt.Errorf("failed to list v1alpha1/VolumeSnapshots: %s", err)
+	}
+
+	for _, alphaSnapshot := range alphaSnapshots.Items {
+		name, _, err := unstructured.NestedString(alphaSnapshot.Object, "metadata", "name")
+		if err != nil {
+			return 0, fmt.Errorf("failed get v1alpha1/VolumeSnapshot.metadata.name: %s", err)
+		}
+
+		volSnapClassNameStr, found, err := unstructured.NestedString(alphaSnapshot.Object, "spec", "snapshotClassName")
+		if err != nil {
+			return 0, fmt.Errorf("failed get v1alpha1/VolumeSnapshot.spec.snapshotClassName for %s: %s", name, err)
+		}
+		if !found && !snapClassDef.dobsIsDefaultClass {
+			fmt.Printf("Skipping v1alpha1/VolumeSnapshot %q because spec.snapshotClassName is missing and DOBS is not the default driver\n", name)
+			continue
+		}
+
+		var volSnapClassName *string
+		if found {
+			if volSnapClassNameStr != snapClassDef.dobsSnapClassName {
+				fmt.Printf("Skipping v1alpha1/VolumeSnapshot %q because object is managed by non-DOBS driver %q\n", name, volSnapClassNameStr)
+				continue
+			}
+
+			volSnapClassName = &volSnapClassNameStr
+		}
+
+		namespace, _, err := unstructured.NestedString(alphaSnapshot.Object, "metadata", "namespace")
+		if err != nil {
+			return 0, fmt.Errorf("failed get v1alpha1/VolumeSnapshot.metadata.namespace for %s: %s", name, err)
+		}
+
+		var persistentVolumeClaimName *string
+		persistentVolumeClaimNameStr, found, err := unstructured.NestedString(alphaSnapshot.Object, "spec", "source", "name")
+		if err != nil {
+			return 0, fmt.Errorf("failed get v1alpha1/VolumeSnapshot.spec.source.name for %s: %s", name, err)
+		}
+		if found {
+			persistentVolumeClaimName = &persistentVolumeClaimNameStr
+		}
+
+		var snapshotContentName *string
+		snapshotContentNameStr, found, err := unstructured.NestedString(alphaSnapshot.Object, "spec", "snapshotContentName")
+		if err != nil {
+			return 0, fmt.Errorf("failed get v1alpha1/VolumeSnapshot.spec.snapshotContentName for %s: %s", name, err)
+		}
+		if found {
+			snapshotContentName = &snapshotContentNameStr
+		}
+
+		betaSnapshot := v1beta1snapshot.VolumeSnapshot{
+			TypeMeta: metav1.TypeMeta{
+				Kind:       "VolumeSnapshot",
+				APIVersion: v1beta1snapshot.SchemeGroupVersion.String(),
+			},
+			ObjectMeta: metav1.ObjectMeta{
+				Name:      name,
+				Namespace: namespace,
+			},
+			Spec: v1beta1snapshot.VolumeSnapshotSpec{
+				Source: v1beta1snapshot.VolumeSnapshotSource{
+					VolumeSnapshotContentName: snapshotContentName,
+				},
+				VolumeSnapshotClassName: volSnapClassName,
+			},
+		}
+
+		if snapshotContentName == nil {
+			// The VolumeSnapshot might not be bound to a VolumeSnapshotContent
+			// yet (either due to delay or pathological reasons). Associate any
+			// PersistentVolumeClaim name we may have instead. (Note that
+			// exactly one of VolumeSnapshotContentName and
+			// PersistentVolumeClaimName may be defined on a VolumeSnapshot at
+			// any given time.)
+			betaSnapshot.Spec.Source.PersistentVolumeClaimName = persistentVolumeClaimName
+		}
+
+		fmt.Printf("--- Writing VolumeSnapshot %q\n", name)
+		err = volWriter.writeVolumeSnapshot(betaSnapshot)
+		if err != nil {
+			return 0, fmt.Errorf("failed to write VolumeSnapshot %q: %s", name, err)
+		}
+
+		num++
+	}
+
+	return num, nil
+}