Skip to content

Commit

Permalink
Add support for Velero 1.14 (#4782)
Browse files Browse the repository at this point in the history
  • Loading branch information
@sgalsaleh
sgalsaleh authored Jul 26, 2024
1 parent 1fec2a7 commit 3829fb0
Show file tree
Hide file tree
Showing 3 changed files with 238 additions and 41 deletions.
6 changes: 3 additions & 3 deletions pkg/embeddedcluster/util.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,8 +18,8 @@ import (
)

const (
seaweedfsNamespace = "seaweedfs"
seaweedfsS3SVCName = "ec-seaweedfs-s3"
SeaweedfsNamespace = "seaweedfs"
SeaweedfsS3SVCName = "ec-seaweedfs-s3"
)

// ErrNoInstallations is returned when no installation object is found in the cluster.
Expand Down Expand Up @@ -101,7 +101,7 @@ func ClusterConfig(ctx context.Context, kbClient kbclient.Client) (*embeddedclus
}

func GetSeaweedFSS3ServiceIP(ctx context.Context, kbClient kbclient.Client) (string, error) {
nsn := k8stypes.NamespacedName{Name: seaweedfsS3SVCName, Namespace: seaweedfsNamespace}
nsn := k8stypes.NamespacedName{Name: SeaweedfsS3SVCName, Namespace: SeaweedfsNamespace}
var svc corev1.Service
if err := kbClient.Get(ctx, nsn, &svc); err != nil && !k8serrors.IsNotFound(err) {
return "", fmt.Errorf("failed to get seaweedfs s3 service: %w", err)
Expand Down
71 changes: 47 additions & 24 deletions pkg/kotsadmsnapshot/backup.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ import (

units "github.com/docker/go-units"
"github.com/pkg/errors"
embeddedclusterv1beta1 "github.com/replicatedhq/embedded-cluster-kinds/apis/v1beta1"
downstreamtypes "github.com/replicatedhq/kots/pkg/api/downstream/types"
apptypes "github.com/replicatedhq/kots/pkg/app/types"
"github.com/replicatedhq/kots/pkg/embeddedcluster"
Expand All @@ -35,6 +36,7 @@ import (
"k8s.io/apimachinery/pkg/fields"
"k8s.io/apimachinery/pkg/labels"
"k8s.io/client-go/kubernetes"
kbclient "sigs.k8s.io/controller-runtime/pkg/client"
)

func CreateApplicationBackup(ctx context.Context, a *apptypes.App, isScheduled bool) (*velerov1.Backup, error) {
Expand Down Expand Up @@ -128,7 +130,7 @@ func CreateApplicationBackup(ctx context.Context, a *apptypes.App, isScheduled b
includedNamespaces = append(includedNamespaces, veleroBackup.Spec.IncludedNamespaces...)
includedNamespaces = append(includedNamespaces, kotsKinds.KotsApplication.Spec.AdditionalNamespaces...)

veleroBackup.Spec.IncludedNamespaces = prepareIncludedNamespaces(includedNamespaces, util.IsEmbeddedCluster())
veleroBackup.Spec.IncludedNamespaces = prepareIncludedNamespaces(includedNamespaces)

snapshotTrigger := "manual"
if isScheduled {
Expand Down Expand Up @@ -382,21 +384,14 @@ func CreateInstanceBackup(ctx context.Context, cluster *downstreamtypes.Downstre
if err != nil {
return nil, fmt.Errorf("failed to get current installation: %w", err)
}
seaweedFSS3ServiceIP, err := embeddedcluster.GetSeaweedFSS3ServiceIP(ctx, kbClient)
ecAnnotations, err := ecBackupAnnotations(ctx, kbClient, installation)
if err != nil {
return nil, fmt.Errorf("failed to get seaweedfs s3 service ip: %w", err)
return nil, fmt.Errorf("failed to get embedded cluster backup annotations: %w", err)
}
if seaweedFSS3ServiceIP != "" {
backupAnnotations["kots.io/embedded-cluster-seaweedfs-s3-ip"] = seaweedFSS3ServiceIP
}
backupAnnotations["kots.io/embedded-cluster"] = "true"
backupAnnotations["kots.io/embedded-cluster-id"] = util.EmbeddedClusterID()
backupAnnotations["kots.io/embedded-cluster-version"] = util.EmbeddedClusterVersion()
backupAnnotations["kots.io/embedded-cluster-is-ha"] = strconv.FormatBool(installation.Spec.HighAvailability)
if installation.Spec.Network != nil {
backupAnnotations["kots.io/embedded-cluster-pod-cidr"] = installation.Spec.Network.PodCIDR
backupAnnotations["kots.io/embedded-cluster-service-cidr"] = installation.Spec.Network.ServiceCIDR
for k, v := range ecAnnotations {
backupAnnotations[k] = v
}
includedNamespaces = append(includedNamespaces, ecIncludedNamespaces(installation)...)
}

includeClusterResources := true
Expand All @@ -409,7 +404,7 @@ func CreateInstanceBackup(ctx context.Context, cluster *downstreamtypes.Downstre
},
Spec: velerov1.BackupSpec{
StorageLocation: "default",
IncludedNamespaces: prepareIncludedNamespaces(includedNamespaces, util.IsEmbeddedCluster()),
IncludedNamespaces: prepareIncludedNamespaces(includedNamespaces),
ExcludedNamespaces: excludedNamespaces,
IncludeClusterResources: &includeClusterResources,
OrLabelSelectors: instanceBackupLabelSelectors(util.IsEmbeddedCluster()),
Expand Down Expand Up @@ -985,11 +980,47 @@ func mergeLabelSelector(kots metav1.LabelSelector, app metav1.LabelSelector) met
return kots
}

// ecBackupAnnotations returns the annotations that should be added to an embedded cluster backup
func ecBackupAnnotations(ctx context.Context, kbClient kbclient.Client, in *embeddedclusterv1beta1.Installation) (map[string]string, error) {
annotations := map[string]string{}

seaweedFSS3ServiceIP, err := embeddedcluster.GetSeaweedFSS3ServiceIP(ctx, kbClient)
if err != nil {
return nil, fmt.Errorf("failed to get seaweedfs s3 service ip: %w", err)
}
if seaweedFSS3ServiceIP != "" {
annotations["kots.io/embedded-cluster-seaweedfs-s3-ip"] = seaweedFSS3ServiceIP
}

annotations["kots.io/embedded-cluster"] = "true"
annotations["kots.io/embedded-cluster-id"] = util.EmbeddedClusterID()
annotations["kots.io/embedded-cluster-version"] = util.EmbeddedClusterVersion()
annotations["kots.io/embedded-cluster-is-ha"] = strconv.FormatBool(in.Spec.HighAvailability)

if in.Spec.Network != nil {
annotations["kots.io/embedded-cluster-pod-cidr"] = in.Spec.Network.PodCIDR
annotations["kots.io/embedded-cluster-service-cidr"] = in.Spec.Network.ServiceCIDR
}

return annotations, nil
}

// ecIncludedNamespaces returns the namespaces that should be included in an embedded cluster backup
func ecIncludedNamespaces(in *embeddedclusterv1beta1.Installation) []string {
includedNamespaces := []string{"embedded-cluster", "kube-system", "openebs"}
if in.Spec.AirGap {
includedNamespaces = append(includedNamespaces, "registry")
if in.Spec.HighAvailability {
includedNamespaces = append(includedNamespaces, "seaweedfs")
}
}
return includedNamespaces
}

// Prepares the list of unique namespaces that will be included in a backup. Empty namespaces are excluded.
// If a wildcard is specified, any specific namespaces will not be included since the backup will include all namespaces.
// Velero does not allow for both a wildcard and specific namespaces and will consider the backup invalid if both are present.
// If this is an embedded-cluster installation, the "embedded-cluster", "openebs" and "kube-system" namespaces will be included.
func prepareIncludedNamespaces(namespaces []string, isEC bool) []string {
func prepareIncludedNamespaces(namespaces []string) []string {
uniqueNamespaces := make(map[string]bool)
for _, n := range namespaces {
if n == "" {
Expand All @@ -1000,14 +1031,6 @@ func prepareIncludedNamespaces(namespaces []string, isEC bool) []string {
uniqueNamespaces[n] = true
}

if isEC {
uniqueNamespaces["embedded-cluster"] = true
uniqueNamespaces["kube-system"] = true
uniqueNamespaces["openebs"] = true
uniqueNamespaces["registry"] = true
uniqueNamespaces["seaweedfs"] = true
}

includedNamespaces := make([]string, len(uniqueNamespaces))
i := 0
for k := range uniqueNamespaces {
Expand Down
202 changes: 188 additions & 14 deletions pkg/kotsadmsnapshot/backup_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,8 @@ import (
"context"
"testing"

embeddedclusterv1beta1 "github.com/replicatedhq/embedded-cluster-kinds/apis/v1beta1"
"github.com/replicatedhq/kots/pkg/embeddedcluster"
kotsadmtypes "github.com/replicatedhq/kots/pkg/kotsadm/types"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
Expand All @@ -17,14 +19,15 @@ import (
"k8s.io/client-go/kubernetes"
"k8s.io/client-go/kubernetes/fake"
coretest "k8s.io/client-go/testing"
kbclient "sigs.k8s.io/controller-runtime/pkg/client"
fakekbclient "sigs.k8s.io/controller-runtime/pkg/client/fake"
)

func TestPrepareIncludedNamespaces(t *testing.T) {
tests := []struct {
name string
namespaces []string
want []string
isEC bool
}{
{
name: "empty",
Expand Down Expand Up @@ -76,23 +79,11 @@ func TestPrepareIncludedNamespaces(t *testing.T) {
namespaces: []string{"*", "", "test"},
want: []string{"*"},
},
{
name: "wildcard with embedded cluster",
namespaces: []string{"*", "test"},
want: []string{"*"},
isEC: true,
},
{
name: "embedded-cluster install",
namespaces: []string{"test", "abcapp"},
want: []string{"test", "abcapp", "embedded-cluster", "kube-system", "openebs", "registry", "seaweedfs"},
isEC: true,
},
}

for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got := prepareIncludedNamespaces(tt.namespaces, tt.isEC)
got := prepareIncludedNamespaces(tt.namespaces)
if !assert.ElementsMatch(t, tt.want, got) {
t.Errorf("prepareIncludedNamespaces() = %v, want %v", got, tt.want)
}
Expand Down Expand Up @@ -690,3 +681,186 @@ func Test_instanceBackupLabelSelectors(t *testing.T) {
})
}
}

func Test_ecBackupAnnotations(t *testing.T) {
scheme := runtime.NewScheme()
corev1.AddToScheme(scheme)
embeddedclusterv1beta1.AddToScheme(scheme)

tests := []struct {
name string
kbClient kbclient.Client
in *embeddedclusterv1beta1.Installation
env map[string]string
want map[string]string
}{
{
name: "basic",
kbClient: fakekbclient.NewClientBuilder().WithScheme(scheme).Build(),
in: &embeddedclusterv1beta1.Installation{},
env: map[string]string{
"EMBEDDED_CLUSTER_ID": "embedded-cluster-id",
"EMBEDDED_CLUSTER_VERSION": "embedded-cluster-version",
},
want: map[string]string{
"kots.io/embedded-cluster": "true",
"kots.io/embedded-cluster-id": "embedded-cluster-id",
"kots.io/embedded-cluster-version": "embedded-cluster-version",
"kots.io/embedded-cluster-is-ha": "false",
},
},
{
name: "online ha",
kbClient: fakekbclient.NewClientBuilder().WithScheme(scheme).Build(),
in: &embeddedclusterv1beta1.Installation{
Spec: embeddedclusterv1beta1.InstallationSpec{
HighAvailability: true,
},
},
env: map[string]string{
"EMBEDDED_CLUSTER_ID": "embedded-cluster-id",
"EMBEDDED_CLUSTER_VERSION": "embedded-cluster-version",
},
want: map[string]string{
"kots.io/embedded-cluster": "true",
"kots.io/embedded-cluster-id": "embedded-cluster-id",
"kots.io/embedded-cluster-version": "embedded-cluster-version",
"kots.io/embedded-cluster-is-ha": "true",
},
},
{
name: "airgap ha",
kbClient: fakekbclient.NewClientBuilder().WithScheme(scheme).WithObjects(
&corev1.Service{
ObjectMeta: metav1.ObjectMeta{
Name: embeddedcluster.SeaweedfsS3SVCName,
Namespace: embeddedcluster.SeaweedfsNamespace,
},
Spec: corev1.ServiceSpec{
ClusterIP: "10.96.0.10",
},
},
).Build(),
in: &embeddedclusterv1beta1.Installation{
Spec: embeddedclusterv1beta1.InstallationSpec{
HighAvailability: true,
AirGap: true,
},
},
env: map[string]string{
"EMBEDDED_CLUSTER_ID": "embedded-cluster-id",
"EMBEDDED_CLUSTER_VERSION": "embedded-cluster-version",
},
want: map[string]string{
"kots.io/embedded-cluster": "true",
"kots.io/embedded-cluster-id": "embedded-cluster-id",
"kots.io/embedded-cluster-version": "embedded-cluster-version",
"kots.io/embedded-cluster-is-ha": "true",
"kots.io/embedded-cluster-seaweedfs-s3-ip": "10.96.0.10",
},
},
{
name: "with pod and service cidrs",
kbClient: fakekbclient.NewClientBuilder().WithScheme(scheme).Build(),
in: &embeddedclusterv1beta1.Installation{
Spec: embeddedclusterv1beta1.InstallationSpec{
Network: &embeddedclusterv1beta1.NetworkSpec{
PodCIDR: "10.128.0.0/20",
ServiceCIDR: "10.129.0.0/20",
},
},
},
env: map[string]string{
"EMBEDDED_CLUSTER_ID": "embedded-cluster-id",
"EMBEDDED_CLUSTER_VERSION": "embedded-cluster-version",
},
want: map[string]string{
"kots.io/embedded-cluster": "true",
"kots.io/embedded-cluster-id": "embedded-cluster-id",
"kots.io/embedded-cluster-version": "embedded-cluster-version",
"kots.io/embedded-cluster-is-ha": "false",
"kots.io/embedded-cluster-pod-cidr": "10.128.0.0/20",
"kots.io/embedded-cluster-service-cidr": "10.129.0.0/20",
},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
req := require.New(t)
for k, v := range tt.env {
t.Setenv(k, v)
}
got, err := ecBackupAnnotations(context.TODO(), tt.kbClient, tt.in)
req.NoError(err)
req.Equal(tt.want, got)
})
}
}

func Test_ecIncludedNamespaces(t *testing.T) {
tests := []struct {
name string
in *embeddedclusterv1beta1.Installation
want []string
}{
{
name: "online",
in: &embeddedclusterv1beta1.Installation{},
want: []string{
"embedded-cluster",
"kube-system",
"openebs",
},
},
{
name: "online ha",
in: &embeddedclusterv1beta1.Installation{
Spec: embeddedclusterv1beta1.InstallationSpec{
HighAvailability: true,
},
},
want: []string{
"embedded-cluster",
"kube-system",
"openebs",
},
},
{
name: "airgap",
in: &embeddedclusterv1beta1.Installation{
Spec: embeddedclusterv1beta1.InstallationSpec{
AirGap: true,
},
},
want: []string{
"embedded-cluster",
"kube-system",
"openebs",
"registry",
},
},
{
name: "airgap ha",
in: &embeddedclusterv1beta1.Installation{
Spec: embeddedclusterv1beta1.InstallationSpec{
HighAvailability: true,
AirGap: true,
},
},
want: []string{
"embedded-cluster",
"kube-system",
"openebs",
"registry",
"seaweedfs",
},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
req := require.New(t)
got := ecIncludedNamespaces(tt.in)
req.Equal(tt.want, got)
})
}
}

0 comments on commit 3829fb0

Please sign in to comment.