Skip to content
This repository has been archived by the owner on Oct 11, 2024. It is now read-only.

Commit

Permalink
Groups version bump (#4561)
Browse files Browse the repository at this point in the history
Bump the backup version and force a full backup if
there's a backup for teams/groups that has base(s)
from an older version of corso

This will avoid propagating older details formats
forward. Those formats don't have all the data
newer formats do

This is mostly a stop-gap, a more robust solution
can be added later

Manually tested that it forces a full backup

---

#### Does this PR need a docs update or release note?

- [ ] ✅ Yes, it's included
- [ ] 🕐 Yes, but in a later PR
- [x] ⛔ No

#### Type of change

- [ ] 🌻 Feature
- [x] 🐛 Bugfix
- [ ] 🗺️ Documentation
- [ ] 🤖 Supportability/Tests
- [ ] 💻 CI/Deployment
- [ ] 🧹 Tech Debt/Cleanup

#### Issue(s)

* #4569

#### Test Plan

- [x] 💪 Manual
- [ ] ⚡ Unit test
- [ ] 💚 E2E
  • Loading branch information
ashmrtn authored Oct 27, 2023
1 parent fd887f4 commit f61448d
Show file tree
Hide file tree
Showing 13 changed files with 819 additions and 19 deletions.
19 changes: 19 additions & 0 deletions src/internal/kopia/backup_bases.go
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,9 @@ type BackupBases interface {
// MinBackupVersion returns the lowest version of all merge backups in the
// BackupBases.
MinBackupVersion() int
// MinAssisttVersion returns the lowest version of all assist backups in the
// BackupBases.
MinAssistVersion() int
// MergeBackupBases takes another BackupBases and merges it's contained assist
// and merge bases into this BackupBases. The passed in BackupBases is
// considered an older alternative to this BackupBases meaning bases from
Expand Down Expand Up @@ -119,6 +122,22 @@ func (bb *backupBases) MinBackupVersion() int {
return min
}

func (bb *backupBases) MinAssistVersion() int {
min := version.NoBackup

if bb == nil {
return min
}

for _, base := range bb.assistBases {
if min == version.NoBackup || base.Backup.Version < min {
min = base.Backup.Version
}
}

return min
}

func (bb backupBases) MergeBases() []BackupBase {
return slices.Clone(bb.mergeBases)
}
Expand Down
113 changes: 101 additions & 12 deletions src/internal/kopia/backup_bases_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -92,20 +92,23 @@ func TestBackupBasesUnitSuite(t *testing.T) {
suite.Run(t, &BackupBasesUnitSuite{Suite: tester.NewUnitSuite(t)})
}

func (suite *BackupBasesUnitSuite) TestMinBackupVersion() {
func (suite *BackupBasesUnitSuite) TestBackupBases_minVersions() {
table := []struct {
name string
bb *backupBases
expectedVersion int
name string
bb *backupBases
expectedBackupVersion int
expectedAssistVersion int
}{
{
name: "Nil BackupBase",
expectedVersion: version.NoBackup,
name: "Nil BackupBase",
expectedBackupVersion: version.NoBackup,
expectedAssistVersion: version.NoBackup,
},
{
name: "No Backups",
bb: &backupBases{},
expectedVersion: version.NoBackup,
name: "No Backups",
bb: &backupBases{},
expectedBackupVersion: version.NoBackup,
expectedAssistVersion: version.NoBackup,
},
{
name: "Unsorted Backups",
Expand All @@ -128,7 +131,8 @@ func (suite *BackupBasesUnitSuite) TestMinBackupVersion() {
},
},
},
expectedVersion: 0,
expectedBackupVersion: 0,
expectedAssistVersion: version.NoBackup,
},
{
name: "Only Assist Bases",
Expand All @@ -151,12 +155,97 @@ func (suite *BackupBasesUnitSuite) TestMinBackupVersion() {
},
},
},
expectedVersion: version.NoBackup,
expectedBackupVersion: version.NoBackup,
expectedAssistVersion: 0,
},
{
name: "Assist and Merge Bases, min merge",
bb: &backupBases{
mergeBases: []BackupBase{
{
Backup: &backup.Backup{
Version: 1,
},
},
{
Backup: &backup.Backup{
Version: 5,
},
},
{
Backup: &backup.Backup{
Version: 3,
},
},
},
assistBases: []BackupBase{
{
Backup: &backup.Backup{
Version: 4,
},
},
{
Backup: &backup.Backup{
Version: 2,
},
},
{
Backup: &backup.Backup{
Version: 6,
},
},
},
},
expectedBackupVersion: 1,
expectedAssistVersion: 2,
},
{
name: "Assist and Merge Bases, min assist",
bb: &backupBases{
mergeBases: []BackupBase{
{
Backup: &backup.Backup{
Version: 7,
},
},
{
Backup: &backup.Backup{
Version: 5,
},
},
{
Backup: &backup.Backup{
Version: 3,
},
},
},
assistBases: []BackupBase{
{
Backup: &backup.Backup{
Version: 4,
},
},
{
Backup: &backup.Backup{
Version: 2,
},
},
{
Backup: &backup.Backup{
Version: 6,
},
},
},
},
expectedBackupVersion: 3,
expectedAssistVersion: 2,
},
}
for _, test := range table {
suite.Run(test.name, func() {
assert.Equal(suite.T(), test.expectedVersion, test.bb.MinBackupVersion())
t := suite.T()
assert.Equal(t, test.expectedBackupVersion, test.bb.MinBackupVersion(), "backup")
assert.Equal(t, test.expectedAssistVersion, test.bb.MinAssistVersion(), "assist")
})
}
}
Expand Down
4 changes: 2 additions & 2 deletions src/internal/m365/onedrive_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -228,7 +228,7 @@ func (suite *SharePointIntegrationSuite) TestLinkSharesInheritanceRestoreAndBack

func (suite *SharePointIntegrationSuite) TestRestoreFolderNamedFolderRegression() {
// No reason why it couldn't work with previous versions, but this is when it got introduced.
testRestoreFolderNamedFolderRegression(suite, version.All8MigrateUserPNToID)
testRestoreFolderNamedFolderRegression(suite, version.Backup)
}

// ---------------------------------------------------------------------------
Expand Down Expand Up @@ -292,7 +292,7 @@ func (suite *OneDriveIntegrationSuite) TestLinkSharesInheritanceRestoreAndBackup

func (suite *OneDriveIntegrationSuite) TestRestoreFolderNamedFolderRegression() {
// No reason why it couldn't work with previous versions, but this is when it got introduced.
testRestoreFolderNamedFolderRegression(suite, version.All8MigrateUserPNToID)
testRestoreFolderNamedFolderRegression(suite, version.Backup)
}

// ---------------------------------------------------------------------------
Expand Down
4 changes: 2 additions & 2 deletions src/internal/m365/service/onedrive/restore_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@ func TestRestoreUnitSuite(t *testing.T) {
func (suite *RestoreUnitSuite) TestAugmentRestorePaths() {
// Adding a simple test here so that we can be sure that this
// function gets updated whenever we add a new version.
require.LessOrEqual(suite.T(), version.Backup, version.All8MigrateUserPNToID, "unsupported backup version")
require.LessOrEqual(suite.T(), version.Backup, version.Groups9Update, "unsupported backup version")

table := []struct {
name string
Expand Down Expand Up @@ -216,7 +216,7 @@ func (suite *RestoreUnitSuite) TestAugmentRestorePaths() {
func (suite *RestoreUnitSuite) TestAugmentRestorePaths_DifferentRestorePath() {
// Adding a simple test here so that we can be sure that this
// function gets updated whenever we add a new version.
require.LessOrEqual(suite.T(), version.Backup, version.All8MigrateUserPNToID, "unsupported backup version")
require.LessOrEqual(suite.T(), version.Backup, version.Groups9Update, "unsupported backup version")

type pathPair struct {
storage string
Expand Down
6 changes: 4 additions & 2 deletions src/internal/m365/service/onedrive/stub/stub.go
Original file line number Diff line number Diff line change
Expand Up @@ -217,7 +217,8 @@ func (c *collection) withFile(name string, fileData []byte, meta MetaData) (*col
c.Aux = append(c.Aux, md)

// v6+ current metadata design
case version.OneDrive6NameInMeta, version.OneDrive7LocationRef, version.All8MigrateUserPNToID:
case version.OneDrive6NameInMeta, version.OneDrive7LocationRef,
version.All8MigrateUserPNToID, version.Groups9Update:
item, err := FileWithData(
name+metadata.DataFileSuffix,
name+metadata.DataFileSuffix,
Expand Down Expand Up @@ -251,7 +252,8 @@ func (c *collection) withFile(name string, fileData []byte, meta MetaData) (*col
func (c *collection) withFolder(name string, meta MetaData) (*collection, error) {
switch c.BackupVersion {
case 0, version.OneDrive4DirIncludesPermissions, version.OneDrive5DirMetaNoName,
version.OneDrive6NameInMeta, version.OneDrive7LocationRef, version.All8MigrateUserPNToID:
version.OneDrive6NameInMeta, version.OneDrive7LocationRef,
version.All8MigrateUserPNToID, version.Groups9Update:
return c, nil

case version.OneDrive1DataAndMetaFiles, 2, version.OneDrive3IsMetaMarker:
Expand Down
28 changes: 28 additions & 0 deletions src/internal/operations/backup.go
Original file line number Diff line number Diff line change
Expand Up @@ -371,6 +371,34 @@ func (op *BackupOperation) do(
return nil, clues.Wrap(err, "producing manifests and metadata")
}

// Force full backups if the base is an older corso version. Those backups
// don't have all the data we want to pull forward.
//
// TODO(ashmrtn): We can push this check further down the stack to either:
// * the metadata fetch code to disable individual bases (requires a
// function to completely remove a base from the set)
// * the base finder code to skip over older bases (breaks isolation a bit
// by requiring knowledge of good/bad backup versions for different
// services)
if op.Selectors.PathService() == path.GroupsService {
if mans.MinBackupVersion() != version.NoBackup &&
mans.MinBackupVersion() < version.Groups9Update {
logger.Ctx(ctx).Info("dropping merge bases due to groups version change")

mans.DisableMergeBases()
mans.DisableAssistBases()

canUseMetadata = false
mdColls = nil
}

if mans.MinAssistVersion() != version.NoBackup &&
mans.MinAssistVersion() < version.Groups9Update {
logger.Ctx(ctx).Info("disabling assist bases due to groups version change")
mans.DisableAssistBases()
}
}

ctx = clues.Add(
ctx,
"can_use_metadata", canUseMetadata,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -61,6 +61,40 @@ func (suite *RestorePathTransformerUnitSuite) TestGetPaths() {
expectErr assert.ErrorAssertionFunc
expected []expectPaths
}{
{
name: "Groups List Errors v9",
// No version bump for the change so we always have to check for this.
backupVersion: version.Groups9Update,
input: []*details.Entry{
{
RepoRef: GroupsRootItemPath.RR.String(),
LocationRef: GroupsRootItemPath.Loc.String(),
ItemInfo: details.ItemInfo{
Groups: &details.GroupsInfo{
ItemType: details.SharePointList,
},
},
},
},
expectErr: assert.Error,
},
{
name: "Groups Page Errors v9",
// No version bump for the change so we always have to check for this.
backupVersion: version.Groups9Update,
input: []*details.Entry{
{
RepoRef: GroupsRootItemPath.RR.String(),
LocationRef: GroupsRootItemPath.Loc.String(),
ItemInfo: details.ItemInfo{
Groups: &details.GroupsInfo{
ItemType: details.SharePointPage,
},
},
},
},
expectErr: assert.Error,
},
{
name: "Groups List Errors",
// No version bump for the change so we always have to check for this.
Expand Down
Loading

0 comments on commit f61448d

Please sign in to comment.