Skip to content

Commit

Permalink
Handle out of bounds during dataSets retreival when merging dataSets …
Browse files Browse the repository at this point in the history 
…from two

collections due to using improper slice counter/index.
  • Loading branch information
@lostlevels
lostlevels committed Aug 15, 2023
1 parent cf3755b commit 0b53b23
Show file tree
Hide file tree
Showing 2 changed files with 242 additions and 8 deletions.
22 changes: 14 additions & 8 deletions data/store/mongo/mongo_data.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -55,7 +55,7 @@ func (d *DataRepository) GetDataSetsForUserByID(ctx context.Context, userID stri
// Because there may be some dataSets in the old deviceData collection we
// must read from both and merge the results while migration isn't
// complete. Can delete this code when migration is complete.
merged := mergeSortedUploads(newUploads, prevUploads)
merged := MergeSortedUploads(newUploads, prevUploads)
if pagination != nil && len(merged) > pagination.Size {
merged = merged[:pagination.Size]
}
Expand Down Expand Up @@ -131,7 +131,7 @@ func (d *DataRepository) ListUserDataSets(ctx context.Context, userID string, fi
// Because there may be some dataSets in the old deviceData collection we
// must read from both and merge the results while migration isn't
// complete. Can delete this code when migration is complete.
merged := mergeSortedDataSets(newDataSets, prevDataSets)
merged := MergeSortedDataSets(newDataSets, prevDataSets)
if pagination != nil && len(merged) > pagination.Size {
merged = merged[:pagination.Size]
}
Expand Down Expand Up @@ -465,8 +465,8 @@ func (d *DataRepository) mongoClient() *mongo.Client {
return d.DatumRepository.Database().Client()
}

// mergeSortedUploads combines the unique Uploads by UploadID into a new slice.
func mergeSortedUploads(newUploads, prevUploads []*upload.Upload) []*upload.Upload {
// MergeSortedUploads combines the unique Uploads by UploadID into a new slice.
func MergeSortedUploads(newUploads, prevUploads []*upload.Upload) []*upload.Upload {
combined := make([]*upload.Upload, 0, len(newUploads)+len(prevUploads))

// Merge the two datasets like the merge step in merge sort. Note we don't
Expand All @@ -482,7 +482,10 @@ func mergeSortedUploads(newUploads, prevUploads []*upload.Upload) []*upload.Uplo
combined = append(combined, newUploads[newCounter])
newCounter++
}
combined = append(combined, prevUploads[newCounter])
// Always add the dataSet/upload in the "old" deviceData collection
// because the dataSet/upload may not have been finished migrating
// into the new deviceDataSets collection
combined = append(combined, dataSet)
// Skip duplicate of newUploads in prevUploads if it exists.
if newCounter < len(newUploads) && *newUploads[newCounter].UploadID == *dataSet.UploadID {
newCounter++
Expand All @@ -492,8 +495,8 @@ func mergeSortedUploads(newUploads, prevUploads []*upload.Upload) []*upload.Uplo
return combined
}

// mergeSortedDataSets combines the unique Uploads by UploadID into a new slice.
func mergeSortedDataSets(newDataSets, prevDataSets data.DataSets) data.DataSets {
// MergeSortedDataSets combines the unique Uploads by UploadID into a new slice.
func MergeSortedDataSets(newDataSets, prevDataSets data.DataSets) data.DataSets {
combined := make(data.DataSets, 0, len(newDataSets)+len(prevDataSets))

// Merge the two datasets like the merge step in merge sort. Note we don't
Expand All @@ -509,7 +512,10 @@ func mergeSortedDataSets(newDataSets, prevDataSets data.DataSets) data.DataSets
combined = append(combined, newDataSets[newCounter])
newCounter++
}
combined = append(combined, prevDataSets[newCounter])
// Always add the dataSet/upload in the "old" deviceData collection
// because the dataSet/upload may not have been finished migrating
// into the new deviceDataSets collection
combined = append(combined, dataSet)
// Skip duplicate of newDataSets in prevDataSets if it exists.
if newCounter < len(newDataSets) && *newDataSets[newCounter].UploadID == *dataSet.UploadID {
newCounter++
Expand Down
228 changes: 228 additions & 0 deletions data/store/mongo/mongo_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -262,6 +262,234 @@ var _ = Describe("Mongo", func() {
})
})

Context("Utility Functions", func() {
Context("MergeSortedUploads", func() {
prevUpload1 := &upload.Upload{
Base: types.Base{
Active: true,
CreatedTime: pointer.FromTime(time.Date(2016, time.December, 1, 20, 21, 23, 0, time.UTC)),
CreatedUserID: pointer.FromString("user1"),
DeviceID: pointer.FromString("deviceId"),
DeviceTime: pointer.FromString("2016-12-01T20:21:23"),
GUID: pointer.FromString("guid1"),
ID: pointer.FromString("id1"),
ModifiedTime: pointer.FromTime(time.Date(2016, time.December, 1, 20, 21, 23, 0, time.UTC)),
Source: pointer.FromString("source"),
Time: pointer.FromTime(time.Date(2016, time.December, 1, 20, 21, 23, 0, time.UTC)),
Type: "upload",
UploadID: pointer.FromString("upload1"),
UserID: pointer.FromString("user1"),
VersionInternal: 0,
},
}
prevUpload2 := &upload.Upload{
Base: types.Base{
Active: true,
CreatedTime: pointer.FromTime(time.Date(2017, time.December, 1, 20, 21, 23, 0, time.UTC)),
CreatedUserID: pointer.FromString("user1"),
DeviceID: pointer.FromString("deviceId"),
DeviceTime: pointer.FromString("2017-12-01T20:21:23"),
GUID: pointer.FromString("guid2"),
ID: pointer.FromString("id2"),
ModifiedTime: pointer.FromTime(time.Date(2017, time.December, 1, 20, 21, 23, 0, time.UTC)),
Source: pointer.FromString("source"),
Time: pointer.FromTime(time.Date(2017, time.December, 1, 20, 21, 23, 0, time.UTC)),
Type: "upload",
UploadID: pointer.FromString("upload2"),
UserID: pointer.FromString("user1"),
VersionInternal: 0,
},
}
newUpload1 := &upload.Upload{
Base: types.Base{
Active: true,
CreatedTime: pointer.FromTime(time.Date(2016, time.December, 1, 20, 21, 23, 0, time.UTC)),
CreatedUserID: pointer.FromString("user1"),
DeviceID: pointer.FromString("deviceId"),
DeviceTime: pointer.FromString("2016-12-01T20:21:23"),
GUID: pointer.FromString("guid1"),
ID: pointer.FromString("id1"),
ModifiedTime: pointer.FromTime(time.Date(2016, time.December, 1, 20, 21, 23, 0, time.UTC)),
Source: pointer.FromString("source"),
Time: pointer.FromTime(time.Date(2016, time.December, 1, 20, 21, 23, 0, time.UTC)),
Type: "upload",
UploadID: pointer.FromString("upload1"),
UserID: pointer.FromString("user1"),
VersionInternal: 1,
},
}
newUpload2 := &upload.Upload{
Base: types.Base{
Active: true,
CreatedTime: pointer.FromTime(time.Date(2017, time.December, 1, 20, 21, 23, 0, time.UTC)),
CreatedUserID: pointer.FromString("user1"),
DeviceID: pointer.FromString("deviceId"),
DeviceTime: pointer.FromString("2017-12-01T20:21:23"),
GUID: pointer.FromString("guid2"),
ID: pointer.FromString("id2"),
ModifiedTime: pointer.FromTime(time.Date(2017, time.December, 1, 20, 21, 23, 0, time.UTC)),
Source: pointer.FromString("source"),
Time: pointer.FromTime(time.Date(2017, time.December, 1, 20, 21, 23, 0, time.UTC)),
Type: "upload",
UploadID: pointer.FromString("upload2"),
UserID: pointer.FromString("user1"),
VersionInternal: 1,
},
}
It("works with more previous Uploads than new Uploads", func() {
prevSets := []*upload.Upload{
prevUpload1,
prevUpload2,
}
newSets := []*upload.Upload{
newUpload1,
}
sets := dataStoreMongo.MergeSortedUploads(newSets, prevSets)
Expect(len(sets)).To(Equal(2))
Expect(sets[0]).To(Equal(prevUpload1))
Expect(sets[1]).To(Equal(prevUpload2))
})

It("works with more new Uploads than previous Uploads", func() {
prevSets := []*upload.Upload{
prevUpload2,
}
newSets := []*upload.Upload{
newUpload1,
newUpload2,
}
sets := dataStoreMongo.MergeSortedUploads(newSets, prevSets)
Expect(len(sets)).To(Equal(2))
Expect(sets[0]).To(Equal(newUpload1))
Expect(sets[1]).To(Equal(prevUpload2))
})

It("works with equal new Uploads and previous Uploads", func() {
prevSets := []*upload.Upload{
prevUpload1,
prevUpload2,
}
newSets := []*upload.Upload{
newUpload1,
newUpload2,
}
sets := dataStoreMongo.MergeSortedUploads(newSets, prevSets)
Expect(len(sets)).To(Equal(2))
Expect(sets[0]).To(Equal(prevUpload1))
Expect(sets[1]).To(Equal(prevUpload2))
})
})

Context("MergeSortedDataSets", func() {
prevDataSet1 := &data.DataSet{
Active: true,
ByUser: pointer.FromString("abcdef"),
ComputerTime: pointer.FromString("2016-12-01T20:21:23"),
CreatedTime: pointer.FromTime(time.Date(2016, time.December, 1, 20, 21, 23, 0, time.UTC)),
DataSetType: pointer.FromString("upload"),
DeviceID: pointer.FromString("my-device"),
DeviceModel: pointer.FromString("device-model"),
ID: pointer.FromString("1"),
Time: pointer.FromTime(time.Date(2016, time.December, 1, 20, 21, 23, 0, time.UTC)),
Type: "upload",
UploadID: pointer.FromString("1"),
UserID: pointer.FromString("User1"),
Version: pointer.FromString("0"),
VersionInternal: 0,
}
prevDataSet2 := &data.DataSet{
Active: true,
ByUser: pointer.FromString("abcdef"),
ComputerTime: pointer.FromString("2017-12-01T20:21:23"),
CreatedTime: pointer.FromTime(time.Date(2017, time.December, 1, 20, 21, 23, 0, time.UTC)),
DataSetType: pointer.FromString("upload"),
DeviceID: pointer.FromString("my-device"),
DeviceModel: pointer.FromString("device-model"),
ID: pointer.FromString("2"),
Time: pointer.FromTime(time.Date(2017, time.December, 1, 20, 21, 23, 0, time.UTC)),
Type: "upload",
UploadID: pointer.FromString("2"),
UserID: pointer.FromString("User1"),
Version: pointer.FromString("0"),
VersionInternal: 0,
}
newDataSet1 := &data.DataSet{
Active: true,
ByUser: pointer.FromString("abcdef"),
ComputerTime: pointer.FromString("2016-12-01T20:21:23"),
CreatedTime: pointer.FromTime(time.Date(2016, time.December, 1, 20, 21, 23, 0, time.UTC)),
DataSetType: pointer.FromString("upload"),
DeviceID: pointer.FromString("my-device"),
DeviceModel: pointer.FromString("device-model"),
ID: pointer.FromString("1"),
Time: pointer.FromTime(time.Date(2016, time.December, 1, 20, 21, 23, 0, time.UTC)),
Type: "upload",
UploadID: pointer.FromString("1"),
UserID: pointer.FromString("User1"),
Version: pointer.FromString("1"),
VersionInternal: 1,
}
newDataSet2 := &data.DataSet{
Active: true,
ByUser: pointer.FromString("abcdef"),
ComputerTime: pointer.FromString("2017-12-01T20:21:23"),
CreatedTime: pointer.FromTime(time.Date(2017, time.December, 1, 20, 21, 23, 0, time.UTC)),
DataSetType: pointer.FromString("upload"),
DeviceID: pointer.FromString("my-device"),
DeviceModel: pointer.FromString("device-model"),
ID: pointer.FromString("2"),
Time: pointer.FromTime(time.Date(2017, time.December, 1, 20, 21, 23, 0, time.UTC)),
Type: "upload",
UploadID: pointer.FromString("2"),
UserID: pointer.FromString("User1"),
Version: pointer.FromString("1"),
VersionInternal: 1,
}

It("works with more previous DataSets than new DataSets", func() {
prevSets := data.DataSets{
prevDataSet1,
prevDataSet2,
}
newSets := data.DataSets{
newDataSet1,
}
sets := dataStoreMongo.MergeSortedDataSets(newSets, prevSets)
Expect(len(sets)).To(Equal(2))
Expect(sets[0]).To(Equal(prevDataSet1))
Expect(sets[1]).To(Equal(prevDataSet2))
})

It("works with more new DataSets than previous DataSets", func() {
prevSets := data.DataSets{
prevDataSet1,
}
newSets := data.DataSets{
newDataSet1,
newDataSet2,
}
sets := dataStoreMongo.MergeSortedDataSets(newSets, prevSets)
Expect(len(sets)).To(Equal(2))
Expect(sets[0]).To(Equal(prevDataSet1))
Expect(sets[1]).To(Equal(newDataSet2))
})

It("works with equal new DataSets and previous DataSets", func() {
prevSets := data.DataSets{
prevDataSet1,
prevDataSet2,
}
newSets := data.DataSets{
newDataSet1,
newDataSet2,
}
sets := dataStoreMongo.MergeSortedDataSets(newSets, prevSets)
Expect(len(sets)).To(Equal(2))
Expect(sets[0]).To(Equal(prevDataSet1))
Expect(sets[1]).To(Equal(prevDataSet2))
})
})
})
Context("with a new store", func() {
var collection *mongo.Collection
var dataSetCollection *mongo.Collection
Expand Down

0 comments on commit 0b53b23

Please sign in to comment.