From 0b53b23c84152d098577add1e84c53f86f548404 Mon Sep 17 00:00:00 2001 From: lostlevels Date: Tue, 15 Aug 2023 11:33:47 -0700 Subject: [PATCH] Handle out of bounds during dataSets retreival when merging dataSets from two collections due to using improper slice counter/index. --- data/store/mongo/mongo_data.go | 22 ++-- data/store/mongo/mongo_test.go | 228 +++++++++++++++++++++++++++++++++ 2 files changed, 242 insertions(+), 8 deletions(-) diff --git a/data/store/mongo/mongo_data.go b/data/store/mongo/mongo_data.go index 95597d65c..bae2e036e 100644 --- a/data/store/mongo/mongo_data.go +++ b/data/store/mongo/mongo_data.go @@ -55,7 +55,7 @@ func (d *DataRepository) GetDataSetsForUserByID(ctx context.Context, userID stri // Because there may be some dataSets in the old deviceData collection we // must read from both and merge the results while migration isn't // complete. Can delete this code when migration is complete. - merged := mergeSortedUploads(newUploads, prevUploads) + merged := MergeSortedUploads(newUploads, prevUploads) if pagination != nil && len(merged) > pagination.Size { merged = merged[:pagination.Size] } @@ -131,7 +131,7 @@ func (d *DataRepository) ListUserDataSets(ctx context.Context, userID string, fi // Because there may be some dataSets in the old deviceData collection we // must read from both and merge the results while migration isn't // complete. Can delete this code when migration is complete. - merged := mergeSortedDataSets(newDataSets, prevDataSets) + merged := MergeSortedDataSets(newDataSets, prevDataSets) if pagination != nil && len(merged) > pagination.Size { merged = merged[:pagination.Size] } @@ -465,8 +465,8 @@ func (d *DataRepository) mongoClient() *mongo.Client { return d.DatumRepository.Database().Client() } -// mergeSortedUploads combines the unique Uploads by UploadID into a new slice. -func mergeSortedUploads(newUploads, prevUploads []*upload.Upload) []*upload.Upload { +// MergeSortedUploads combines the unique Uploads by UploadID into a new slice. +func MergeSortedUploads(newUploads, prevUploads []*upload.Upload) []*upload.Upload { combined := make([]*upload.Upload, 0, len(newUploads)+len(prevUploads)) // Merge the two datasets like the merge step in merge sort. Note we don't @@ -482,7 +482,10 @@ func mergeSortedUploads(newUploads, prevUploads []*upload.Upload) []*upload.Uplo combined = append(combined, newUploads[newCounter]) newCounter++ } - combined = append(combined, prevUploads[newCounter]) + // Always add the dataSet/upload in the "old" deviceData collection + // because the dataSet/upload may not have been finished migrating + // into the new deviceDataSets collection + combined = append(combined, dataSet) // Skip duplicate of newUploads in prevUploads if it exists. if newCounter < len(newUploads) && *newUploads[newCounter].UploadID == *dataSet.UploadID { newCounter++ @@ -492,8 +495,8 @@ func mergeSortedUploads(newUploads, prevUploads []*upload.Upload) []*upload.Uplo return combined } -// mergeSortedDataSets combines the unique Uploads by UploadID into a new slice. -func mergeSortedDataSets(newDataSets, prevDataSets data.DataSets) data.DataSets { +// MergeSortedDataSets combines the unique Uploads by UploadID into a new slice. +func MergeSortedDataSets(newDataSets, prevDataSets data.DataSets) data.DataSets { combined := make(data.DataSets, 0, len(newDataSets)+len(prevDataSets)) // Merge the two datasets like the merge step in merge sort. Note we don't @@ -509,7 +512,10 @@ func mergeSortedDataSets(newDataSets, prevDataSets data.DataSets) data.DataSets combined = append(combined, newDataSets[newCounter]) newCounter++ } - combined = append(combined, prevDataSets[newCounter]) + // Always add the dataSet/upload in the "old" deviceData collection + // because the dataSet/upload may not have been finished migrating + // into the new deviceDataSets collection + combined = append(combined, dataSet) // Skip duplicate of newDataSets in prevDataSets if it exists. if newCounter < len(newDataSets) && *newDataSets[newCounter].UploadID == *dataSet.UploadID { newCounter++ diff --git a/data/store/mongo/mongo_test.go b/data/store/mongo/mongo_test.go index fc97ea8e1..4bec7dae6 100644 --- a/data/store/mongo/mongo_test.go +++ b/data/store/mongo/mongo_test.go @@ -262,6 +262,234 @@ var _ = Describe("Mongo", func() { }) }) + Context("Utility Functions", func() { + Context("MergeSortedUploads", func() { + prevUpload1 := &upload.Upload{ + Base: types.Base{ + Active: true, + CreatedTime: pointer.FromTime(time.Date(2016, time.December, 1, 20, 21, 23, 0, time.UTC)), + CreatedUserID: pointer.FromString("user1"), + DeviceID: pointer.FromString("deviceId"), + DeviceTime: pointer.FromString("2016-12-01T20:21:23"), + GUID: pointer.FromString("guid1"), + ID: pointer.FromString("id1"), + ModifiedTime: pointer.FromTime(time.Date(2016, time.December, 1, 20, 21, 23, 0, time.UTC)), + Source: pointer.FromString("source"), + Time: pointer.FromTime(time.Date(2016, time.December, 1, 20, 21, 23, 0, time.UTC)), + Type: "upload", + UploadID: pointer.FromString("upload1"), + UserID: pointer.FromString("user1"), + VersionInternal: 0, + }, + } + prevUpload2 := &upload.Upload{ + Base: types.Base{ + Active: true, + CreatedTime: pointer.FromTime(time.Date(2017, time.December, 1, 20, 21, 23, 0, time.UTC)), + CreatedUserID: pointer.FromString("user1"), + DeviceID: pointer.FromString("deviceId"), + DeviceTime: pointer.FromString("2017-12-01T20:21:23"), + GUID: pointer.FromString("guid2"), + ID: pointer.FromString("id2"), + ModifiedTime: pointer.FromTime(time.Date(2017, time.December, 1, 20, 21, 23, 0, time.UTC)), + Source: pointer.FromString("source"), + Time: pointer.FromTime(time.Date(2017, time.December, 1, 20, 21, 23, 0, time.UTC)), + Type: "upload", + UploadID: pointer.FromString("upload2"), + UserID: pointer.FromString("user1"), + VersionInternal: 0, + }, + } + newUpload1 := &upload.Upload{ + Base: types.Base{ + Active: true, + CreatedTime: pointer.FromTime(time.Date(2016, time.December, 1, 20, 21, 23, 0, time.UTC)), + CreatedUserID: pointer.FromString("user1"), + DeviceID: pointer.FromString("deviceId"), + DeviceTime: pointer.FromString("2016-12-01T20:21:23"), + GUID: pointer.FromString("guid1"), + ID: pointer.FromString("id1"), + ModifiedTime: pointer.FromTime(time.Date(2016, time.December, 1, 20, 21, 23, 0, time.UTC)), + Source: pointer.FromString("source"), + Time: pointer.FromTime(time.Date(2016, time.December, 1, 20, 21, 23, 0, time.UTC)), + Type: "upload", + UploadID: pointer.FromString("upload1"), + UserID: pointer.FromString("user1"), + VersionInternal: 1, + }, + } + newUpload2 := &upload.Upload{ + Base: types.Base{ + Active: true, + CreatedTime: pointer.FromTime(time.Date(2017, time.December, 1, 20, 21, 23, 0, time.UTC)), + CreatedUserID: pointer.FromString("user1"), + DeviceID: pointer.FromString("deviceId"), + DeviceTime: pointer.FromString("2017-12-01T20:21:23"), + GUID: pointer.FromString("guid2"), + ID: pointer.FromString("id2"), + ModifiedTime: pointer.FromTime(time.Date(2017, time.December, 1, 20, 21, 23, 0, time.UTC)), + Source: pointer.FromString("source"), + Time: pointer.FromTime(time.Date(2017, time.December, 1, 20, 21, 23, 0, time.UTC)), + Type: "upload", + UploadID: pointer.FromString("upload2"), + UserID: pointer.FromString("user1"), + VersionInternal: 1, + }, + } + It("works with more previous Uploads than new Uploads", func() { + prevSets := []*upload.Upload{ + prevUpload1, + prevUpload2, + } + newSets := []*upload.Upload{ + newUpload1, + } + sets := dataStoreMongo.MergeSortedUploads(newSets, prevSets) + Expect(len(sets)).To(Equal(2)) + Expect(sets[0]).To(Equal(prevUpload1)) + Expect(sets[1]).To(Equal(prevUpload2)) + }) + + It("works with more new Uploads than previous Uploads", func() { + prevSets := []*upload.Upload{ + prevUpload2, + } + newSets := []*upload.Upload{ + newUpload1, + newUpload2, + } + sets := dataStoreMongo.MergeSortedUploads(newSets, prevSets) + Expect(len(sets)).To(Equal(2)) + Expect(sets[0]).To(Equal(newUpload1)) + Expect(sets[1]).To(Equal(prevUpload2)) + }) + + It("works with equal new Uploads and previous Uploads", func() { + prevSets := []*upload.Upload{ + prevUpload1, + prevUpload2, + } + newSets := []*upload.Upload{ + newUpload1, + newUpload2, + } + sets := dataStoreMongo.MergeSortedUploads(newSets, prevSets) + Expect(len(sets)).To(Equal(2)) + Expect(sets[0]).To(Equal(prevUpload1)) + Expect(sets[1]).To(Equal(prevUpload2)) + }) + }) + + Context("MergeSortedDataSets", func() { + prevDataSet1 := &data.DataSet{ + Active: true, + ByUser: pointer.FromString("abcdef"), + ComputerTime: pointer.FromString("2016-12-01T20:21:23"), + CreatedTime: pointer.FromTime(time.Date(2016, time.December, 1, 20, 21, 23, 0, time.UTC)), + DataSetType: pointer.FromString("upload"), + DeviceID: pointer.FromString("my-device"), + DeviceModel: pointer.FromString("device-model"), + ID: pointer.FromString("1"), + Time: pointer.FromTime(time.Date(2016, time.December, 1, 20, 21, 23, 0, time.UTC)), + Type: "upload", + UploadID: pointer.FromString("1"), + UserID: pointer.FromString("User1"), + Version: pointer.FromString("0"), + VersionInternal: 0, + } + prevDataSet2 := &data.DataSet{ + Active: true, + ByUser: pointer.FromString("abcdef"), + ComputerTime: pointer.FromString("2017-12-01T20:21:23"), + CreatedTime: pointer.FromTime(time.Date(2017, time.December, 1, 20, 21, 23, 0, time.UTC)), + DataSetType: pointer.FromString("upload"), + DeviceID: pointer.FromString("my-device"), + DeviceModel: pointer.FromString("device-model"), + ID: pointer.FromString("2"), + Time: pointer.FromTime(time.Date(2017, time.December, 1, 20, 21, 23, 0, time.UTC)), + Type: "upload", + UploadID: pointer.FromString("2"), + UserID: pointer.FromString("User1"), + Version: pointer.FromString("0"), + VersionInternal: 0, + } + newDataSet1 := &data.DataSet{ + Active: true, + ByUser: pointer.FromString("abcdef"), + ComputerTime: pointer.FromString("2016-12-01T20:21:23"), + CreatedTime: pointer.FromTime(time.Date(2016, time.December, 1, 20, 21, 23, 0, time.UTC)), + DataSetType: pointer.FromString("upload"), + DeviceID: pointer.FromString("my-device"), + DeviceModel: pointer.FromString("device-model"), + ID: pointer.FromString("1"), + Time: pointer.FromTime(time.Date(2016, time.December, 1, 20, 21, 23, 0, time.UTC)), + Type: "upload", + UploadID: pointer.FromString("1"), + UserID: pointer.FromString("User1"), + Version: pointer.FromString("1"), + VersionInternal: 1, + } + newDataSet2 := &data.DataSet{ + Active: true, + ByUser: pointer.FromString("abcdef"), + ComputerTime: pointer.FromString("2017-12-01T20:21:23"), + CreatedTime: pointer.FromTime(time.Date(2017, time.December, 1, 20, 21, 23, 0, time.UTC)), + DataSetType: pointer.FromString("upload"), + DeviceID: pointer.FromString("my-device"), + DeviceModel: pointer.FromString("device-model"), + ID: pointer.FromString("2"), + Time: pointer.FromTime(time.Date(2017, time.December, 1, 20, 21, 23, 0, time.UTC)), + Type: "upload", + UploadID: pointer.FromString("2"), + UserID: pointer.FromString("User1"), + Version: pointer.FromString("1"), + VersionInternal: 1, + } + + It("works with more previous DataSets than new DataSets", func() { + prevSets := data.DataSets{ + prevDataSet1, + prevDataSet2, + } + newSets := data.DataSets{ + newDataSet1, + } + sets := dataStoreMongo.MergeSortedDataSets(newSets, prevSets) + Expect(len(sets)).To(Equal(2)) + Expect(sets[0]).To(Equal(prevDataSet1)) + Expect(sets[1]).To(Equal(prevDataSet2)) + }) + + It("works with more new DataSets than previous DataSets", func() { + prevSets := data.DataSets{ + prevDataSet1, + } + newSets := data.DataSets{ + newDataSet1, + newDataSet2, + } + sets := dataStoreMongo.MergeSortedDataSets(newSets, prevSets) + Expect(len(sets)).To(Equal(2)) + Expect(sets[0]).To(Equal(prevDataSet1)) + Expect(sets[1]).To(Equal(newDataSet2)) + }) + + It("works with equal new DataSets and previous DataSets", func() { + prevSets := data.DataSets{ + prevDataSet1, + prevDataSet2, + } + newSets := data.DataSets{ + newDataSet1, + newDataSet2, + } + sets := dataStoreMongo.MergeSortedDataSets(newSets, prevSets) + Expect(len(sets)).To(Equal(2)) + Expect(sets[0]).To(Equal(prevDataSet1)) + Expect(sets[1]).To(Equal(prevDataSet2)) + }) + }) + }) Context("with a new store", func() { var collection *mongo.Collection var dataSetCollection *mongo.Collection