From 35951ed068a39e9d471cf191c73c02ecbeede7df Mon Sep 17 00:00:00 2001 From: Clara Youdale Date: Wed, 29 Nov 2023 13:08:43 -0300 Subject: [PATCH 1/7] Create 2023.11.29T14.56.25.delete-duplicated-subscriptions.js Migration that removes duplicates in subscriptions collection --- ...4.56.25.delete-duplicated-subscriptions.js | 74 +++++++++++++++++++ 1 file changed, 74 insertions(+) create mode 100644 site/gatsby-site/migrations/2023.11.29T14.56.25.delete-duplicated-subscriptions.js diff --git a/site/gatsby-site/migrations/2023.11.29T14.56.25.delete-duplicated-subscriptions.js b/site/gatsby-site/migrations/2023.11.29T14.56.25.delete-duplicated-subscriptions.js new file mode 100644 index 0000000000..35064fc693 --- /dev/null +++ b/site/gatsby-site/migrations/2023.11.29T14.56.25.delete-duplicated-subscriptions.js @@ -0,0 +1,74 @@ +const config = require('../config'); + +/** @type {import('umzug').MigrationFn} */ +exports.up = async ({ context: { client } }) => { + const db = client.db(config.realm.production_db.db_custom_data); + + const subscriptions = db.collection('subscriptions'); + + // Remove duplicates of type "incident", "new-incidents" and "submission-promoted", based on type, incident_id and userId + subscriptions + .aggregate([ + { + $match: { + type: { $in: ['incident', 'new-incidents', 'submission-promoted'] }, + }, + }, + { + $group: { + _id: { incident_id: '$incident_id', type: '$type', userId: '$userId' }, + uniqueIds: { $addToSet: '$_id' }, + count: { $sum: 1 }, + }, + }, + { + $match: { + count: { $gt: 1 }, + }, + }, + ]) + .forEach(function (doc) { + if (doc._id.incident_id && doc._id.type && doc._id.userId) { + console.log( + `Removing ${doc.count - 1} duplicated subscriptions for incident ${ + doc._id.incident_id + } of type ${doc._id.type} for user ${doc._id.userId}` + ); + doc.uniqueIds.pop(); // Remove one from the array to keep + subscriptions.deleteOne({ _id: { $in: doc.uniqueIds } }); + } + }); + + // Remove duplicates of type "entity", based on type, entityId and userId + subscriptions + .aggregate([ + { + $match: { + type: { $in: ['entity'] }, + }, + }, + { + $group: { + _id: { entityId: '$entityId', type: '$type', userId: '$userId' }, + uniqueIds: { $addToSet: '$_id' }, + count: { $sum: 1 }, + }, + }, + { + $match: { + count: { $gt: 1 }, + }, + }, + ]) + .forEach(function (doc) { + if (doc._id.entityId && doc._id.type && doc._id.userId) { + console.log( + `Removing ${doc.count - 1} duplicated subscriptions for entity ${ + doc._id.entityId + } of type ${doc._id.type} for user ${doc._id.userId}` + ); + doc.uniqueIds.pop(); // Remove one from the array to keep + subscriptions.deleteOne({ _id: { $in: doc.uniqueIds } }); + } + }); +}; From 89d764d48a3830d120f2e75f1b3d7ee45f3f6ee5 Mon Sep 17 00:00:00 2001 From: Clara Youdale Date: Fri, 8 Dec 2023 09:37:21 -0300 Subject: [PATCH 2/7] Fix migration to delete all duplicates --- ...4.56.25.delete-duplicated-subscriptions.js | 65 ++++++++++++------- 1 file changed, 41 insertions(+), 24 deletions(-) diff --git a/site/gatsby-site/migrations/2023.11.29T14.56.25.delete-duplicated-subscriptions.js b/site/gatsby-site/migrations/2023.11.29T14.56.25.delete-duplicated-subscriptions.js index 35064fc693..b414f3c1a9 100644 --- a/site/gatsby-site/migrations/2023.11.29T14.56.25.delete-duplicated-subscriptions.js +++ b/site/gatsby-site/migrations/2023.11.29T14.56.25.delete-duplicated-subscriptions.js @@ -1,3 +1,5 @@ +const { ObjectID } = require('bson'); + const config = require('../config'); /** @type {import('umzug').MigrationFn} */ @@ -7,7 +9,8 @@ exports.up = async ({ context: { client } }) => { const subscriptions = db.collection('subscriptions'); // Remove duplicates of type "incident", "new-incidents" and "submission-promoted", based on type, incident_id and userId - subscriptions + + const result = await subscriptions .aggregate([ { $match: { @@ -27,20 +30,28 @@ exports.up = async ({ context: { client } }) => { }, }, ]) - .forEach(function (doc) { - if (doc._id.incident_id && doc._id.type && doc._id.userId) { - console.log( - `Removing ${doc.count - 1} duplicated subscriptions for incident ${ - doc._id.incident_id - } of type ${doc._id.type} for user ${doc._id.userId}` - ); - doc.uniqueIds.pop(); // Remove one from the array to keep - subscriptions.deleteOne({ _id: { $in: doc.uniqueIds } }); - } - }); + .toArray(); + + for (const doc of result) { + const uniqueIds = doc.uniqueIds.map((id) => new ObjectID(id)); + + if (doc._id.incident_id && doc._id.type && doc._id.userId) { + console.log( + `Removing ${doc.count - 1} duplicated subscriptions for incident ${ + doc._id.incident_id + } of type ${doc._id.type} for user ${doc._id.userId}` + ); + uniqueIds.pop(); // Remove one from the array to keep + const deleteResult = await subscriptions.deleteMany({ + _id: { $in: uniqueIds }, + }); + + console.log('Delete Result: ', deleteResult); + } + } // Remove duplicates of type "entity", based on type, entityId and userId - subscriptions + const resultEntities = await subscriptions .aggregate([ { $match: { @@ -60,15 +71,21 @@ exports.up = async ({ context: { client } }) => { }, }, ]) - .forEach(function (doc) { - if (doc._id.entityId && doc._id.type && doc._id.userId) { - console.log( - `Removing ${doc.count - 1} duplicated subscriptions for entity ${ - doc._id.entityId - } of type ${doc._id.type} for user ${doc._id.userId}` - ); - doc.uniqueIds.pop(); // Remove one from the array to keep - subscriptions.deleteOne({ _id: { $in: doc.uniqueIds } }); - } - }); + .toArray(); + + for (const doc of resultEntities) { + const uniqueIds = doc.uniqueIds.map((id) => new ObjectID(id)); + + if (doc._id.entityId && doc._id.type && doc._id.userId) { + console.log( + `Removing ${doc.count - 1} duplicated subscriptions for entity ${ + doc._id.entityId + } of type ${doc._id.type} for user ${doc._id.userId}` + ); + uniqueIds.pop(); // Remove one from the array to keep + const deleteResult = await subscriptions.deleteMany({ _id: { $in: uniqueIds } }); + + console.log('Delete Result: ', deleteResult); + } + } }; From 63a9e31cc689cb62653bf32397ac84f9e2ee1915 Mon Sep 17 00:00:00 2001 From: Clara Youdale Date: Mon, 11 Dec 2023 10:04:47 -0300 Subject: [PATCH 3/7] Fix migration to remove duplicates of new-incidents --- ...4.56.25.delete-duplicated-subscriptions.js | 72 ++++++++++++------- 1 file changed, 47 insertions(+), 25 deletions(-) diff --git a/site/gatsby-site/migrations/2023.11.29T14.56.25.delete-duplicated-subscriptions.js b/site/gatsby-site/migrations/2023.11.29T14.56.25.delete-duplicated-subscriptions.js index b414f3c1a9..0ba731729d 100644 --- a/site/gatsby-site/migrations/2023.11.29T14.56.25.delete-duplicated-subscriptions.js +++ b/site/gatsby-site/migrations/2023.11.29T14.56.25.delete-duplicated-subscriptions.js @@ -14,7 +14,7 @@ exports.up = async ({ context: { client } }) => { .aggregate([ { $match: { - type: { $in: ['incident', 'new-incidents', 'submission-promoted'] }, + type: { $in: ['incident', 'submission-promoted'] }, }, }, { @@ -32,23 +32,7 @@ exports.up = async ({ context: { client } }) => { ]) .toArray(); - for (const doc of result) { - const uniqueIds = doc.uniqueIds.map((id) => new ObjectID(id)); - - if (doc._id.incident_id && doc._id.type && doc._id.userId) { - console.log( - `Removing ${doc.count - 1} duplicated subscriptions for incident ${ - doc._id.incident_id - } of type ${doc._id.type} for user ${doc._id.userId}` - ); - uniqueIds.pop(); // Remove one from the array to keep - const deleteResult = await subscriptions.deleteMany({ - _id: { $in: uniqueIds }, - }); - - console.log('Delete Result: ', deleteResult); - } - } + await removeDuplicates(subscriptions, result); // Remove duplicates of type "entity", based on type, entityId and userId const resultEntities = await subscriptions @@ -73,19 +57,57 @@ exports.up = async ({ context: { client } }) => { ]) .toArray(); - for (const doc of resultEntities) { + await removeDuplicates(subscriptions, resultEntities); + + // Remove duplicates of type "incident", "new-incidents" and "submission-promoted", based on type, incident_id and userId + + const resultNewIncidents = await subscriptions + .aggregate([ + { + $match: { + type: { $in: ['new-incidents'] }, + }, + }, + { + $group: { + _id: { incident_id: '$incident_id', type: '$type', userId: '$userId' }, + uniqueIds: { $addToSet: '$_id' }, + count: { $sum: 1 }, + }, + }, + { + $match: { + count: { $gt: 1 }, + }, + }, + ]) + .toArray(); + + await removeDuplicates(subscriptions, resultNewIncidents); +}; + +async function removeDuplicates(subscriptions, results) { + for (const doc of results) { const uniqueIds = doc.uniqueIds.map((id) => new ObjectID(id)); - if (doc._id.entityId && doc._id.type && doc._id.userId) { + if (doc._id.type && doc._id.userId) { console.log( - `Removing ${doc.count - 1} duplicated subscriptions for entity ${ - doc._id.entityId - } of type ${doc._id.type} for user ${doc._id.userId}` + `Removing ${doc.count - 1} duplicated subscriptions of type ${doc._id.type} for user ${ + doc._id.userId + } ${ + doc._id.incident_id + ? ' and for incident ' + doc._id.incident_id + : doc._id.entityId + ? ' and for entity ' + doc._id.entityId + : '' + }` ); uniqueIds.pop(); // Remove one from the array to keep - const deleteResult = await subscriptions.deleteMany({ _id: { $in: uniqueIds } }); + const deleteResult = await subscriptions.deleteMany({ + _id: { $in: uniqueIds }, + }); console.log('Delete Result: ', deleteResult); } } -}; +} From 535c48dcf19f62df05f92538881e5c900dc05c62 Mon Sep 17 00:00:00 2001 From: kepae Date: Mon, 11 Dec 2023 16:43:37 -0500 Subject: [PATCH 4/7] nit: update comments --- .../2023.11.29T14.56.25.delete-duplicated-subscriptions.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/site/gatsby-site/migrations/2023.11.29T14.56.25.delete-duplicated-subscriptions.js b/site/gatsby-site/migrations/2023.11.29T14.56.25.delete-duplicated-subscriptions.js index 0ba731729d..2a29269487 100644 --- a/site/gatsby-site/migrations/2023.11.29T14.56.25.delete-duplicated-subscriptions.js +++ b/site/gatsby-site/migrations/2023.11.29T14.56.25.delete-duplicated-subscriptions.js @@ -8,7 +8,7 @@ exports.up = async ({ context: { client } }) => { const subscriptions = db.collection('subscriptions'); - // Remove duplicates of type "incident", "new-incidents" and "submission-promoted", based on type, incident_id and userId + // Remove duplicates of type "incident" and "submission-promoted", based on type, incident_id and userId const result = await subscriptions .aggregate([ From 32a55fa9866446b44bf13b675e1e4a03a7c2ba54 Mon Sep 17 00:00:00 2001 From: kepae Date: Mon, 11 Dec 2023 16:44:00 -0500 Subject: [PATCH 5/7] nit: update comments --- .../2023.11.29T14.56.25.delete-duplicated-subscriptions.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/site/gatsby-site/migrations/2023.11.29T14.56.25.delete-duplicated-subscriptions.js b/site/gatsby-site/migrations/2023.11.29T14.56.25.delete-duplicated-subscriptions.js index 2a29269487..51b44f3d86 100644 --- a/site/gatsby-site/migrations/2023.11.29T14.56.25.delete-duplicated-subscriptions.js +++ b/site/gatsby-site/migrations/2023.11.29T14.56.25.delete-duplicated-subscriptions.js @@ -59,7 +59,7 @@ exports.up = async ({ context: { client } }) => { await removeDuplicates(subscriptions, resultEntities); - // Remove duplicates of type "incident", "new-incidents" and "submission-promoted", based on type, incident_id and userId + // Remove duplicates of type "new-incidents" based on type, incident_id and userId const resultNewIncidents = await subscriptions .aggregate([ From 9d4a2d4472b87f4cba789f3acc271b8fc3bc6a5f Mon Sep 17 00:00:00 2001 From: Clara Youdale Date: Tue, 12 Dec 2023 17:06:55 -0300 Subject: [PATCH 6/7] Add test for delete duplicate migration --- ...6.25.delete-duplicated-subscriptions.cy.js | 144 ++++++++++++++++++ 1 file changed, 144 insertions(+) create mode 100644 site/gatsby-site/cypress/e2e/unit/migrations/2023.11.29T14.56.25.delete-duplicated-subscriptions.cy.js diff --git a/site/gatsby-site/cypress/e2e/unit/migrations/2023.11.29T14.56.25.delete-duplicated-subscriptions.cy.js b/site/gatsby-site/cypress/e2e/unit/migrations/2023.11.29T14.56.25.delete-duplicated-subscriptions.cy.js new file mode 100644 index 0000000000..a64e9fc16e --- /dev/null +++ b/site/gatsby-site/cypress/e2e/unit/migrations/2023.11.29T14.56.25.delete-duplicated-subscriptions.cy.js @@ -0,0 +1,144 @@ +const { ObjectID } = require('bson'); + +const { + up, +} = require('../../../../migrations/2023.11.29T14.56.25.delete-duplicated-subscriptions'); + +describe('Migration Script - Remove Duplicated Subscriptions', () => { + it('Should remove duplicated subscriptions correctly', () => { + // Mocked data for all three cases + const testSubscriptions = { + incident: [ + { + _id: { + type: 'incident', + userId: '642188372947d07020c1319d', + incident_id: 600, + }, + uniqueIds: [ + new ObjectID('5f9f6b9b5f9c4c0001a3b3a3'), + new ObjectID('5f9f6b9b5f9c4c0001a3b3a4'), + ], + }, + { + _id: { + type: 'submission-promoted', + userId: '642188372947d07020c1319d', + incident_id: 600, + }, + uniqueIds: [ + new ObjectID('5f9f6b9b5f9c4c0001a3b3a0'), + new ObjectID('5f9f6b9b5f9c4c0001a3b3a1'), + new ObjectID('5f9f6b9b5f9c4c0001a3b3a2'), + ], + }, + ], + entity: [ + { + _id: { + type: 'entity', + userId: '642188372947d07020c1319d', + entity_id: 'trans-women', + }, + uniqueIds: ['5f9f6b9b5f9c4c0001a3b3a5', '5f9f6b9b5f9c4c0001a3b3a6'], + }, + ], + 'new-incidents': [ + { + _id: { + type: 'new-incidents', + userId: '642188372947d07020c1319d', + }, + uniqueIds: ['5f9f6b9b5f9c4c0001a3b3a7', '5f9f6b9b5f9c4c0001a3b3a8'], + }, + ], + }; + + // Mocked collection with stubbed methods + const subscriptionsCollection = { + aggregate: cy.stub().callsFake((query) => { + const type = query[0].$match.type.$in[0]; + + console.log(type); + return { + toArray: cy.stub().resolves(testSubscriptions[type]), + }; + }), + deleteMany: cy.stub().callsFake((query) => { + console.log('deleteMany called with:', query._id.$in[0].toString()); // Log the query + return Promise.resolve({ deletedCount: 1 }); + }), + }; + + // Mocked context with database client + const context = { + client: { + db: cy.stub().returns({ + collection: cy.stub().withArgs('subscriptions').returns(subscriptionsCollection), + }), + }, + }; + + // Execute the migration script + cy.wrap(up({ context })).then(() => { + // Assertions for each case + const args = subscriptionsCollection.deleteMany.getCall(0).args[0]; + + const argsSubmissionPromoted = subscriptionsCollection.deleteMany.getCall(1).args[0]; + + const argsEntity = subscriptionsCollection.deleteMany.getCall(2).args[0]; + + const argsNewIncidents = subscriptionsCollection.deleteMany.getCall(3).args[0]; + + let modifiedObjectIncident = { + _id: { + $in: args._id.$in.map((id) => id.toString()), + }, + }; + + let modifiedObjectSubmissionPromoted = { + _id: { + $in: argsSubmissionPromoted._id.$in.map((id) => id.toString()), + }, + }; + + let modifiedObjectEntity = { + _id: { + $in: argsEntity._id.$in.map((id) => id.toString()), + }, + }; + + let modifiedObjectNewIncidents = { + _id: { + $in: argsNewIncidents._id.$in.map((id) => id.toString()), + }, + }; + + expect(modifiedObjectIncident).to.be.deep.equal({ + _id: { + $in: testSubscriptions['incident'][0].uniqueIds.slice(0, 1).map((id) => id.toString()), + }, + }); + + expect(modifiedObjectSubmissionPromoted).to.be.deep.equal({ + _id: { + $in: testSubscriptions['incident'][1].uniqueIds.slice(0, 2).map((id) => id.toString()), + }, + }); + + expect(modifiedObjectEntity).to.be.deep.equal({ + _id: { + $in: testSubscriptions['entity'][0].uniqueIds.slice(0, 1).map((id) => id.toString()), + }, + }); + + expect(modifiedObjectNewIncidents).to.be.deep.equal({ + _id: { + $in: testSubscriptions['new-incidents'][0].uniqueIds + .slice(0, 1) + .map((id) => id.toString()), + }, + }); + }); + }); +}); From c85934d5c1097fc47d9d2c4568a3da39ab8d34c1 Mon Sep 17 00:00:00 2001 From: Clara Youdale Date: Wed, 13 Dec 2023 09:04:40 -0300 Subject: [PATCH 7/7] keep consistency in object IDs --- ....29T14.56.25.delete-duplicated-subscriptions.cy.js | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/site/gatsby-site/cypress/e2e/unit/migrations/2023.11.29T14.56.25.delete-duplicated-subscriptions.cy.js b/site/gatsby-site/cypress/e2e/unit/migrations/2023.11.29T14.56.25.delete-duplicated-subscriptions.cy.js index a64e9fc16e..956861ad3d 100644 --- a/site/gatsby-site/cypress/e2e/unit/migrations/2023.11.29T14.56.25.delete-duplicated-subscriptions.cy.js +++ b/site/gatsby-site/cypress/e2e/unit/migrations/2023.11.29T14.56.25.delete-duplicated-subscriptions.cy.js @@ -40,7 +40,10 @@ describe('Migration Script - Remove Duplicated Subscriptions', () => { userId: '642188372947d07020c1319d', entity_id: 'trans-women', }, - uniqueIds: ['5f9f6b9b5f9c4c0001a3b3a5', '5f9f6b9b5f9c4c0001a3b3a6'], + uniqueIds: [ + new ObjectID('5f9f6b9b5f9c4c0001a3b3a5'), + new ObjectID('5f9f6b9b5f9c4c0001a3b3a6'), + ], }, ], 'new-incidents': [ @@ -49,7 +52,10 @@ describe('Migration Script - Remove Duplicated Subscriptions', () => { type: 'new-incidents', userId: '642188372947d07020c1319d', }, - uniqueIds: ['5f9f6b9b5f9c4c0001a3b3a7', '5f9f6b9b5f9c4c0001a3b3a8'], + uniqueIds: [ + new ObjectID('5f9f6b9b5f9c4c0001a3b3a7'), + new ObjectID('5f9f6b9b5f9c4c0001a3b3a8'), + ], }, ], }; @@ -59,7 +65,6 @@ describe('Migration Script - Remove Duplicated Subscriptions', () => { aggregate: cy.stub().callsFake((query) => { const type = query[0].$match.type.$in[0]; - console.log(type); return { toArray: cy.stub().resolves(testSubscriptions[type]), };