diff --git a/site/gatsby-site/migrations/2024.12.11T14.39.38.refactor-reports-translations.js b/site/gatsby-site/migrations/2024.12.11T14.39.38.refactor-reports-translations.js new file mode 100644 index 0000000000..7fc9b764cc --- /dev/null +++ b/site/gatsby-site/migrations/2024.12.11T14.39.38.refactor-reports-translations.js @@ -0,0 +1,77 @@ +/** @type {import('umzug').MigrationFn} */ +exports.up = async ({ context: { client } }) => { + const db = client.db('translations'); + + const reportsEnCollection = db.collection('reports_en'); + + const reportsEsCollection = db.collection('reports_es'); + + const reportsFrCollection = db.collection('reports_fr'); + + const reportsJaCollection = db.collection('reports_ja'); + + const reportsCollection = db.collection('reports'); + + // Fetch all documents from reports_en, reports_es, reports_fr, and reports_ja + const reportsEn = await reportsEnCollection.find().toArray(); + + const reportsEs = await reportsEsCollection.find().toArray(); + + const reportsFr = await reportsFrCollection.find().toArray(); + + const reportsJa = await reportsJaCollection.find().toArray(); + + // Transform and insert documents into reports with a new field "language" + const transformedReportsEn = reportsEn.map((report) => ({ + ...report, + language: 'en', + })); + + const transformedReportsEs = reportsEs.map((report) => ({ + ...report, + language: 'es', + })); + + const transformedReportsFr = reportsFr.map((report) => ({ + ...report, + language: 'fr', + })); + + const transformedReportsJa = reportsJa.map((report) => ({ + ...report, + language: 'ja', + })); + + // Insert transformed documents into reports + const resultEn = await reportsCollection.insertMany(transformedReportsEn); + + const resultEs = await reportsCollection.insertMany(transformedReportsEs); + + const resultFr = await reportsCollection.insertMany(transformedReportsFr); + + const resultJa = await reportsCollection.insertMany(transformedReportsJa); + + console.log( + `${resultEn.insertedCount} documents with language "en" were inserted into the "reports" collection.` + ); + console.log( + `${resultEs.insertedCount} documents with language "es" were inserted into the "reports" collection.` + ); + console.log( + `${resultFr.insertedCount} documents with language "fr" were inserted into the "reports" collection.` + ); + console.log( + `${resultJa.insertedCount} documents with language "ja" were inserted into the "reports" collection.` + ); +}; + +/** @type {import('umzug').MigrationFn} */ +exports.down = async ({ context: { client } }) => { + const db = client.db('translations'); + + const reportsCollection = db.collection('reports'); + + // Drop the reports collection + await reportsCollection.drop(); + console.log('Collection "reports" has been dropped.'); +}; diff --git a/site/gatsby-site/playwright/e2e/unit/translator.spec.ts b/site/gatsby-site/playwright/e2e/unit/translator.spec.ts index 35b64d1725..c323ecb590 100644 --- a/site/gatsby-site/playwright/e2e/unit/translator.spec.ts +++ b/site/gatsby-site/playwright/e2e/unit/translator.spec.ts @@ -102,16 +102,33 @@ test('Translations - Should translate languages only if report language differs insertMany: sinon.stub().resolves({ insertedCount: 1 }), }; + const reportsTranslationsCollection = { + insertMany: sinon.stub().resolves({ insertedCount: 1 }), + }; + const mongoClient = { connect: sinon.stub().resolves(), close: sinon.stub().resolves(), - db: sinon.stub().returns({ - collection: (name: string) => { - if (name === 'reports') return reportsCollection; - if (name === 'reports_en') return reportsENCollection; - if (name === 'reports_es') return reportsESCollection; - return null; - }, + db: sinon.stub().callsFake((dbName: string) => { + if (dbName === 'aiidprod') { + return { + collection: (name: string) => { + if (name === 'reports') return reportsCollection; + return null; + }, + }; + } else if (dbName === 'translations') { + return { + collection: (name: string) => { + if (name === 'reports_en') return reportsENCollection; + if (name === 'reports_es') return reportsESCollection; + if (name === 'reports') return reportsTranslationsCollection; + return null; + }, + }; + } else { + throw new Error(`Unexpected database name: ${dbName}`); + } }), }; @@ -154,6 +171,32 @@ test('Translations - Should translate languages only if report language differs title: 'test-es-Report 1 title', plain_text: 'test-es-Report 1 text\n', }]); + + sinon.assert.calledTwice(reportsTranslationsCollection.insertMany); + sinon.assert.calledWith(reportsTranslationsCollection.insertMany, [ + { + report_number: 2, + text: 'test-en-Reporte 2 **texto**', + title: 'test-en-Título del reporte 2', + plain_text: 'test-en-Reporte 2 texto\n', + language: 'en', + }, + { + report_number: 3, + text: 'test-en-Reporte 3 **texto**', + title: 'test-en-Título del reporte 3', + plain_text: 'test-en-Reporte 3 texto\n', + language: 'en', + } + ]); + sinon.assert.calledWith(reportsTranslationsCollection.insertMany, [{ + report_number: 1, + text: 'test-es-Report 1 **text**', + title: 'test-es-Report 1 title', + plain_text: 'test-es-Report 1 text\n', + language: 'es', + }]); + sinon.assert.calledOnce(mongoClient.close); }); @@ -184,16 +227,33 @@ test("Translations - Shouldn't call Google's translate api and use translation p insertMany: sinon.stub().resolves({ insertedCount: 1 }), }; + const reportsTranslationsCollection = { + insertMany: sinon.stub().resolves({ insertedCount: 1 }), + }; + const mongoClient = { connect: sinon.stub().resolves(), close: sinon.stub().resolves(), - db: sinon.stub().returns({ - collection: (name: string) => { - if (name === 'reports') return reportsCollection; - if (name === 'reports_en') return reportsENCollection; - if (name === 'reports_es') return reportsESCollection; - return null; - }, + db: sinon.stub().callsFake((dbName: string) => { + if (dbName === 'aiidprod') { + return { + collection: (name: string) => { + if (name === 'reports') return reportsCollection; + return null; + }, + }; + } else if (dbName === 'translations') { + return { + collection: (name: string) => { + if (name === 'reports_en') return reportsENCollection; + if (name === 'reports_es') return reportsESCollection; + if (name === 'reports') return reportsTranslationsCollection; + return null; + }, + }; + } else { + throw new Error(`Unexpected database name: ${dbName}`); + } }), }; @@ -236,6 +296,32 @@ test("Translations - Shouldn't call Google's translate api and use translation p title: 'translated-es-Report 1 title', plain_text: 'translated-es-Report 1 text\n', }]); + + sinon.assert.calledTwice(reportsTranslationsCollection.insertMany); + sinon.assert.calledWith(reportsTranslationsCollection.insertMany, [ + { + report_number: 2, + text: 'translated-en-Reporte 2 **texto**', + title: 'translated-en-Título del reporte 2', + plain_text: 'translated-en-Reporte 2 texto\n', + language: 'en', + }, + { + report_number: 3, + text: 'translated-en-Reporte 3 **texto**', + title: 'translated-en-Título del reporte 3', + plain_text: 'translated-en-Reporte 3 texto\n', + language: 'en', + } + ]); + sinon.assert.calledWith(reportsTranslationsCollection.insertMany, [{ + report_number: 1, + text: 'translated-es-Report 1 **text**', + title: 'translated-es-Report 1 title', + plain_text: 'translated-es-Report 1 text\n', + language: 'es', + }]); + sinon.assert.calledOnce(mongoClient.close); }); @@ -268,16 +354,33 @@ test('Translations - Should translate reports with submission date greater than insertMany: sinon.stub().resolves({ insertedCount: 1 }), }; + const reportsTranslationsCollection = { + insertMany: sinon.stub().resolves({ insertedCount: 1 }), + }; + const mongoClient = { connect: sinon.stub().resolves(), close: sinon.stub().resolves(), - db: sinon.stub().returns({ - collection: (name: string) => { - if (name === 'reports') return reportsCollection; - if (name === 'reports_en') return reportsENCollection; - if (name === 'reports_es') return reportsESCollection; - return null; - }, + db: sinon.stub().callsFake((dbName: string) => { + if (dbName === 'aiidprod') { + return { + collection: (name: string) => { + if (name === 'reports') return reportsCollection; + return null; + }, + }; + } else if (dbName === 'translations') { + return { + collection: (name: string) => { + if (name === 'reports_en') return reportsENCollection; + if (name === 'reports_es') return reportsESCollection; + if (name === 'reports') return reportsTranslationsCollection; + return null; + }, + }; + } else { + throw new Error(`Unexpected database name: ${dbName}`); + } }), }; @@ -310,6 +413,18 @@ test('Translations - Should translate reports with submission date greater than } ]); sinon.assert.notCalled(reportsESCollection.insertMany); + + sinon.assert.calledOnce(reportsTranslationsCollection.insertMany); + sinon.assert.calledWith(reportsTranslationsCollection.insertMany, [ + { + report_number: 3, + text: 'test-en-Reporte 3 **texto**', + title: 'test-en-Título del reporte 3', + plain_text: 'test-en-Reporte 3 texto\n', + language: 'en', + } + ]); + sinon.assert.calledOnce(mongoClient.close); }); @@ -363,16 +478,33 @@ test('Translations - Should not translate if the report was already translated', insertMany: sinon.stub().resolves({ insertedCount: 1 }), }; + const reportsTranslationsCollection = { + insertMany: sinon.stub().resolves({ insertedCount: 1 }), + }; + const mongoClient = { connect: sinon.stub().resolves(), close: sinon.stub().resolves(), - db: sinon.stub().returns({ - collection: (name: string) => { - if (name === 'reports') return reportsCollection; - if (name === 'reports_en') return reportsENCollection; - if (name === 'reports_es') return reportsESCollection; - return null; - }, + db: sinon.stub().callsFake((dbName: string) => { + if (dbName === 'aiidprod') { + return { + collection: (name: string) => { + if (name === 'reports') return reportsCollection; + return null; + }, + }; + } else if (dbName === 'translations') { + return { + collection: (name: string) => { + if (name === 'reports_en') return reportsENCollection; + if (name === 'reports_es') return reportsESCollection; + if (name === 'reports') return reportsTranslationsCollection; + return null; + }, + }; + } else { + throw new Error(`Unexpected database name: ${dbName}`); + } }), }; @@ -395,5 +527,6 @@ test('Translations - Should not translate if the report was already translated', sinon.assert.notCalled(translateClient.translate); sinon.assert.notCalled(reportsENCollection.insertMany); sinon.assert.notCalled(reportsESCollection.insertMany); + sinon.assert.notCalled(reportsTranslationsCollection.insertMany); sinon.assert.calledOnce(mongoClient.close); }); diff --git a/site/gatsby-site/src/scripts/Translator.js b/site/gatsby-site/src/scripts/Translator.js index 1a2ae362b0..a47fc97faf 100644 --- a/site/gatsby-site/src/scripts/Translator.js +++ b/site/gatsby-site/src/scripts/Translator.js @@ -108,6 +108,8 @@ class Translator { .db('translations') .collection(`reports_${language}`); + const reportsTranslationsCollection = this.mongoClient.db('translations').collection('reports'); + const translated = []; for (const item of items) { @@ -118,7 +120,13 @@ class Translator { translated.push({ report_number, text, title, plain_text }); } - return reportsTranslatedCollection.insertMany(translated); + // TODO: remove this line when the "reports_xx" collections are no longer needed + await reportsTranslatedCollection.insertMany(translated); + + // Insert the translated reports into the reports collection with the language field + const reportsTranslated = translated.map((t) => ({ ...t, language })); + + return reportsTranslationsCollection.insertMany(reportsTranslated); } async translateReport({ entry, to }) {