Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[Translations] 1 - New single Reports translation collection #3302

Open
wants to merge 3 commits into
base: staging
Choose a base branch
from
Open

[Translations] 1 - New single Reports translation collection #3302

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
f2d53ab
Refactor reports translations migration to consolidate language docum…
pdcp1 Dec 20, 2024
8ce62db
Refactor Translator script to consolidate report translations into a …
pdcp1 Dec 20, 2024
dd6972e
Merge branch 'staging' into feature/new-translations-reports-collection
pdcp1 Dec 23, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
77 changes: 77 additions & 0 deletions site/gatsby-site/migrations/2024.12.11T14.39.38.refactor-reports-translations.js
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,77 @@
/** @type {import('umzug').MigrationFn<any>} */
exports.up = async ({ context: { client } }) => {
const db = client.db('translations');

const reportsEnCollection = db.collection('reports_en');

const reportsEsCollection = db.collection('reports_es');

const reportsFrCollection = db.collection('reports_fr');

const reportsJaCollection = db.collection('reports_ja');

const reportsCollection = db.collection('reports');

// Fetch all documents from reports_en, reports_es, reports_fr, and reports_ja
const reportsEn = await reportsEnCollection.find().toArray();

const reportsEs = await reportsEsCollection.find().toArray();

const reportsFr = await reportsFrCollection.find().toArray();

const reportsJa = await reportsJaCollection.find().toArray();

// Transform and insert documents into reports with a new field "language"
const transformedReportsEn = reportsEn.map((report) => ({
...report,
language: 'en',
}));

const transformedReportsEs = reportsEs.map((report) => ({
...report,
language: 'es',
}));

const transformedReportsFr = reportsFr.map((report) => ({
...report,
language: 'fr',
}));

const transformedReportsJa = reportsJa.map((report) => ({
...report,
language: 'ja',
}));

// Insert transformed documents into reports
const resultEn = await reportsCollection.insertMany(transformedReportsEn);

const resultEs = await reportsCollection.insertMany(transformedReportsEs);

const resultFr = await reportsCollection.insertMany(transformedReportsFr);

const resultJa = await reportsCollection.insertMany(transformedReportsJa);

console.log(
`${resultEn.insertedCount} documents with language "en" were inserted into the "reports" collection.`
);
console.log(
`${resultEs.insertedCount} documents with language "es" were inserted into the "reports" collection.`
);
console.log(
`${resultFr.insertedCount} documents with language "fr" were inserted into the "reports" collection.`
);
console.log(
`${resultJa.insertedCount} documents with language "ja" were inserted into the "reports" collection.`
);
};

/** @type {import('umzug').MigrationFn<any>} */
exports.down = async ({ context: { client } }) => {
const db = client.db('translations');

const reportsCollection = db.collection('reports');

// Drop the reports collection
await reportsCollection.drop();
console.log('Collection "reports" has been dropped.');
};
189 changes: 161 additions & 28 deletions site/gatsby-site/playwright/e2e/unit/translator.spec.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -102,16 +102,33 @@ test('Translations - Should translate languages only if report language differs
insertMany: sinon.stub().resolves({ insertedCount: 1 }),
};

const reportsTranslationsCollection = {
insertMany: sinon.stub().resolves({ insertedCount: 1 }),
};

const mongoClient = {
connect: sinon.stub().resolves(),
close: sinon.stub().resolves(),
db: sinon.stub().returns({
collection: (name: string) => {
if (name === 'reports') return reportsCollection;
if (name === 'reports_en') return reportsENCollection;
if (name === 'reports_es') return reportsESCollection;
return null;
},
db: sinon.stub().callsFake((dbName: string) => {
if (dbName === 'aiidprod') {
return {
collection: (name: string) => {
if (name === 'reports') return reportsCollection;
return null;
},
};
} else if (dbName === 'translations') {
return {
collection: (name: string) => {
if (name === 'reports_en') return reportsENCollection;
if (name === 'reports_es') return reportsESCollection;
if (name === 'reports') return reportsTranslationsCollection;
return null;
},
};
} else {
throw new Error(`Unexpected database name: ${dbName}`);
}
}),
};

Expand Down Expand Up @@ -154,6 +171,32 @@ test('Translations - Should translate languages only if report language differs
title: 'test-es-Report 1 title',
plain_text: 'test-es-Report 1 text\n',
}]);

sinon.assert.calledTwice(reportsTranslationsCollection.insertMany);
sinon.assert.calledWith(reportsTranslationsCollection.insertMany, [
{
report_number: 2,
text: 'test-en-Reporte 2 **texto**',
title: 'test-en-Título del reporte 2',
plain_text: 'test-en-Reporte 2 texto\n',
language: 'en',
},
{
report_number: 3,
text: 'test-en-Reporte 3 **texto**',
title: 'test-en-Título del reporte 3',
plain_text: 'test-en-Reporte 3 texto\n',
language: 'en',
}
]);
sinon.assert.calledWith(reportsTranslationsCollection.insertMany, [{
report_number: 1,
text: 'test-es-Report 1 **text**',
title: 'test-es-Report 1 title',
plain_text: 'test-es-Report 1 text\n',
language: 'es',
}]);

sinon.assert.calledOnce(mongoClient.close);
});

Expand Down Expand Up @@ -184,16 +227,33 @@ test("Translations - Shouldn't call Google's translate api and use translation p
insertMany: sinon.stub().resolves({ insertedCount: 1 }),
};

const reportsTranslationsCollection = {
insertMany: sinon.stub().resolves({ insertedCount: 1 }),
};

const mongoClient = {
connect: sinon.stub().resolves(),
close: sinon.stub().resolves(),
db: sinon.stub().returns({
collection: (name: string) => {
if (name === 'reports') return reportsCollection;
if (name === 'reports_en') return reportsENCollection;
if (name === 'reports_es') return reportsESCollection;
return null;
},
db: sinon.stub().callsFake((dbName: string) => {
if (dbName === 'aiidprod') {
return {
collection: (name: string) => {
if (name === 'reports') return reportsCollection;
return null;
},
};
} else if (dbName === 'translations') {
return {
collection: (name: string) => {
if (name === 'reports_en') return reportsENCollection;
if (name === 'reports_es') return reportsESCollection;
if (name === 'reports') return reportsTranslationsCollection;
return null;
},
};
} else {
throw new Error(`Unexpected database name: ${dbName}`);
}
}),
};

Expand Down Expand Up @@ -236,6 +296,32 @@ test("Translations - Shouldn't call Google's translate api and use translation p
title: 'translated-es-Report 1 title',
plain_text: 'translated-es-Report 1 text\n',
}]);

sinon.assert.calledTwice(reportsTranslationsCollection.insertMany);
sinon.assert.calledWith(reportsTranslationsCollection.insertMany, [
{
report_number: 2,
text: 'translated-en-Reporte 2 **texto**',
title: 'translated-en-Título del reporte 2',
plain_text: 'translated-en-Reporte 2 texto\n',
language: 'en',
},
{
report_number: 3,
text: 'translated-en-Reporte 3 **texto**',
title: 'translated-en-Título del reporte 3',
plain_text: 'translated-en-Reporte 3 texto\n',
language: 'en',
}
]);
sinon.assert.calledWith(reportsTranslationsCollection.insertMany, [{
report_number: 1,
text: 'translated-es-Report 1 **text**',
title: 'translated-es-Report 1 title',
plain_text: 'translated-es-Report 1 text\n',
language: 'es',
}]);

sinon.assert.calledOnce(mongoClient.close);
});

Expand Down Expand Up @@ -268,16 +354,33 @@ test('Translations - Should translate reports with submission date greater than
insertMany: sinon.stub().resolves({ insertedCount: 1 }),
};

const reportsTranslationsCollection = {
insertMany: sinon.stub().resolves({ insertedCount: 1 }),
};

const mongoClient = {
connect: sinon.stub().resolves(),
close: sinon.stub().resolves(),
db: sinon.stub().returns({
collection: (name: string) => {
if (name === 'reports') return reportsCollection;
if (name === 'reports_en') return reportsENCollection;
if (name === 'reports_es') return reportsESCollection;
return null;
},
db: sinon.stub().callsFake((dbName: string) => {
if (dbName === 'aiidprod') {
return {
collection: (name: string) => {
if (name === 'reports') return reportsCollection;
return null;
},
};
} else if (dbName === 'translations') {
return {
collection: (name: string) => {
if (name === 'reports_en') return reportsENCollection;
if (name === 'reports_es') return reportsESCollection;
if (name === 'reports') return reportsTranslationsCollection;
return null;
},
};
} else {
throw new Error(`Unexpected database name: ${dbName}`);
}
}),
};

Expand Down Expand Up @@ -310,6 +413,18 @@ test('Translations - Should translate reports with submission date greater than
}
]);
sinon.assert.notCalled(reportsESCollection.insertMany);

sinon.assert.calledOnce(reportsTranslationsCollection.insertMany);
sinon.assert.calledWith(reportsTranslationsCollection.insertMany, [
{
report_number: 3,
text: 'test-en-Reporte 3 **texto**',
title: 'test-en-Título del reporte 3',
plain_text: 'test-en-Reporte 3 texto\n',
language: 'en',
}
]);

sinon.assert.calledOnce(mongoClient.close);
});

Expand Down Expand Up @@ -363,16 +478,33 @@ test('Translations - Should not translate if the report was already translated',
insertMany: sinon.stub().resolves({ insertedCount: 1 }),
};

const reportsTranslationsCollection = {
insertMany: sinon.stub().resolves({ insertedCount: 1 }),
};

const mongoClient = {
connect: sinon.stub().resolves(),
close: sinon.stub().resolves(),
db: sinon.stub().returns({
collection: (name: string) => {
if (name === 'reports') return reportsCollection;
if (name === 'reports_en') return reportsENCollection;
if (name === 'reports_es') return reportsESCollection;
return null;
},
db: sinon.stub().callsFake((dbName: string) => {
if (dbName === 'aiidprod') {
return {
collection: (name: string) => {
if (name === 'reports') return reportsCollection;
return null;
},
};
} else if (dbName === 'translations') {
return {
collection: (name: string) => {
if (name === 'reports_en') return reportsENCollection;
if (name === 'reports_es') return reportsESCollection;
if (name === 'reports') return reportsTranslationsCollection;
return null;
},
};
} else {
throw new Error(`Unexpected database name: ${dbName}`);
}
}),
};

Expand All @@ -395,5 +527,6 @@ test('Translations - Should not translate if the report was already translated',
sinon.assert.notCalled(translateClient.translate);
sinon.assert.notCalled(reportsENCollection.insertMany);
sinon.assert.notCalled(reportsESCollection.insertMany);
sinon.assert.notCalled(reportsTranslationsCollection.insertMany);
sinon.assert.calledOnce(mongoClient.close);
});
10 changes: 9 additions & 1 deletion site/gatsby-site/src/scripts/Translator.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -108,6 +108,8 @@ class Translator {
.db('translations')
.collection(`reports_${language}`);

const reportsTranslationsCollection = this.mongoClient.db('translations').collection('reports');

const translated = [];

for (const item of items) {
Expand All @@ -118,7 +120,13 @@ class Translator {
translated.push({ report_number, text, title, plain_text });
}

return reportsTranslatedCollection.insertMany(translated);
// TODO: remove this line when the "reports_xx" collections are no longer needed
await reportsTranslatedCollection.insertMany(translated);

// Insert the translated reports into the reports collection with the language field
const reportsTranslated = translated.map((t) => ({ ...t, language }));

return reportsTranslationsCollection.insertMany(reportsTranslated);
}

async translateReport({ entry, to }) {
Expand Down
Loading