+ 
+
diff --git a/site/gatsby-site/blog/join-raic/index.fr.mdx b/site/gatsby-site/blog/join-raic/index.fr.mdx
index 7a419b07fc..94466db376 100644
--- a/site/gatsby-site/blog/join-raic/index.fr.mdx
+++ b/site/gatsby-site/blog/join-raic/index.fr.mdx
@@ -1,5 +1,5 @@
---
-titre: "Rejoignez l'équipe fondatrice de Responsible AI Collaborative"
+title: "Rejoignez l'équipe fondatrice de Responsible AI Collaborative"
metaTitle: "Rejoignez l'équipe fondatrice de Responsible AI Collaborative"
metaDescription: ''
date: '2022-03-29'
diff --git a/site/gatsby-site/gatsby-node.js b/site/gatsby-site/gatsby-node.js
index 99fa3f5c34..7161cdbd68 100644
--- a/site/gatsby-site/gatsby-node.js
+++ b/site/gatsby-site/gatsby-node.js
@@ -4,8 +4,6 @@ const fs = require('fs');
const { Client: GoogleMapsAPIClient } = require('@googlemaps/google-maps-services-js');
-const { Translate } = require('@google-cloud/translate').v2;
-
const { startCase, differenceWith } = require('lodash');
const config = require('./config');
@@ -38,8 +36,6 @@ const createMissingTranslationsPage = require('./page-creators/createMissingTran
const algoliasearch = require('algoliasearch');
-const Translator = require('./src/utils/Translator');
-
const { MongoClient } = require('mongodb');
const { getLanguages } = require('./i18n');
@@ -286,10 +282,11 @@ exports.onPreBootstrap = async ({ reporter }) => {
migrationsActivity.end();
}
+ // Algolia index update process
if (process.env.CONTEXT === 'production') {
- const translationsActivity = reporter.activityTimer(`Translations`);
+ const algoliaUpdaterActivity = reporter.activityTimer(`Algolia`);
- translationsActivity.start();
+ algoliaUpdaterActivity.start();
const configuredLanguages = getLanguages();
@@ -309,30 +306,15 @@ exports.onPreBootstrap = async ({ reporter }) => {
if (
config.mongodb.translationsConnectionString &&
- config.i18n.translateApikey &&
config.i18n.availableLanguages &&
config.header.search.algoliaAdminKey &&
config.header.search.algoliaAppId
) {
- if (process.env.TRANSLATE_DRY_RUN !== 'false') {
- reporter.warn(
- 'Please set `TRANSLATE_DRY_RUN=false` to disable dry running of translation process.'
- );
- }
-
- translationsActivity.setStatus('Translating incident reports...');
-
- const translateClient = new Translate({ key: config.i18n.translateApikey });
-
const mongoClient = new MongoClient(config.mongodb.translationsConnectionString);
const languages = getLanguages();
- const translator = new Translator({ mongoClient, translateClient, languages, reporter });
-
- await translator.run();
-
- translationsActivity.setStatus('Updating incidents indexes...');
+ algoliaUpdaterActivity.setStatus('Updating Algolia incidents indexes...');
try {
const algoliaClient = algoliasearch(
@@ -352,12 +334,12 @@ exports.onPreBootstrap = async ({ reporter }) => {
reporter.panicOnBuild('Error updating Algolia index:', e);
}
} else {
- throw `Missing environment variable, can't run translation process.`;
+ throw `Missing environment variable, can't run Algolia update process.`;
}
- translationsActivity.end();
+ algoliaUpdaterActivity.end();
} else {
- reporter.warn('Netlify CONTEXT is not production, skipping translations.');
+ reporter.warn('Netlify CONTEXT is not production, skipping Algolia index update process.');
}
};
diff --git a/site/gatsby-site/migrations/2024.11.13T19.53.42.delete-all-translation-placeholders.js b/site/gatsby-site/migrations/2024.11.13T19.53.42.delete-all-translation-placeholders.js
new file mode 100644
index 0000000000..3c31d0ba36
--- /dev/null
+++ b/site/gatsby-site/migrations/2024.11.13T19.53.42.delete-all-translation-placeholders.js
@@ -0,0 +1,24 @@
+const languages = require('../i18n/config.json');
+
+/**
+ *
+ * @param {{context: {client: import('mongodb').MongoClient}}} context
+ */
+
+exports.up = async ({ context: { client } }) => {
+ const db = client.db('translations');
+
+ for (const language of languages) {
+ const name = `reports_${language.code}`;
+
+ console.log(`Deleting dummy translations from ${name}`);
+
+ const translations = db.collection(name);
+
+ const result = await translations.deleteMany({
+ text: /^translated-/,
+ });
+
+ console.log(`Deleted ${result.deletedCount} dummy translations for ${language.code}`);
+ }
+};
diff --git a/site/gatsby-site/package.json b/site/gatsby-site/package.json
index 63f1a9b2ab..80e8cf9db6 100644
--- a/site/gatsby-site/package.json
+++ b/site/gatsby-site/package.json
@@ -138,9 +138,10 @@
"codegen": "graphql-codegen --config codegen.ts",
"start:api": "node --env-file=.env --watch -r ts-node/register server/index.ts",
"start:memory-mongo": "node --env-file=.env -r ts-node/register playwright/memory-mongo.ts",
+ "start:memory-mongo:ci": "node -r ts-node/register playwright/memory-mongo.ts",
"process-notifications": "node --env-file=.env -r ts-node/register src/scripts/process-notifications.ts",
"process-notifications:ci": "node -r ts-node/register src/scripts/process-notifications.ts",
- "start:memory-mongo:ci": "node -r ts-node/register playwright/memory-mongo.ts"
+ "translate-reports": "node src/scripts/translateReports.js"
},
"devDependencies": {
"@babel/plugin-proposal-export-default-from": "^7.23.3",
diff --git a/site/gatsby-site/playwright/e2e-full/subscription.spec.ts b/site/gatsby-site/playwright/e2e-full/subscription.spec.ts
index bca263bf6f..fa3b91eb15 100644
--- a/site/gatsby-site/playwright/e2e-full/subscription.spec.ts
+++ b/site/gatsby-site/playwright/e2e-full/subscription.spec.ts
@@ -13,7 +13,7 @@ test.describe('Subscriptions', () => {
await init();
- const [userId] = await login(process.env.E2E_ADMIN_USERNAME, process.env.E2E_ADMIN_PASSWORD, { customData: { roles: ['admin'], first_name: 'John', last_name: 'Doe' } });
+ const [userId] = await login(process.env.E2E_ADMIN_USERNAME, process.env.E2E_ADMIN_PASSWORD, { customData: { roles: ['subscriber'], first_name: 'John', last_name: 'Doe' } });
const subscriptions: DBSubscription[] = [
{
@@ -36,7 +36,7 @@ test.describe('Subscriptions', () => {
test("Incident Updates: Should display a information message if the user doesn't have subscriptions", async ({ page, login }) => {
await init();
- await login(process.env.E2E_ADMIN_USERNAME, process.env.E2E_ADMIN_PASSWORD, { customData: { roles: ['admin'], first_name: 'John', last_name: 'Doe' } });
+ await login(process.env.E2E_ADMIN_USERNAME, process.env.E2E_ADMIN_PASSWORD, { customData: { roles: ['subscriber'], first_name: 'John', last_name: 'Doe' } });
await page.goto(url);
@@ -47,7 +47,7 @@ test.describe('Subscriptions', () => {
await init();
- const [userId, accessToken] = await login(process.env.E2E_ADMIN_USERNAME, process.env.E2E_ADMIN_PASSWORD, { customData: { roles: ['admin'], first_name: 'John', last_name: 'Doe' } });
+ const [userId, accessToken] = await login(process.env.E2E_ADMIN_USERNAME, process.env.E2E_ADMIN_PASSWORD, { customData: { roles: ['subscriber'], first_name: 'John', last_name: 'Doe' } });
const subscriptions: DBSubscription[] = [
{
@@ -96,7 +96,7 @@ test.describe('Subscriptions', () => {
test('New Incidents: Should display the switch toggle off if user does not have a subscription', async ({ page, login }) => {
- await login(process.env.E2E_ADMIN_USERNAME, process.env.E2E_ADMIN_PASSWORD, { customData: { roles: ['admin'], first_name: 'John', last_name: 'Doe' } });
+ await login(process.env.E2E_ADMIN_USERNAME, process.env.E2E_ADMIN_PASSWORD, { customData: { roles: ['subscriber'], first_name: 'John', last_name: 'Doe' } });
await page.goto(url);
@@ -109,7 +109,7 @@ test.describe('Subscriptions', () => {
await init();
- const [userId] = await login(process.env.E2E_ADMIN_USERNAME, process.env.E2E_ADMIN_PASSWORD, { customData: { roles: ['admin'], first_name: 'John', last_name: 'Doe' } });
+ const [userId] = await login(process.env.E2E_ADMIN_USERNAME, process.env.E2E_ADMIN_PASSWORD, { customData: { roles: ['subscriber'], first_name: 'John', last_name: 'Doe' } });
const subscriptions: DBSubscription[] = [
{
@@ -152,7 +152,7 @@ test.describe('Subscriptions', () => {
await init();
- const [userId] = await login(process.env.E2E_ADMIN_USERNAME, process.env.E2E_ADMIN_PASSWORD, { customData: { roles: ['admin'], first_name: 'John', last_name: 'Doe' } });
+ const [userId] = await login(process.env.E2E_ADMIN_USERNAME, process.env.E2E_ADMIN_PASSWORD, { customData: { roles: ['subscriber'], first_name: 'John', last_name: 'Doe' } });
const subscriptions: DBSubscription[] = [
{
@@ -182,7 +182,7 @@ test.describe('Subscriptions', () => {
await init();
- const [userId] = await login(process.env.E2E_ADMIN_USERNAME, process.env.E2E_ADMIN_PASSWORD, { customData: { roles: ['admin'], first_name: 'John', last_name: 'Doe' } });
+ const [userId] = await login(process.env.E2E_ADMIN_USERNAME, process.env.E2E_ADMIN_PASSWORD, { customData: { roles: ['subscriber'], first_name: 'John', last_name: 'Doe' } });
const subscriptions: DBSubscription[] = [
{
@@ -212,7 +212,7 @@ test.describe('Subscriptions', () => {
await init();
- await login(process.env.E2E_ADMIN_USERNAME, process.env.E2E_ADMIN_PASSWORD, { customData: { roles: ['admin'], first_name: 'John', last_name: 'Doe' } });
+ await login(process.env.E2E_ADMIN_USERNAME, process.env.E2E_ADMIN_PASSWORD, { customData: { roles: ['subscriber'], first_name: 'John', last_name: 'Doe' } });
await page.goto(url);
@@ -224,7 +224,7 @@ test.describe('Subscriptions', () => {
await init();
- const [userId, accessToken] = await login(process.env.E2E_ADMIN_USERNAME, process.env.E2E_ADMIN_PASSWORD, { customData: { roles: ['admin'], first_name: 'John', last_name: 'Doe' } });
+ const [userId, accessToken] = await login(process.env.E2E_ADMIN_USERNAME, process.env.E2E_ADMIN_PASSWORD, { customData: { roles: ['subscriber'], first_name: 'John', last_name: 'Doe' } });
const subscriptions: DBSubscription[] = [
{
diff --git a/site/gatsby-site/playwright/e2e/translationBadge.spec.ts b/site/gatsby-site/playwright/e2e-full/translationBadge.spec.ts
similarity index 76%
rename from site/gatsby-site/playwright/e2e/translationBadge.spec.ts
rename to site/gatsby-site/playwright/e2e-full/translationBadge.spec.ts
index 5964c8845b..c86902cf79 100644
--- a/site/gatsby-site/playwright/e2e/translationBadge.spec.ts
+++ b/site/gatsby-site/playwright/e2e-full/translationBadge.spec.ts
@@ -20,9 +20,14 @@ test.describe('Translation Badges', () => {
await expect(page.locator('[data-cy="5d34b8c29ced494f010ed45c"]').locator('[data-cy="translation-badge"]').getByText('Traducido por IA')).toBeVisible();
});
- test('Should be visible on an incident card on the citation page', async ({ page, skipOnEmptyEnvironment }) => {
- await page.goto('/es/cite/1#r1');
- await expect(page.locator('#r1').locator('[data-cy="translation-badge"]').getByText('Traducido por IA')).toBeVisible();
+ test('Should be visible on an report card on the citation page if it was translated', async ({ page, skipOnEmptyEnvironment }) => {
+ await page.goto('/es/cite/3#r3');
+ await expect(page.locator('#r3').locator('[data-cy="translation-badge"]').getByText('Traducido por IA')).toBeVisible();
+ });
+
+ test('Should not be visible on an report card on the citation page if it was not translated', async ({ page, skipOnEmptyEnvironment }) => {
+ await page.goto('/es/cite/3#r4');
+ await expect(page.locator('#r4').locator('[data-cy="translation-badge"]').getByText('Traducido por IA')).not.toBeVisible();
});
test('Should be visible on documentation pages', async ({ page }) => {
diff --git a/site/gatsby-site/playwright/e2e/blog.spec.ts b/site/gatsby-site/playwright/e2e/blog.spec.ts
index 35c79b1473..42d384a7d1 100644
--- a/site/gatsby-site/playwright/e2e/blog.spec.ts
+++ b/site/gatsby-site/playwright/e2e/blog.spec.ts
@@ -3,6 +3,66 @@ import { expect } from '@playwright/test';
test.describe('Blog', () => {
+ test('Should load mdx blog post', async ({ page, skipOnEmptyEnvironment }) => {
+ await page.setViewportSize({ width: 1280, height: 1000 });
+ await page.goto('/blog/the-first-taxonomy-of-ai-incidents');
+
+ await expect(page.locator('.titleWrapper h1')).toHaveText('The First Taxonomy of AI Incidents');
+
+ const div = await page.locator("[data-testid='blog-content']");
+ const textContent = await div.textContent();
+ expect(textContent).toContain('In November the Partnership on AI AI Incident Database (AIID) publicly invited users to instantly search through thousands of pages of text to better understand the limitations of AI products within the real world. Since November, tens of thousands of people from 157 countries have connected to the AIID. Today marks the launch of the next stage of AI Incident Database with its first complete AI incident taxonomy.');
+
+ });
+
+ test('Should load mdx blog post in spanish', async ({ page, skipOnEmptyEnvironment }) => {
+ await page.setViewportSize({ width: 1280, height: 1000 });
+ await page.goto('/es/blog/representation-and-imagination');
+
+ await expect(page.locator('.titleWrapper h1')).toHaveText('Representación e imaginación para prevenir los daños de la IA');
+
+ const div = await page.locator("[data-testid='blog-content']");
+ const textContent = await div.textContent();
+ expect(textContent).toContain('La base de datos de incidentes de IA se lanzó públicamente en noviembre de 2020 por Partnership on AI como un panel de control de los daños de IA realizados en el mundo real.');
+
+ });
+
+ test('Should load mdx blog post in french', async ({ page, skipOnEmptyEnvironment }) => {
+ await page.setViewportSize({ width: 1280, height: 1000 });
+ await page.goto('/fr/blog/join-raic');
+
+ await expect(page.locator('.titleWrapper h1')).toHaveText("Rejoignez l'équipe fondatrice de Responsible AI Collaborative");
+
+ const div = await page.locator("[data-testid='blog-content']");
+ const textContent = await div.textContent();
+ expect(textContent).toContain("La base de données d'incidents d'IA lancée publiquement en novembre 2020 en tant que tableau de bord des dommages causés par l'IA dans le monde réel.");
+
+ });
+
+ test('Should load prismic blog post', async ({ page, skipOnEmptyEnvironment }) => {
+ await page.setViewportSize({ width: 1280, height: 1000 });
+ await page.goto('/blog/incident-report-2024-january');
+
+ await expect(page.locator('h1')).toHaveText('AI Incident Roundup – January ‘24');
+
+ const div = await page.locator("[data-testid='blog-content']");
+ const textContent = await div.textContent();
+ expect(textContent).toContain('Read our month-in-review newsletter recapping new incidents in the AI Incident Database and looking at the trends.');
+
+ });
+
+ test('Should load prismic blog post in spanish', async ({ page, skipOnEmptyEnvironment }) => {
+ await page.setViewportSize({ width: 1280, height: 1000 });
+ await page.goto('/es/blog/incident-report-2024-january');
+
+ await expect(page.locator('h1')).toHaveText('Resumen de incidentes de IA: 24 de enero');
+
+ const div = await page.locator("[data-testid='blog-content']");
+ const textContent = await div.textContent();
+ expect(textContent).toContain('Lea nuestro boletín informativo mensual que resume los nuevos incidentes en la base de datos de incidentes de IA y analiza las tendencias.');
+
+ });
+
test('Should include outline in blog post', async ({ page, skipOnEmptyEnvironment }) => {
await page.setViewportSize({ width: 1280, height: 1000 });
await page.goto('/blog/the-first-taxonomy-of-ai-incidents');
diff --git a/site/gatsby-site/playwright/e2e/doc.spec.ts b/site/gatsby-site/playwright/e2e/doc.spec.ts
new file mode 100644
index 0000000000..96fd4de374
--- /dev/null
+++ b/site/gatsby-site/playwright/e2e/doc.spec.ts
@@ -0,0 +1,129 @@
+import { test } from '../utils';
+import { expect } from '@playwright/test';
+
+test.describe('Docs', () => {
+
+ test('Should load mdx document in English', async ({ page, skipOnEmptyEnvironment }) => {
+ await page.setViewportSize({ width: 1280, height: 1000 });
+ await page.goto('/research/4-related-work');
+
+ await expect(page.locator('.titleWrapper h1')).toHaveText('Related Work');
+
+ const div = await page.locator("[data-testid='doc-content']");
+ const textContent = await div.textContent();
+ expect(textContent).toContain('While formal AI incident research is relatively new, a number of people have been collecting what could be considered incidents. These include,');
+
+ });
+
+ test('Should load mdx doc in spanish', async ({ page, skipOnEmptyEnvironment }) => {
+ await page.setViewportSize({ width: 1280, height: 1000 });
+ await page.goto('/es/research/4-related-work'); // This doc hasn't been translated yet to Spanish yet, so it should fallback to English
+
+ await expect(page.locator('.titleWrapper h1')).toHaveText('Related Work');
+
+ const div = await page.locator("[data-testid='doc-content']");
+ const textContent = await div.textContent();
+ expect(textContent).toContain('While formal AI incident research is relatively new, a number of people have been collecting what could be considered incidents. These include,');
+
+ });
+
+ test('Should load mdx doc in french', async ({ page, skipOnEmptyEnvironment }) => {
+
+ await page.setViewportSize({ width: 1280, height: 1000 });
+ await page.goto('/fr/research/4-related-work'); // This doc hasn't been translated to French yet, so it will default to English
+
+ await expect(page.locator('.titleWrapper h1')).toHaveText('Related Work');
+
+ const div = await page.locator("[data-testid='doc-content']");
+ const textContent = await div.textContent();
+ expect(textContent).toContain('While formal AI incident research is relatively new, a number of people have been collecting what could be considered incidents. These include,');
+
+ });
+
+ test('Should load mdx doc in japanese', async ({ page, skipOnEmptyEnvironment }) => {
+
+ await page.setViewportSize({ width: 1280, height: 1000 });
+ await page.goto('/ja/research/4-related-work');
+
+ await expect(page.locator('.titleWrapper h1')).toHaveText('関連する研究');
+
+ const div = await page.locator("[data-testid='doc-content']");
+ const textContent = await div.textContent();
+ expect(textContent).toContain('公式なAIインシデント研究は比較的新しいものですが、何人かの人々がインシデントと見なされる可能性のある事例を収集しています。これには');
+
+ });
+
+ test('Should load prismic doc post', async ({ page, skipOnEmptyEnvironment }) => {
+ await page.setViewportSize({ width: 1280, height: 1000 });
+ await page.goto('/about');
+
+ await expect(page.locator('h1')).toHaveText('About');
+
+ const div = await page.locator("[data-testid='markdown-content']");
+ const textContent = await div.textContent();
+ expect(textContent).toContain('Intelligent systems are currently prone to unforeseen and often dangerous failures when they are deployed to the real world.');
+
+ });
+
+ test('Should load prismic doc post in spanish', async ({ page, skipOnEmptyEnvironment }) => {
+ await page.setViewportSize({ width: 1280, height: 1000 });
+ await page.goto('/es/about');
+
+ await expect(page.locator('h1')).toHaveText('Acerca de');
+
+ const div = await page.locator("[data-testid='markdown-content']");
+ const textContent = await div.textContent();
+ expect(textContent).toContain('Actualmente, los sistemas inteligentes son propensos a sufrir fallos imprevistos y, a menudo, peligrosos cuando se implementan en el mundo real.');
+
+ });
+
+ test('Should include outline in About page', async ({ page, skipOnEmptyEnvironment }) => {
+ await page.setViewportSize({ width: 1280, height: 1000 });
+ await page.goto('/about');
+
+ const outlineItems = page.locator('[data-cy="outline"] > li');
+ await expect(outlineItems).toHaveCount(7);
+
+ await expect(page.locator('[data-cy="outline"]:has-text("Why \\"AI Incidents\\"?")')).toBeVisible();
+ await expect(page.locator('[data-cy="outline"]:has-text("What is an Incident?")')).toBeVisible();
+ await expect(page.locator('[data-cy="outline"]:has-text("Current and Future Users")')).toBeVisible();
+ await expect(page.locator('[data-cy="outline"]:has-text("When Should You Report an Incident?")')).toBeVisible();
+ await expect(page.locator('[data-cy="outline"]:has-text("Board of Directors")')).toBeVisible();
+ await expect(page.locator('[data-cy="outline"]:has-text("Collaborators")')).toBeVisible();
+
+ await page.setViewportSize({ width: 800, height: 1000 });
+
+
+ await expect(page.locator('[data-cy="outline"]:has-text("Why \\"AI Incidents\\"?")')).not.toBeVisible();
+ await expect(page.locator('[data-cy="outline"]:has-text("What is an Incident?")')).not.toBeVisible();
+ await expect(page.locator('[data-cy="outline"]:has-text("Current and Future Users")')).not.toBeVisible();
+ await expect(page.locator('[data-cy="outline"]:has-text("When Should You Report an Incident?")')).not.toBeVisible();
+ await expect(page.locator('[data-cy="outline"]:has-text("Board of Directors")')).not.toBeVisible();
+ await expect(page.locator('[data-cy="outline"]:has-text("Collaborators")')).not.toBeVisible();
+ });
+
+ test('Should include outline in Spanish About page', async ({ page, skipOnEmptyEnvironment }) => {
+ await page.setViewportSize({ width: 1280, height: 1000 });
+ await page.goto('/es/about');
+
+ const outlineItems = page.locator('[data-cy="outline"] > li');
+ await expect(outlineItems).toHaveCount(7);
+
+ await expect(page.locator('[data-cy="outline"]:has-text("¿Por qué \\"incidentes de IA\\"?")')).toBeVisible();
+ await expect(page.locator('[data-cy="outline"]:has-text("¿Qué es un incidente?")')).toBeVisible();
+ await expect(page.locator('[data-cy="outline"]:has-text("Usuarios actuales y futuros")')).toBeVisible();
+ await expect(page.locator('[data-cy="outline"]:has-text("¿Cuándo debería informar un incidente?")')).toBeVisible();
+ await expect(page.locator('[data-cy="outline"]:has-text("Junta Directiva")')).toBeVisible();
+ await expect(page.locator('[data-cy="outline"]:has-text("Colaboradores")')).toBeVisible();
+
+ await page.setViewportSize({ width: 800, height: 1000 });
+
+
+ await expect(page.locator('[data-cy="outline"]:has-text("¿Por qué \\"incidentes de IA\\"?")')).not.toBeVisible();
+ await expect(page.locator('[data-cy="outline"]:has-text("¿Qué es un incidente?")')).not.toBeVisible();
+ await expect(page.locator('[data-cy="outline"]:has-text("Usuarios actuales y futuros")')).not.toBeVisible();
+ await expect(page.locator('[data-cy="outline"]:has-text("¿Cuándo debería informar un incidente?")')).not.toBeVisible();
+ await expect(page.locator('[data-cy="outline"]:has-text("Junta Directiva")')).not.toBeVisible();
+ await expect(page.locator('[data-cy="outline"]:has-text("Colaboradores")')).not.toBeVisible();
+ });
+});
\ No newline at end of file
diff --git a/site/gatsby-site/playwright/e2e/unit/translator.spec.ts b/site/gatsby-site/playwright/e2e/unit/translator.spec.ts
index e998bbeb03..35b64d1725 100644
--- a/site/gatsby-site/playwright/e2e/unit/translator.spec.ts
+++ b/site/gatsby-site/playwright/e2e/unit/translator.spec.ts
@@ -1,12 +1,17 @@
import { ObjectId } from 'bson';
import { test } from '../../utils';
-import Translator from '../../../src/utils/Translator';
+import Translator from '../../../src/scripts/Translator';
import sinon from 'sinon';
const reports = [
{
_id: new ObjectId('60dd465f80935bc89e6f9b01'),
- authors: ['Alistair Barr'],
+ report_number: 1,
+ title: 'Report 1 title',
+ text: 'Report 1 **text**',
+ plain_text: 'Report 1 text',
+ language: 'en',
+ authors: ['Jhon Doe'],
date_downloaded: '2019-04-13',
date_modified: '2020-06-14',
date_published: '2015-05-19',
@@ -17,61 +22,65 @@ const reports = [
epoch_date_published: 1431993600,
epoch_date_submitted: 1559347200,
image_url: 'http://url.com',
- language: 'en',
- report_number: 1,
source_domain: 'blogs.wsj.com',
- submitters: ['Roman Yampolskiy'],
+ submitters: ['Jason Smith'],
tags: [],
- text: 'Report 1 **text**',
- plain_text: 'Report 1 text',
- title: 'Report 1 title',
url: 'https://url.com/stuff',
},
{
_id: new ObjectId('60dd465f80935bc89e6f9b02'),
- authors: ['Alistair Barr'],
+ report_number: 2,
+ title: 'Título del reporte 2',
+ text: 'Reporte 2 **texto**',
+ plain_text: 'Reporte 2 texto',
+ language: 'es',
+ authors: ['Juan Perez'],
date_downloaded: '2019-04-13',
date_modified: '2020-06-14',
date_published: '2015-05-19',
- date_submitted: '2019-06-01',
- description: 'Description of report 2',
+ date_submitted: '2020-06-01',
+ description: 'Descripción del reporte 2',
epoch_date_downloaded: 1555113600,
epoch_date_modified: 1592092800,
epoch_date_published: 1431993600,
epoch_date_submitted: 1559347200,
image_url: 'http://url.com',
+ source_domain: 'blogs.wsj.com',
+ submitters: ['Ramon Gomez'],
+ tags: [],
+ url: 'https://url.com/stuff',
+ },
+ {
+ _id: new ObjectId('60dd465f80935bc89e6f9b03'),
+ report_number: 3,
+ title: 'Título del reporte 3',
+ text: 'Reporte 3 **texto**',
+ plain_text: 'Reporte 3 texto',
language: 'es',
- report_number: 2,
+ authors: ['Juan Perez'],
+ date_downloaded: '2019-04-13',
+ date_modified: '2020-06-14',
+ date_published: '2015-05-19',
+ date_submitted: '2021-06-01',
+ description: 'Descripción del reporte 3',
+ epoch_date_downloaded: 1555113600,
+ epoch_date_modified: 1592092800,
+ epoch_date_published: 1431993600,
+ epoch_date_submitted: 1559347200,
+ image_url: 'http://url.com',
source_domain: 'blogs.wsj.com',
- submitters: ['Roman Yampolskiy'],
+ submitters: ['Ramon Gomez'],
tags: [],
- text: 'Report 2 **text**',
- plain_text: 'Report 2 text',
- title: 'Report 2 title',
url: 'https://url.com/stuff',
},
];
test('Translations - Should translate languages only if report language differs from target language', async ({ page }) => {
- const translatedReportsEN = [
- {
- _id: '61d5ad9f102e6e30fca90ddf',
- text: 'translated-en-text report 1',
- title: 'translated-en-title report 1',
- report_number: 1,
- },
- ];
+ const translatedReportsEN = [];
- const translatedReportsES = [
- {
- _id: '61d5ad9f102e6e30fca90ddf',
- text: 'translated-es-text report 2',
- title: 'translated-es-title report 2',
- report_number: 2,
- },
- ];
+ const translatedReportsES = [];
- const reporter = { log: sinon.stub() };
+ const reporter = { log: sinon.stub(), error: sinon.stub(), warn: sinon.stub() };
const reportsCollection = {
find: sinon.stub().returns({
@@ -121,13 +130,23 @@ test('Translations - Should translate languages only if report language differs
await translator.run();
sinon.assert.calledOnce(mongoClient.connect);
+ sinon.assert.calledOnce(reportsCollection.find);
+ sinon.assert.calledThrice(translateClient.translate);
sinon.assert.calledOnce(reportsENCollection.insertMany);
- sinon.assert.calledWith(reportsENCollection.insertMany, [{
- report_number: 2,
- text: 'test-en-Report 2 **text**',
- title: 'test-en-Report 2 title',
- plain_text: 'test-en-Report 2 text\n',
- }]);
+ sinon.assert.calledWith(reportsENCollection.insertMany, [
+ {
+ report_number: 2,
+ text: 'test-en-Reporte 2 **texto**',
+ title: 'test-en-Título del reporte 2',
+ plain_text: 'test-en-Reporte 2 texto\n',
+ },
+ {
+ report_number: 3,
+ text: 'test-en-Reporte 3 **texto**',
+ title: 'test-en-Título del reporte 3',
+ plain_text: 'test-en-Reporte 3 texto\n'
+ }
+ ]);
sinon.assert.calledOnce(reportsESCollection.insertMany);
sinon.assert.calledWith(reportsESCollection.insertMany, [{
report_number: 1,
@@ -138,26 +157,191 @@ test('Translations - Should translate languages only if report language differs
sinon.assert.calledOnce(mongoClient.close);
});
-test("Translations - Shouldn't call Google's translate api if dryRun is true", async ({ page }) => {
- const translatedReportsEN = [
+test("Translations - Shouldn't call Google's translate api and use translation placeholders if dryRun is true", async ({ page }) => {
+ const translatedReportsEN = [];
+
+ const translatedReportsES = [];
+
+ const reporter = { log: sinon.stub(), error: sinon.stub(), warn: sinon.stub() };
+
+ const reportsCollection = {
+ find: sinon.stub().returns({
+ toArray: sinon.stub().resolves(reports),
+ }),
+ };
+
+ const reportsENCollection = {
+ find: sinon.stub().returns({
+ toArray: sinon.stub().resolves(translatedReportsEN),
+ }),
+ insertMany: sinon.stub().resolves({ insertedCount: 1 }),
+ };
+
+ const reportsESCollection = {
+ find: sinon.stub().returns({
+ toArray: sinon.stub().resolves(translatedReportsES),
+ }),
+ insertMany: sinon.stub().resolves({ insertedCount: 1 }),
+ };
+
+ const mongoClient = {
+ connect: sinon.stub().resolves(),
+ close: sinon.stub().resolves(),
+ db: sinon.stub().returns({
+ collection: (name: string) => {
+ if (name === 'reports') return reportsCollection;
+ if (name === 'reports_en') return reportsENCollection;
+ if (name === 'reports_es') return reportsESCollection;
+ return null;
+ },
+ }),
+ };
+
+ const translateClient = {
+ translate: sinon.stub().callsFake((payload, { to }) => [payload.map((p: any) => `translated-${to}-${p}`)]),
+ };
+
+ const translator = new Translator({
+ mongoClient,
+ translateClient,
+ languages: [{ code: 'es' }, { code: 'en' }],
+ reporter,
+ dryRun: true,
+ });
+
+ await translator.run();
+
+ sinon.assert.calledOnce(mongoClient.connect);
+ sinon.assert.calledOnce(reportsCollection.find);
+ sinon.assert.notCalled(translateClient.translate);
+ sinon.assert.calledOnce(reportsENCollection.insertMany);
+ sinon.assert.calledWith(reportsENCollection.insertMany, [
+ {
+ report_number: 2,
+ text: 'translated-en-Reporte 2 **texto**',
+ title: 'translated-en-Título del reporte 2',
+ plain_text: 'translated-en-Reporte 2 texto\n',
+ },
+ {
+ report_number: 3,
+ text: 'translated-en-Reporte 3 **texto**',
+ title: 'translated-en-Título del reporte 3',
+ plain_text: 'translated-en-Reporte 3 texto\n'
+ }
+ ]);
+ sinon.assert.calledOnce(reportsESCollection.insertMany);
+ sinon.assert.calledWith(reportsESCollection.insertMany, [{
+ report_number: 1,
+ text: 'translated-es-Report 1 **text**',
+ title: 'translated-es-Report 1 title',
+ plain_text: 'translated-es-Report 1 text\n',
+ }]);
+ sinon.assert.calledOnce(mongoClient.close);
+});
+
+test('Translations - Should translate reports with submission date greater than an specific report submission date', async ({ page }) => {
+ const submissionDateStart = '2021-01-01';
+
+ const translatedReportsES = [];
+
+ const translatedReportsEN = [];
+
+ const reporter = { log: sinon.stub(), error: sinon.stub(), warn: sinon.stub() };
+
+ const reportsCollection = {
+ find: sinon.stub().returns({
+ toArray: sinon.stub().resolves(reports.filter(r => r.date_submitted > submissionDateStart)),
+ }),
+ };
+
+ const reportsENCollection = {
+ find: sinon.stub().returns({
+ toArray: sinon.stub().resolves(translatedReportsEN),
+ }),
+ insertMany: sinon.stub().resolves({ insertedCount: 1 }),
+ };
+
+ const reportsESCollection = {
+ find: sinon.stub().returns({
+ toArray: sinon.stub().resolves(translatedReportsES),
+ }),
+ insertMany: sinon.stub().resolves({ insertedCount: 1 }),
+ };
+
+ const mongoClient = {
+ connect: sinon.stub().resolves(),
+ close: sinon.stub().resolves(),
+ db: sinon.stub().returns({
+ collection: (name: string) => {
+ if (name === 'reports') return reportsCollection;
+ if (name === 'reports_en') return reportsENCollection;
+ if (name === 'reports_es') return reportsESCollection;
+ return null;
+ },
+ }),
+ };
+
+ const translateClient = {
+ translate: sinon.stub().callsFake((payload, { to }) => [payload.map((p: any) => `test-${to}-${p}`)]),
+ };
+
+ const translator = new Translator({
+ mongoClient,
+ translateClient,
+ languages: [{ code: 'es' }, { code: 'en' }],
+ reporter,
+ dryRun: false,
+ submissionDateStart: submissionDateStart,
+ });
+
+ await translator.run();
+
+ sinon.assert.calledOnce(mongoClient.connect);
+ sinon.assert.calledOnce(reportsCollection.find);
+ sinon.assert.calledWith(reportsCollection.find, { date_submitted: { $gte: new Date(submissionDateStart) } });
+ sinon.assert.calledOnce(translateClient.translate);
+ sinon.assert.calledOnce(reportsENCollection.insertMany);
+ sinon.assert.calledWith(reportsENCollection.insertMany, [
+ {
+ report_number: 3,
+ text: 'test-en-Reporte 3 **texto**',
+ title: 'test-en-Título del reporte 3',
+ plain_text: 'test-en-Reporte 3 texto\n'
+ }
+ ]);
+ sinon.assert.notCalled(reportsESCollection.insertMany);
+ sinon.assert.calledOnce(mongoClient.close);
+});
+
+test('Translations - Should not translate if the report was already translated', async ({ page }) => {
+ const translatedReportsES = [
{
_id: '61d5ad9f102e6e30fca90ddf',
- text: 'translated-en-text report 1',
- title: 'translated-en-title report 1',
report_number: 1,
+ title: 'Título del reporte 1',
+ text: 'Reporte 1 **texto**',
+ plain_text: 'Reporte 1 texto',
},
];
- const translatedReportsES = [
+ const translatedReportsEN = [
{
- _id: '61d5ad9f102e6e30fca90ddf',
- text: 'translated-es-text report 2',
- title: 'translated-es-title report 2',
+ _id: '61d5ad9f102e6e30fca90dde',
report_number: 2,
+ title: 'Report 2 title',
+ text: 'Report 2 **text**',
+ plain_text: 'Report 2 text',
+ },
+ {
+ _id: '61d5ad9f102e6e30fca90ddf',
+ report_number: 3,
+ title: 'Report 3 title',
+ text: 'Report 3 **text**',
+ plain_text: 'Report 3 text',
},
];
- const reporter = { log: sinon.stub() };
+ const reporter = { log: sinon.stub(), error: sinon.stub(), warn: sinon.stub() };
const reportsCollection = {
find: sinon.stub().returns({
@@ -193,7 +377,7 @@ test("Translations - Shouldn't call Google's translate api if dryRun is true", a
};
const translateClient = {
- translate: sinon.stub().callsFake((payload, { to }) => [payload.map((p: any) => `translated-${to}-${p}`)]),
+ translate: sinon.stub().callsFake((payload, { to }) => [payload.map((p: any) => `test-${to}-${p}`)]),
};
const translator = new Translator({
@@ -201,10 +385,15 @@ test("Translations - Shouldn't call Google's translate api if dryRun is true", a
translateClient,
languages: [{ code: 'es' }, { code: 'en' }],
reporter,
- dryRun: true,
+ dryRun: false,
});
await translator.run();
+ sinon.assert.calledOnce(mongoClient.connect);
+ sinon.assert.calledOnce(reportsCollection.find);
sinon.assert.notCalled(translateClient.translate);
+ sinon.assert.notCalled(reportsENCollection.insertMany);
+ sinon.assert.notCalled(reportsESCollection.insertMany);
+ sinon.assert.calledOnce(mongoClient.close);
});
diff --git a/site/gatsby-site/plugins/gatsby-theme-i18n/gatsby-node.js b/site/gatsby-site/plugins/gatsby-theme-i18n/gatsby-node.js
index 2a3cf7eec6..bfc953315c 100644
--- a/site/gatsby-site/plugins/gatsby-theme-i18n/gatsby-node.js
+++ b/site/gatsby-site/plugins/gatsby-theme-i18n/gatsby-node.js
@@ -220,50 +220,40 @@ exports.onCreatePage = ({ page, actions }, themeOptions) => {
const [template, mdxFile] = page.component.split(`?__contentFilePath=`);
- // If the mdxFile path has a language, let's strip the language from it.
- // ex: index.de.mdx ==> index.mdx
+ // Process the mdxFile if it exists
if (mdxFile) {
- let thePath = path.dirname(mdxFile);
+ const directory = path.dirname(mdxFile); // Get the directory path
- let fileName = path.basename(mdxFile);
+ const fileName = path.basename(mdxFile, `.mdx`); // Extract the file name without `.mdx`
- let ext = path.extname(mdxFile);
+ const ext = path.extname(mdxFile); // Get the file extension
- if (ext !== '.mdx') {
- throw new Error(`Unexpected file extension in mdx path parsing: ${mdxFile}`);
+ if (ext !== `.mdx`) {
+ throw new Error(`Unexpected file extension in MDX path parsing: ${mdxFile}`);
}
- // Split the filename in three parts split by the dot. We expect two or three components.
- let fileNamePieces = fileName.split('.');
+ // Extract the locale from the file name if present (e.g., "index.es" → "es")
+ const nameParts = fileName.split(`.`);
- if (fileNamePieces.length == 2) {
- // ex: index.mdx
- theFilePath = mdxFile;
- } else if (fileNamePieces.length == 3) {
- // ex: index.es.mdx
- // Keep everything except the language code.
- theFilePath = `${path.join(thePath, fileNamePieces.at(0))}${ext}`;
- } else {
- throw new Error(`Unexpected file format in mdx path parsing: ${mdxFile}`);
- }
-
- // If we use a non-default language, and the language file is on the disk, then use it.
- if (ext === '.mdx' && locale.code !== defaultLang) {
- // ex: /path/up/to/lang/code
- let potentialPath = path.join(path.dirname(thePath), fileNamePieces.at(0));
+ let baseName = nameParts[0]; // Default to the base name without locale
- // ex: /path/up/to/lang/code.es.mdx
- let potentialLangfile = `${potentialPath}.${locale.code}${ext}`;
+ // If it's a non-default language and the file exists, use the localized file
+ if (locale.code !== defaultLang) {
+ const localizedFile = path.join(directory, `${baseName}.${locale.code}${ext}`);
- if (fs.existsSync(potentialLangfile)) {
- theFilePath = potentialLangfile;
+ if (fs.existsSync(localizedFile)) {
+ theFilePath = `${template}?__contentFilePath=${localizedFile}`;
} else {
- // Nothing to render if file doen't exist.
- theFilePath = '';
+ // If no localized file exists, use the default file
+ theFilePath = `${template}?__contentFilePath=${path.join(
+ directory,
+ `${baseName}${ext}`
+ )}`;
}
+ } else {
+ // For the default language, strip the locale from the path if present
+ theFilePath = `${template}?__contentFilePath=${path.join(directory, `${baseName}${ext}`)}`;
}
-
- theFilePath = `${template}?__contentFilePath=${theFilePath}`;
}
const newPage = {
@@ -293,7 +283,6 @@ exports.onCreatePage = ({ page, actions }, themeOptions) => {
// Check if the page is a localized 404
if (newPage.path.match(/^\/[a-z]{2}\/404\/$/)) {
- // Match all paths starting with this code (apart from other valid paths)
newPage.matchPath = `/${locale.code}/*`;
}
diff --git a/site/gatsby-site/server/rules.ts b/site/gatsby-site/server/rules.ts
index fb3cc448f5..c9337f0cbb 100644
--- a/site/gatsby-site/server/rules.ts
+++ b/site/gatsby-site/server/rules.ts
@@ -55,7 +55,7 @@ export const isSubscriptionOwner = () => rule()(
const collection = context.client.db('customData').collection('subscriptions');
const simpleType = getSimplifiedType(SubscriptionType);
- const filter = getMongoDbFilter(simpleType, info.variableValues.filter as GraphQLFilter);
+ const filter = getMongoDbFilter(simpleType, args.filter as GraphQLFilter);
const subscriptions = await collection.find