From c0b599ac7a27b1eeb5c9ae7003f234fcd9d0ac8c Mon Sep 17 00:00:00 2001 From: Guillaume Grossetie Date: Fri, 13 Dec 2024 13:22:28 +0100 Subject: [PATCH] =?UTF-8?q?chore:=20migration=20des=20valeurs=20'legacy'?= =?UTF-8?q?=20dans=20les=20m=C3=A9tadonn=C3=A9es=20(#1143)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * chore: migration des valeurs 'legacy' dans les métadonnées * fix: mise à jour des migrations afin d'être compatible avec les données de production --- .../yamleditor/ArticleEditorMetadataForm.jsx | 4 +- .../src/components/metadata/MetadataForm.jsx | 4 +- .../src/components/metadata/MetadataValues.js | 32 ---------- graphql/helpers/metadata.js | 7 ++- .../20211118080000-working-version.js | 50 +++++++++------- .../20220112080000-article-contributors.js | 33 +++++----- .../20230113080000-sparse-username-index.js | 60 +++++++++++++++++-- .../20230504080000-sorting-indexes.js | 34 ++++++----- .../20230505080000-tag-color-hex.js | 3 - .../20230626080000-version-owner.js | 33 +++++----- .../20230704080000-article-version-missing.js | 33 ++++++---- ...0241017080000-article-passwords-cleanup.js | 1 - .../20241127170000-article-metadata.js | 4 +- ...10110000-article-metadata-legacy-values.js | 58 ++++++++++++++++++ 14 files changed, 226 insertions(+), 130 deletions(-) delete mode 100644 front/src/components/metadata/MetadataValues.js create mode 100644 graphql/migrations/20241210110000-article-metadata-legacy-values.js diff --git a/front/src/components/Write/yamleditor/ArticleEditorMetadataForm.jsx b/front/src/components/Write/yamleditor/ArticleEditorMetadataForm.jsx index 1f81078d1..1211311dd 100644 --- a/front/src/components/Write/yamleditor/ArticleEditorMetadataForm.jsx +++ b/front/src/components/Write/yamleditor/ArticleEditorMetadataForm.jsx @@ -2,7 +2,6 @@ import React, { useCallback, useMemo } from 'react' import { merge } from 'allof-merge' import PropTypes from 'prop-types' import Form from '../../Form' -import { convertLegacyValues } from '../../metadata/MetadataValues.js' import uiSchema from '../../../schemas/article-ui-schema.json' import schema from '../../../schemas/article-metadata.schema.json' @@ -10,7 +9,6 @@ export default function ArticleEditorMetadataForm({ metadata, onChange = () => {}, }) { - const formData = convertLegacyValues(metadata) const schemaMerged = useMemo(() => merge(schema), [schema]) const handleChange = useCallback( (newFormData) => onChange(newFormData), @@ -18,7 +16,7 @@ export default function ArticleEditorMetadataForm({ ) return (
{ - if ( - block && - typeof block === 'object' && - 'list_f' in block && - typeof block.list_f === 'string' - ) { - block.list_f = block.list_f.split(',').map((word) => word.trim()) - } - return block - }) - } - - return data -} diff --git a/graphql/helpers/metadata.js b/graphql/helpers/metadata.js index 7cf454f3d..8632de2ef 100644 --- a/graphql/helpers/metadata.js +++ b/graphql/helpers/metadata.js @@ -444,7 +444,10 @@ function fromLegacyFormat (metadata) { translations, ...extra } = metadata - const translatedAbstracts = abstract?.filter(a => a?.lang !== lang) + const abstractNormalized = typeof abstract === 'string' + ? [{ lang: 'fr', text_f: abstract }] + : abstract + const translatedAbstracts = abstractNormalized?.filter(a => a?.lang !== lang) const translatedTitles = translatedTitle?.filter(a => a?.lang !== lang) const translatedKeywords = keywords?.filter(a => a?.lang !== lang) const languages = Array.from(new Set([...translatedAbstracts?.map(a => a?.lang) ?? [], ...translatedTitles?.map(t => t?.lang) ?? [], ...translatedKeywords?.map(k => k?.lang) ?? []])) @@ -486,7 +489,7 @@ function fromLegacyFormat (metadata) { title: title_f, subtitle: subtitle_f, acknowledgements, - abstract: abstract?.find(a => a?.lang === lang)?.text_f, + abstract: abstractNormalized?.find(a => a?.lang === lang)?.text_f, keywords: keywords?.find(k => k?.lang === lang)?.list_f, controlledKeywords: controlledKeywords, publicationDate: date, diff --git a/graphql/migrations/20211118080000-working-version.js b/graphql/migrations/20211118080000-working-version.js index f2dbbc254..b82a65d41 100644 --- a/graphql/migrations/20211118080000-working-version.js +++ b/graphql/migrations/20211118080000-working-version.js @@ -1,30 +1,34 @@ exports.up = async function (db) { - const articles = await db._find('articles', {}) - - for await (const article of articles) { - const latestVersionId = article.versions.slice(-1)[0] - - if (typeof article.workingVersion === 'undefined') { - if (latestVersionId) { - const [latestVersion] = await db._find('versions', { _id: latestVersionId }) - await db._run('update', 'articles', { - query: { _id: article._id }, - update: { - $set: { - workingVersion: { - bib: latestVersion.bib, - md: latestVersion.md, - yaml: latestVersion.yaml, - }, + const mongo = await db._run('getDbInstance', true) + const articles = mongo.collection('articles') + const versions = mongo.collection('versions') + const articlesCursor = articles.find({}) + try { + while (await articlesCursor.hasNext()) { + const article = await articlesCursor.next() + if (typeof article.workingVersion === 'undefined') { + const latestVersionId = article.versions.slice(-1)[0] + if (latestVersionId) { + const [latestVersion] = await versions.findOne({ _id: latestVersionId }) + await articles.updateOne({ _id: article._id }, { + $set: { + workingVersion: { + bib: latestVersion.bib, + md: latestVersion.md, + yaml: latestVersion.yaml, + } + } }, - }, - options: { upsert: false } - }) - - // if the latest version is "autosave", remove! - await db.collections('versions').deleteOne({ _id: latestVersionId, autosave: true }) + { upsert: false } + ) + // if the latest version is "autosave", remove! + await versions.deleteOne({ _id: latestVersionId, autosave: true }) + } } } + } finally { + await articlesCursor.close() + await mongo.close() } } diff --git a/graphql/migrations/20220112080000-article-contributors.js b/graphql/migrations/20220112080000-article-contributors.js index b9aaadc4b..49ab1e1b6 100644 --- a/graphql/migrations/20220112080000-article-contributors.js +++ b/graphql/migrations/20220112080000-article-contributors.js @@ -1,18 +1,23 @@ exports.up = async function (db) { - const articles = await db._find('articles', {}) - - for await (const article of articles) { - const [ ownerId, ...contributors ] = article.owners ?? [article.owner] - - await db._run('update', 'articles', { - query: { _id: article._id }, - update: { - $set: { - owner: ownerId, - contributors: contributors.map(userId => ({ user: userId, roles: ['read', 'write']})) - } - } - }) + const mongo = await db._run('getDbInstance', true) + const articles = mongo.collection('articles') + const articlesCursor = articles.find({}) + try { + while (await articlesCursor.hasNext()) { + const article = await articlesCursor.next() + const [ownerId, ...contributors] = article.owners ?? [article.owner] + await articles.updateOne( + { _id: article._id }, + { + $set: { + owner: ownerId, + contributors: contributors.map(userId => ({ user: userId, roles: ['read', 'write'] })) + } + }) + } + } finally { + await articlesCursor.close() + await mongo.close() } } diff --git a/graphql/migrations/20230113080000-sparse-username-index.js b/graphql/migrations/20230113080000-sparse-username-index.js index 851d7cb6b..4fd72df51 100644 --- a/graphql/migrations/20230113080000-sparse-username-index.js +++ b/graphql/migrations/20230113080000-sparse-username-index.js @@ -1,10 +1,60 @@ exports.up = async function (db) { const mongo = await db._run("getDbInstance", true) - await mongo - .collection('users') - .createIndex({ username: 1 }, { sparse: true, unique: true }) - - return mongo.close() + try { + const users = mongo.collection("users") + const duplicateUsernameList = await users.aggregate([ + { + $match: { + "username": { + "$exists": true + } + } + }, + { + $group: { + _id: "$username", + count: { + $sum: 1 + }, + duplicates: { + $addToSet: "$_id" + } + } + }, + { + $match: { + count: { + $gt: 1 + } + } + }, + { + $project: { + _id: 0, + name: "$_id", + email: "$email", + duplicates: 1 + } + } + ]).toArray() + // update user with non-unique username + for (const item of duplicateUsernameList) { + for (const duplicate of item.duplicates) { + console.log(`Username ${item.name} is non unique, update username to email on user with id: ${duplicate}`) + await users.updateOne( + { _id: duplicate }, + [{ + $set: { + username: "$email" + } + }] + ) + } + } + await users.createIndex({ username: 1 }, { sparse: true, unique: true }) + } finally { + await mongo.close() + } } exports.down = async function (db) { diff --git a/graphql/migrations/20230504080000-sorting-indexes.js b/graphql/migrations/20230504080000-sorting-indexes.js index 1703c3d91..f26c39b6f 100644 --- a/graphql/migrations/20230504080000-sorting-indexes.js +++ b/graphql/migrations/20230504080000-sorting-indexes.js @@ -1,20 +1,24 @@ exports.up = async function (db) { const mongo = await db._run("getDbInstance", true) - await mongo - .collection('articles') - .createIndex({ updatedAt: -1 }, { unique: false }) - await mongo - .collection('tags') - .createIndex({ name: -1 }, { unique: false }) - await mongo - .collection('tags') - .createIndex({ createdAt: -1 }, { unique: false }) - await mongo - .collection('articles') - .createIndex({ createdAt: -1 }, { unique: false }) - await mongo - .collection('versions') - .createIndex({ createdAt: -1 }, { unique: false }) + try { + await mongo + .collection('articles') + .createIndex({ updatedAt: -1 }, { unique: false }) + await mongo + .collection('tags') + .createIndex({ name: -1 }, { unique: false }) + await mongo + .collection('tags') + .createIndex({ createdAt: -1 }, { unique: false }) + await mongo + .collection('articles') + .createIndex({ createdAt: -1 }, { unique: false }) + await mongo + .collection('versions') + .createIndex({ createdAt: -1 }, { unique: false }) + } finally { + await mongo.close() + } } exports.down = async function (db) { diff --git a/graphql/migrations/20230505080000-tag-color-hex.js b/graphql/migrations/20230505080000-tag-color-hex.js index 73bd995c9..d43b4e33c 100644 --- a/graphql/migrations/20230505080000-tag-color-hex.js +++ b/graphql/migrations/20230505080000-tag-color-hex.js @@ -145,10 +145,8 @@ const colours = { exports.up = async function (db) { const tags = await db._find('tags', {}) - for await (const tag of tags) { let color - if (tag.color && tag.color.charAt(0) !== '#') { color = colours[tag.color] || '#eeeeee' } else if (tag.color && tag.color.length < 7) { @@ -156,7 +154,6 @@ exports.up = async function (db) { } else if (!tag.color) { color = '#eeeeee' } - if (color) { await db._run('update', 'tags', { query: { _id: tag._id }, diff --git a/graphql/migrations/20230626080000-version-owner.js b/graphql/migrations/20230626080000-version-owner.js index 430960157..b82bfb8ff 100644 --- a/graphql/migrations/20230626080000-version-owner.js +++ b/graphql/migrations/20230626080000-version-owner.js @@ -1,23 +1,28 @@ exports.up = async function (db) { - const versions = await db._find('versions', {"owner": null }) - - for await (const version of versions) { - const articleId = version.article - const articles = await db._find('articles', { _id: articleId }) - - for await (const article of articles) { - if (article.owner !== null) { - await db._run('update', 'versions', { - query: { _id: version._id }, - update: { + const mongo = await db._run('getDbInstance', true) + const articles = mongo.collection('articles') + const versions = mongo.collection('versions') + const versionsCursor = versions.find({ "owner": null }) + try { + while (await versionsCursor.hasNext()) { + const version = await versionsCursor.next() + const articleId = version.article + const article = await articles.findOne({ _id: articleId }) + if (article && article.owner !== null) { + versions.updateOne( + { _id: version._id }, + { $set: { owner: article.owner, - }, + } }, - options: { upsert: false } - }) + { upsert: false } + ) } } + } finally { + await versionsCursor.close() + await mongo.close() } } diff --git a/graphql/migrations/20230704080000-article-version-missing.js b/graphql/migrations/20230704080000-article-version-missing.js index 6def334e6..98474e4d1 100644 --- a/graphql/migrations/20230704080000-article-version-missing.js +++ b/graphql/migrations/20230704080000-article-version-missing.js @@ -1,20 +1,27 @@ exports.up = async function (db) { - const articles = await db._find('articles', {}) - - for await (const article of articles) { - const versionsIds = article.versions - - if (Array.isArray(versionsIds)) { - for await (const _id of versionsIds) { - const [version] = await db._find('versions', { _id }) - if (!version) { - await db._run('update', 'articles', { - query: { _id: article._id }, - update: { $pull: { versions: _id } } - }) + const mongo = await db._run('getDbInstance', true) + const articles = mongo.collection('articles') + const versions = mongo.collection('versions') + const articlesCursor = articles.find({}) + try { + while (await articlesCursor.hasNext()) { + const article = await articlesCursor.next() + const versionsIds = article.versions + if (Array.isArray(versionsIds)) { + for await (const _id of versionsIds) { + const version = await versions.findOne({ _id }) + if (!version) { + await articles.updateOne( + { _id: article._id }, + { $pull: { versions: _id } } + ) + } } } } + } finally { + articlesCursor.close() + mongo.close() } } diff --git a/graphql/migrations/20241017080000-article-passwords-cleanup.js b/graphql/migrations/20241017080000-article-passwords-cleanup.js index 7e9ee8f86..e67617ec1 100644 --- a/graphql/migrations/20241017080000-article-passwords-cleanup.js +++ b/graphql/migrations/20241017080000-article-passwords-cleanup.js @@ -3,7 +3,6 @@ exports.up = async function (db) { const mongo = await db._run("getDbInstance") const collections = (await mongo.listCollections().toArray()).map(c => c.name) - // 1. remove article owners (moved into owner+contributors) await mongo.collection('articles').updateMany({}, { $unset: { owners: ''}}) diff --git a/graphql/migrations/20241127170000-article-metadata.js b/graphql/migrations/20241127170000-article-metadata.js index 573ac3600..2350fba1a 100644 --- a/graphql/migrations/20241127170000-article-metadata.js +++ b/graphql/migrations/20241127170000-article-metadata.js @@ -10,7 +10,7 @@ exports.up = async function (db) { let metadata = {} if (article.workingVersion.yaml) { try { - const [legacyMetadata = {}] = YAML.loadAll(article.workingVersion.yaml, 'utf8') + const [legacyMetadata = {}] = YAML.loadAll(article.workingVersion.yaml, { json: true }) metadata = fromLegacyFormat(legacyMetadata) } catch (error) { console.error(`Invalid metadata format on article with id: ${article._id}, metadata will be empty - reason: ${error.reason}`) @@ -32,7 +32,7 @@ exports.up = async function (db) { let metadata = {} if (version.yaml) { try { - const [legacyMetadata = {}] = YAML.loadAll(version.yaml, 'utf8') + const [legacyMetadata = {}] = YAML.loadAll(version.yaml, { json: true }) metadata = fromLegacyFormat(legacyMetadata) } catch (error) { console.error(`Invalid metadata format on version with id: ${version._id}, metadata will be empty - reason: ${error.reason}`) diff --git a/graphql/migrations/20241210110000-article-metadata-legacy-values.js b/graphql/migrations/20241210110000-article-metadata-legacy-values.js new file mode 100644 index 000000000..8cca5a9d0 --- /dev/null +++ b/graphql/migrations/20241210110000-article-metadata-legacy-values.js @@ -0,0 +1,58 @@ +function updateLegacyValues(metadata) { + if (typeof metadata.publicationDate === 'string') { + metadata.publicationDate = metadata.publicationDate.replace(/\//g, '-') + } + if (typeof metadata.keywords === 'string') { + metadata.keywords = metadata.keywords.split(',').map(word => word.trim()) + } +} + +exports.up = async function (db) { + const mongo = await db._run('getDbInstance', true) + const articles = mongo.collection('articles') + const articlesCursor = articles.find({}) + while (await articlesCursor.hasNext()) { + const article = await articlesCursor.next() + const metadata = article.workingVersion.metadata + if (metadata) { + try { + updateLegacyValues(metadata) + } catch (error) { + console.error(`Unable to update legacy values in metadata on article with id: ${article._id} - reason: ${error.reason}`) + } + } + await articles.updateOne({ _id: article._id }, { + $set: { + 'workingVersion.metadata': metadata + } + }, + { upsert: false } + ) + } + await articlesCursor.close() + const versions = mongo.collection('versions') + const versionsCursor = versions.find({}) + while (await versionsCursor.hasNext()) { + const version = await versionsCursor.next() + const metadata = version.metadata + if (metadata) { + try { + updateLegacyValues(metadata) + } catch (error) { + console.error(`Unable to update legacy values in metadata on version with id: ${version._id} - reason: ${error.reason}`) + } + } + await versions.updateOne({ _id: version._id }, { + $set: { + metadata + } + }, + { upsert: false } + ) + } + await versionsCursor.close() +} + +exports.down = function () { + return null +}