From 319a0144face7c649d003a9d326029b92a856001 Mon Sep 17 00:00:00 2001 From: Arne Gnisa Date: Thu, 9 Nov 2023 12:34:28 +0100 Subject: [PATCH 1/4] N21-1029 renames some collections: user_login_migrations -> user-login-migrations external_tools -> external-tools context_external_tools -> context-external-tools school_external_tools -> school-external-tools --- .../entity/context-external-tool.entity.ts | 2 +- .../entity/external-tool.entity.ts | 2 +- .../entity/school-external-tool.entity.ts | 2 +- .../entity/user-login-migration.entity.ts | 2 +- ...tools.json => context-external-tools.json} | 0 ...xternal_tools.json => external-tools.json} | 0 backup/setup/migrations.json | 11 +++++ ..._tools.json => school-external-tools.json} | 0 backup/setup/user-login-migrations.json | 1 + ...tool-and-user-login-migration-renamings.js | 44 +++++++++++++++++++ 10 files changed, 60 insertions(+), 4 deletions(-) rename backup/setup/{context_external_tools.json => context-external-tools.json} (100%) rename backup/setup/{external_tools.json => external-tools.json} (100%) rename backup/setup/{school_external_tools.json => school-external-tools.json} (100%) create mode 100644 backup/setup/user-login-migrations.json create mode 100644 migrations/1699529266062-tool-and-user-login-migration-renamings.js diff --git a/apps/server/src/modules/tool/context-external-tool/entity/context-external-tool.entity.ts b/apps/server/src/modules/tool/context-external-tool/entity/context-external-tool.entity.ts index 6fdbad2b3af..aebab1a8d5d 100644 --- a/apps/server/src/modules/tool/context-external-tool/entity/context-external-tool.entity.ts +++ b/apps/server/src/modules/tool/context-external-tool/entity/context-external-tool.entity.ts @@ -18,7 +18,7 @@ export interface IContextExternalToolProperties { toolVersion: number; } -@Entity({ tableName: 'context_external_tools' }) +@Entity({ tableName: 'context-external-tools' }) export class ContextExternalToolEntity extends BaseEntityWithTimestamps { @ManyToOne() schoolTool: SchoolExternalToolEntity; diff --git a/apps/server/src/modules/tool/external-tool/entity/external-tool.entity.ts b/apps/server/src/modules/tool/external-tool/entity/external-tool.entity.ts index 3bd3ed9c30d..481ed3b7c2d 100644 --- a/apps/server/src/modules/tool/external-tool/entity/external-tool.entity.ts +++ b/apps/server/src/modules/tool/external-tool/entity/external-tool.entity.ts @@ -6,7 +6,7 @@ import { BasicToolConfigEntity, Lti11ToolConfigEntity, Oauth2ToolConfigEntity } export type IExternalToolProperties = Readonly>; -@Entity({ tableName: 'external_tools' }) +@Entity({ tableName: 'external-tools' }) export class ExternalToolEntity extends BaseEntityWithTimestamps { @Unique() @Property() diff --git a/apps/server/src/modules/tool/school-external-tool/entity/school-external-tool.entity.ts b/apps/server/src/modules/tool/school-external-tool/entity/school-external-tool.entity.ts index a1682e3b7cd..fc7f6703d05 100644 --- a/apps/server/src/modules/tool/school-external-tool/entity/school-external-tool.entity.ts +++ b/apps/server/src/modules/tool/school-external-tool/entity/school-external-tool.entity.ts @@ -11,7 +11,7 @@ export interface ISchoolExternalToolProperties { toolVersion: number; } -@Entity({ tableName: 'school_external_tools' }) +@Entity({ tableName: 'school-external-tools' }) export class SchoolExternalToolEntity extends BaseEntityWithTimestamps { @ManyToOne() tool: ExternalToolEntity; diff --git a/apps/server/src/shared/domain/entity/user-login-migration.entity.ts b/apps/server/src/shared/domain/entity/user-login-migration.entity.ts index 2daf9707f3c..b79b3d862c3 100644 --- a/apps/server/src/shared/domain/entity/user-login-migration.entity.ts +++ b/apps/server/src/shared/domain/entity/user-login-migration.entity.ts @@ -5,7 +5,7 @@ import { BaseEntityWithTimestamps } from './base.entity'; export type IUserLoginMigration = Readonly>; -@Entity({ tableName: 'user_login_migrations' }) +@Entity({ tableName: 'user-login-migrations' }) export class UserLoginMigrationEntity extends BaseEntityWithTimestamps { @OneToOne(() => SchoolEntity, undefined, { nullable: false }) school: SchoolEntity; diff --git a/backup/setup/context_external_tools.json b/backup/setup/context-external-tools.json similarity index 100% rename from backup/setup/context_external_tools.json rename to backup/setup/context-external-tools.json diff --git a/backup/setup/external_tools.json b/backup/setup/external-tools.json similarity index 100% rename from backup/setup/external_tools.json rename to backup/setup/external-tools.json diff --git a/backup/setup/migrations.json b/backup/setup/migrations.json index 90fcee0baa3..e00ca430152 100644 --- a/backup/setup/migrations.json +++ b/backup/setup/migrations.json @@ -339,5 +339,16 @@ "$date": "2023-10-26T13:06:27.322Z" }, "__v": 0 + }, + { + "_id": { + "$oid": "654cc2326b83f786c4227b21" + }, + "state": "up", + "name": "tool-and-user-login-migration-renamings", + "createdAt": { + "$date": "2023-11-09T11:27:46.062Z" + }, + "__v": 0 } ] diff --git a/backup/setup/school_external_tools.json b/backup/setup/school-external-tools.json similarity index 100% rename from backup/setup/school_external_tools.json rename to backup/setup/school-external-tools.json diff --git a/backup/setup/user-login-migrations.json b/backup/setup/user-login-migrations.json new file mode 100644 index 00000000000..fe51488c706 --- /dev/null +++ b/backup/setup/user-login-migrations.json @@ -0,0 +1 @@ +[] diff --git a/migrations/1699529266062-tool-and-user-login-migration-renamings.js b/migrations/1699529266062-tool-and-user-login-migration-renamings.js new file mode 100644 index 00000000000..8b2480a41e4 --- /dev/null +++ b/migrations/1699529266062-tool-and-user-login-migration-renamings.js @@ -0,0 +1,44 @@ +const mongoose = require('mongoose'); +const { info, error } = require('../src/logger'); + +const { connect, close } = require('../src/utils/database'); + +async function renameCollection(oldName, newName) { + try { + await mongoose.connection.collection(oldName).rename(newName); + info(`Renamed collection ${oldName} to ${newName}`); + } catch (err) { + error(`Error renaming collection ${oldName} to ${newName}: ${err.message}`); + throw err; + } +} + +module.exports = { + up: async function up() { + await connect(); + + await renameCollection('user_login_migrations', 'user-login-migrations'); + + await renameCollection('external_tools', 'external-tools'); + + await renameCollection('context_external_tools', 'context-external-tools'); + + await renameCollection('school_external_tools', 'school-external-tools'); + + await close(); + }, + + down: async function down() { + await connect(); + + await renameCollection('user-login-migrations', 'user_login_migrations'); + + await renameCollection('external-tools', 'external_tools'); + + await renameCollection('context-external-tools', 'context_external_tools'); + + await renameCollection('school-external-tools', 'school_external_tools'); + + await close(); + }, +}; From 0520c2656cb38927db696de0b4d026e284d0e9c0 Mon Sep 17 00:00:00 2001 From: Arne Gnisa Date: Fri, 10 Nov 2023 11:08:48 +0100 Subject: [PATCH 2/4] N21-1456 review fixes --- .../templates/configmap_file_init.yml.j2 | 12 ++++++------ .../type/allowed-authorization-object-type.enum.ts | 4 ++-- src/services/school/model.js | 2 +- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/ansible/roles/schulcloud-server-init/templates/configmap_file_init.yml.j2 b/ansible/roles/schulcloud-server-init/templates/configmap_file_init.yml.j2 index a3e5459077f..654d4152b95 100644 --- a/ansible/roles/schulcloud-server-init/templates/configmap_file_init.yml.j2 +++ b/ansible/roles/schulcloud-server-init/templates/configmap_file_init.yml.j2 @@ -390,8 +390,8 @@ data: }' # Add Bettermarks' tools configuration as an external tool - # (stored in the 'external_tools' collection) that uses OAuth. - mongosh $DATABASE__URL --eval 'db.external_tools.replaceOne( + # (stored in the 'external-tools' collection) that uses OAuth. + mongosh $DATABASE__URL --eval 'db.external-tools.replaceOne( { "name": "bettermarks", "config_type": "oauth2" @@ -486,9 +486,9 @@ data: echo "POSTed nextcloud to hydra." # Add Nextcloud' tools configuration as an external tool - # (stored in the 'external_tools' collection) that uses OAuth. - echo "Inserting nextcloud to external_tools..." - mongosh $DATABASE__URL --eval 'db.external_tools.update( + # (stored in the 'external-tools' collection) that uses OAuth. + echo "Inserting nextcloud to external-tools..." + mongosh $DATABASE__URL --eval 'db.external-tools.update( { "name": "nextcloud", "config_type": "oauth2" @@ -512,7 +512,7 @@ data: "upsert": true } );' - echo "Inserted nextcloud to external_tools." + echo "Inserted nextcloud to external-tools." echo "Nextcloud config data init performed successfully." fi diff --git a/apps/server/src/modules/authorization/domain/type/allowed-authorization-object-type.enum.ts b/apps/server/src/modules/authorization/domain/type/allowed-authorization-object-type.enum.ts index f36ca235af1..01f24b21985 100644 --- a/apps/server/src/modules/authorization/domain/type/allowed-authorization-object-type.enum.ts +++ b/apps/server/src/modules/authorization/domain/type/allowed-authorization-object-type.enum.ts @@ -7,7 +7,7 @@ export enum AuthorizableReferenceType { 'Lesson' = 'lessons', 'Team' = 'teams', 'Submission' = 'submissions', - 'SchoolExternalToolEntity' = 'school_external_tools', + 'SchoolExternalToolEntity' = 'school-external-tools', 'BoardNode' = 'boardnodes', - 'ContextExternalToolEntity' = 'context_external_tools', + 'ContextExternalToolEntity' = 'context-external-tools', } diff --git a/src/services/school/model.js b/src/services/school/model.js index 0ec931e4191..787f7b55348 100644 --- a/src/services/school/model.js +++ b/src/services/school/model.js @@ -177,7 +177,7 @@ const gradeLevelSchema = new Schema({ }); const schoolModel = mongoose.model('school', schoolSchema); -const userLoginMigrationModel = mongoose.model('userLoginMigration', userLoginMigrationSchema, 'user_login_migrations'); +const userLoginMigrationModel = mongoose.model('userLoginMigration', userLoginMigrationSchema, 'user-login-migrations'); const schoolGroupModel = mongoose.model('schoolGroup', schoolGroupSchema); const yearModel = mongoose.model('year', yearSchema); const gradeLevelModel = mongoose.model('gradeLevel', gradeLevelSchema); From 24f2ba058c767133b5be5b75ac6594602590947e Mon Sep 17 00:00:00 2001 From: Arne Gnisa Date: Fri, 10 Nov 2023 14:35:52 +0100 Subject: [PATCH 3/4] N21-1029 checks if new table already exists, because of starting of migrations and server at the same time --- ...tool-and-user-login-migration-renamings.js | 41 +++++++++++-------- 1 file changed, 23 insertions(+), 18 deletions(-) diff --git a/migrations/1699529266062-tool-and-user-login-migration-renamings.js b/migrations/1699529266062-tool-and-user-login-migration-renamings.js index 8b2480a41e4..8f1e342f251 100644 --- a/migrations/1699529266062-tool-and-user-login-migration-renamings.js +++ b/migrations/1699529266062-tool-and-user-login-migration-renamings.js @@ -1,14 +1,25 @@ const mongoose = require('mongoose'); const { info, error } = require('../src/logger'); - const { connect, close } = require('../src/utils/database'); -async function renameCollection(oldName, newName) { +async function renameAndDeleteCollection(oldName, newName) { try { - await mongoose.connection.collection(oldName).rename(newName); + const { connection } = mongoose; + + // Check whether the new collection already exists + const newCollectionExists = await connection.db.listCollections({ name: newName }).hasNext(); + + if (newCollectionExists) { + // If the new collection already exists, delete it + await connection.collection(newName).drop(); + info(`Dropped existing collection ${newName}`); + } + + // Rename the old collection to the new collection + await connection.collection(oldName).rename(newName); info(`Renamed collection ${oldName} to ${newName}`); } catch (err) { - error(`Error renaming collection ${oldName} to ${newName}: ${err.message}`); + error(`Error renaming and deleting collection ${oldName} to ${newName}: ${err.message}`); throw err; } } @@ -17,13 +28,10 @@ module.exports = { up: async function up() { await connect(); - await renameCollection('user_login_migrations', 'user-login-migrations'); - - await renameCollection('external_tools', 'external-tools'); - - await renameCollection('context_external_tools', 'context-external-tools'); - - await renameCollection('school_external_tools', 'school-external-tools'); + await renameAndDeleteCollection('user_login_migrations', 'user-login-migrations'); + await renameAndDeleteCollection('external_tools', 'external-tools'); + await renameAndDeleteCollection('context_external_tools', 'context-external-tools'); + await renameAndDeleteCollection('school_external_tools', 'school-external-tools'); await close(); }, @@ -31,13 +39,10 @@ module.exports = { down: async function down() { await connect(); - await renameCollection('user-login-migrations', 'user_login_migrations'); - - await renameCollection('external-tools', 'external_tools'); - - await renameCollection('context-external-tools', 'context_external_tools'); - - await renameCollection('school-external-tools', 'school_external_tools'); + await renameAndDeleteCollection('user-login-migrations', 'user_login_migrations'); + await renameAndDeleteCollection('external-tools', 'external_tools'); + await renameAndDeleteCollection('context-external-tools', 'context_external_tools'); + await renameAndDeleteCollection('school-external-tools', 'school_external_tools'); await close(); }, From e876d79c66c56ea8e991b1e9ad8d498243023ba3 Mon Sep 17 00:00:00 2001 From: Arne Gnisa Date: Fri, 10 Nov 2023 15:09:36 +0100 Subject: [PATCH 4/4] N21-1029 using aggregate for copying data --- ...tool-and-user-login-migration-renamings.js | 38 +++++++++---------- 1 file changed, 18 insertions(+), 20 deletions(-) diff --git a/migrations/1699529266062-tool-and-user-login-migration-renamings.js b/migrations/1699529266062-tool-and-user-login-migration-renamings.js index 8f1e342f251..e1b0695b444 100644 --- a/migrations/1699529266062-tool-and-user-login-migration-renamings.js +++ b/migrations/1699529266062-tool-and-user-login-migration-renamings.js @@ -2,24 +2,22 @@ const mongoose = require('mongoose'); const { info, error } = require('../src/logger'); const { connect, close } = require('../src/utils/database'); -async function renameAndDeleteCollection(oldName, newName) { +async function aggregateAndDropCollection(oldName, newName) { try { const { connection } = mongoose; - // Check whether the new collection already exists - const newCollectionExists = await connection.db.listCollections({ name: newName }).hasNext(); + // Aggregation pipeline for copying the documents + const pipeline = [{ $match: {} }, { $out: newName }]; - if (newCollectionExists) { - // If the new collection already exists, delete it - await connection.collection(newName).drop(); - info(`Dropped existing collection ${newName}`); - } + // Copy documents from the old collection to the new collection + await connection.collection(oldName).aggregate(pipeline).toArray(); + info(`Aggregated and copied documents from ${oldName} to ${newName}`); - // Rename the old collection to the new collection - await connection.collection(oldName).rename(newName); - info(`Renamed collection ${oldName} to ${newName}`); + // Delete old collection + await connection.collection(oldName).drop(); + info(`Dropped collection ${oldName}`); } catch (err) { - error(`Error renaming and deleting collection ${oldName} to ${newName}: ${err.message}`); + error(`Error aggregating, copying, and deleting collection ${oldName} to ${newName}: ${err.message}`); throw err; } } @@ -28,10 +26,10 @@ module.exports = { up: async function up() { await connect(); - await renameAndDeleteCollection('user_login_migrations', 'user-login-migrations'); - await renameAndDeleteCollection('external_tools', 'external-tools'); - await renameAndDeleteCollection('context_external_tools', 'context-external-tools'); - await renameAndDeleteCollection('school_external_tools', 'school-external-tools'); + await aggregateAndDropCollection('user_login_migrations', 'user-login-migrations'); + await aggregateAndDropCollection('external_tools', 'external-tools'); + await aggregateAndDropCollection('context_external_tools', 'context-external-tools'); + await aggregateAndDropCollection('school_external_tools', 'school-external-tools'); await close(); }, @@ -39,10 +37,10 @@ module.exports = { down: async function down() { await connect(); - await renameAndDeleteCollection('user-login-migrations', 'user_login_migrations'); - await renameAndDeleteCollection('external-tools', 'external_tools'); - await renameAndDeleteCollection('context-external-tools', 'context_external_tools'); - await renameAndDeleteCollection('school-external-tools', 'school_external_tools'); + await aggregateAndDropCollection('user-login-migrations', 'user_login_migrations'); + await aggregateAndDropCollection('external-tools', 'external_tools'); + await aggregateAndDropCollection('context-external-tools', 'context_external_tools'); + await aggregateAndDropCollection('school-external-tools', 'school_external_tools'); await close(); },