From d2c472de6fa7567b420b72f8dec0d035243ad23a Mon Sep 17 00:00:00 2001 From: sopy Date: Fri, 6 Oct 2023 16:34:47 +0300 Subject: [PATCH 1/4] Refactor DatabaseDriver setup for adding other setup files Extracted SQL script execution into a separate function, thus enhancing the code readability and maintainability. The previous implementation had redundancy with comments and empty line removal processes. By creating the _executeFile method, these operations are now separately handled. --- src/util/Database/DatabaseDriver.ts | 53 ++++++++++++++++------------- 1 file changed, 29 insertions(+), 24 deletions(-) diff --git a/src/util/Database/DatabaseDriver.ts b/src/util/Database/DatabaseDriver.ts index 0ebecf9..8f974c0 100644 --- a/src/util/Database/DatabaseDriver.ts +++ b/src/util/Database/DatabaseDriver.ts @@ -1,10 +1,10 @@ import EnvVars from '@src/constants/EnvVars'; import { createPool, Pool } from 'mariadb'; -import fs from 'fs'; import path from 'path'; import logger from 'jet-logger'; import { User } from '@src/types/models/User'; import { GenericModelClass } from '@src/types/models/GenericModelClass'; +import {readFileSync} from 'fs'; // database credentials const { DBCred } = EnvVars; @@ -486,34 +486,13 @@ class Database { } protected async _setup() { - // get setup.sql file - let setupSql = fs.readFileSync( - path.join(__dirname, '..', '..', 'sql', 'setup.sql'), - 'utf8', - ); - - // remove comments - setupSql = setupSql.replace(/--.*/g, ''); - - // remove empty lines - setupSql = setupSql.replace(/^\s*[\r, \n]/gm, ''); - - // split sql queries - const queries = setupSql.split(';'); - - // get connection from pool - const conn = await Database.pool.getConnection(); + const pathToSQLScripts = path.join(__dirname, '..', '..', 'sql'); try { - // execute each query - for (const query of queries) { - if (query) await conn.query(query); - } + await this._executeFile(path.join(pathToSQLScripts, 'create.sql')); } catch (e) { logger.err(e); } finally { - // release connection - await conn.release(); Database.isSetup = true; } @@ -531,6 +510,32 @@ class Database { } else await this.insert('users', user, false); } + protected async _executeFile(path: string) { + let fileContent = readFileSync(path, { + encoding: 'utf8', + }).toString(); + + if (!fileContent) return; + + // remove comments + fileContent = fileContent.replace(/--.*/g, ''); + + // remove empty lines + fileContent = fileContent.replace(/^\s*[\r, \n]/gm, ''); + + // split sql queries + const queries = fileContent.split(';'); + + // get connection from pool + const conn = await Database.pool.getConnection(); + + for(const query of queries) { + if (query) await conn.query(query); + } + + await conn.release(); + } + private _buildWhereQuery = ( like: boolean, ...values: unknown[] From 6d81e0847c022aebde20fe983c1035af36809cc6 Mon Sep 17 00:00:00 2001 From: sopy Date: Fri, 6 Oct 2023 19:04:03 +0300 Subject: [PATCH 2/4] Add migrations tracking to database This commit introduces a new 'migrations' table in the database to keep track of executed database migrations. A 'migrations' section was also added accordingly in the 'databaseDriver.ts'. The migration files in the 'migrations' directory will now be sorted and executed in order. After a successful execution, the migration file information will be added to the 'migrations' table. This change was necessary to maintain the order and consistency of database migrations, and to prevent the re-execution of an already executed migration script. --- src/sql/migrations/0-example.sql | 15 ++++ src/sql/setup.sql | 10 ++- src/util/Database/DatabaseDriver.ts | 125 ++++++++++++++++++++++++++-- 3 files changed, 142 insertions(+), 8 deletions(-) create mode 100644 src/sql/migrations/0-example.sql diff --git a/src/sql/migrations/0-example.sql b/src/sql/migrations/0-example.sql new file mode 100644 index 0000000..1e884d4 --- /dev/null +++ b/src/sql/migrations/0-example.sql @@ -0,0 +1,15 @@ +-- example entry +-- please use {number} - {name} as your format to keep migrations in an consistent order +-- Top comment explaining why this is needed +-- 'migrations' table to track executed DB migrations +create table if not exists migrations +( + -- Each migration gets a unique ID + id BIGINT AUTO_INCREMENT PRIMARY KEY, + + -- Timestamp of when the migration was executed + createdAt timestamp default current_timestamp() not null, + + -- Name of the executed migration script + filename varchar(255) not null +) \ No newline at end of file diff --git a/src/sql/setup.sql b/src/sql/setup.sql index aec57e2..af8186b 100644 --- a/src/sql/setup.sql +++ b/src/sql/setup.sql @@ -169,4 +169,12 @@ select `navigo`.`sessionTable`.`id` AS `id`, `navigo`.`sessionTable`.`token` AS `token`, `navigo`.`sessionTable`.`expires` AS `expires` from `navigo`.`sessionTable` -where `navigo`.`sessionTable`.`expires` >= current_timestamp(); \ No newline at end of file +where `navigo`.`sessionTable`.`expires` >= current_timestamp(); + +create table if not exists migrations +( + id BIGINT AUTO_INCREMENT + PRIMARY KEY, + createdAt timestamp default current_timestamp() not null, + filename varchar(255) not null +) \ No newline at end of file diff --git a/src/util/Database/DatabaseDriver.ts b/src/util/Database/DatabaseDriver.ts index 8f974c0..1dc21af 100644 --- a/src/util/Database/DatabaseDriver.ts +++ b/src/util/Database/DatabaseDriver.ts @@ -1,10 +1,10 @@ import EnvVars from '@src/constants/EnvVars'; -import { createPool, Pool } from 'mariadb'; +import {createPool, Pool} from 'mariadb'; import path from 'path'; import logger from 'jet-logger'; -import { User } from '@src/types/models/User'; -import { GenericModelClass } from '@src/types/models/GenericModelClass'; -import {readFileSync} from 'fs'; +import {User} from '@src/types/models/User'; +import {GenericModelClass} from '@src/types/models/GenericModelClass'; +import {readdirSync, readFileSync} from 'fs'; // database credentials const { DBCred } = EnvVars; @@ -59,6 +59,9 @@ const trustedColumns = [ // session 'token', 'expires', + + // migrations + 'filename', ]; // data interface @@ -90,6 +93,12 @@ interface CountQueryPacket extends RowDataPacket { result: bigint; } +interface IMigrations { + id: bigint; + created_at: Date; + filename: string; +} + export interface ResultSetHeader { fieldCount: number; affectedRows: number; @@ -418,6 +427,59 @@ class Database { } } + // TODO: less duplicate code + protected async _insertUnsafe( + table: string, + data: object | Record, + discardId = true, + ): Promise { + const { keys, values } = processData(data, discardId); + + // create sql query - insert into table (keys) values (values) + // ? for values to be replaced by params + const sql = `INSERT INTO ${table} (${keys.join(',')}) + VALUES (${values.map(() => '?').join(',')})`; + // execute query + const result = (await this._queryUnsafe(sql, values)) as ResultSetHeader; + + // return insert id + let insertId = -1n; + if (result) { + insertId = BigInt(result.insertId); + } + return insertId; + } + + protected async getAllUnsafe(table: string): Promise { + // create sql query - select * from table + const sql = `SELECT * + FROM ${table}`; + + // execute query + const result = await this._queryUnsafe(sql); + + // check if T has any properties that are JSON + // if so parse them + return parseResult(result) as T[] | null; + } + + protected async _queryUnsafe( + sql: string, + params?: unknown[], + ): Promise { + if (!sql) return Promise.reject(new Error('No SQL query')); + + // get connection from pool + const conn = await Database.pool.getConnection(); + try { + // execute query and return result + return await conn.query(sql, params); + } finally { + // release connection + await conn.release(); + } + } + protected async _getWhere( table: string, like: boolean, @@ -465,7 +527,7 @@ class Database { const result = await this._query(sql, queryBuilderResult.params); return BigInt( - (result as CountDataPacket[])[0][`SUM(${column})`] as number || 0, + ((result as CountDataPacket[])[0][`SUM(${column})`] as number) || 0, ); } @@ -488,8 +550,57 @@ class Database { protected async _setup() { const pathToSQLScripts = path.join(__dirname, '..', '..', 'sql'); + const numberRegex = /^\d+/gi; + + // run the setup sql + await this._executeFile(path.join(pathToSQLScripts, 'setup.sql')); + + // read the migrations directory and sort files by number id + const migrationFiles = readdirSync( + path.join(pathToSQLScripts, 'migrations'), + ) + .filter((file) => file.endsWith('.sql')) + .sort((fileA, fileB) => { + const numberA = parseInt(fileA.match(numberRegex)?.[0] || ''); + const numberB = parseInt(fileB.match(numberRegex)?.[0] || ''); + + return numberA - numberB; + }); + + try { + await this._executeFile(path.join(pathToSQLScripts, 'setup.sql')); + } catch (e) { + logger.err(e); + } + + const migrations = await this.getAllUnsafe('migrations'); + + if (!!migrations) { + migrations.forEach((migration) => { + const index = migrationFiles.indexOf(migration.filename); + if (index > -1) migrationFiles.splice(index, 1); + }); + } + try { - await this._executeFile(path.join(pathToSQLScripts, 'create.sql')); + if (migrationFiles.length) logger.info('Starting migrations...'); + for (const migrationFile of migrationFiles) { + logger.info(`Now running: ${migrationFile}...`); + await this._executeFile( + path.join(pathToSQLScripts, 'migrations', migrationFile), + ); + + const success = await this._insertUnsafe('migrations', { + id: -1n, + createdAt: new Date(), + filename: migrationFile, + }); + + if (success < 0n) { + throw new Error('Failed to insert migration file into database'); + } + logger.info(`Successfully ran: ${migrationFile}`); + } } catch (e) { logger.err(e); } finally { @@ -529,7 +640,7 @@ class Database { // get connection from pool const conn = await Database.pool.getConnection(); - for(const query of queries) { + for (const query of queries) { if (query) await conn.query(query); } From 2112595920cce6db47523d0d1b4bc8ca47b9173c Mon Sep 17 00:00:00 2001 From: sopy Date: Wed, 25 Oct 2023 21:33:17 +0300 Subject: [PATCH 3/4] Added option to disable the profanity check --- src/controllers/roadmapController.ts | 38 ++++++++++++++++++++-------- 1 file changed, 27 insertions(+), 11 deletions(-) diff --git a/src/controllers/roadmapController.ts b/src/controllers/roadmapController.ts index e09f2b9..faff9b5 100644 --- a/src/controllers/roadmapController.ts +++ b/src/controllers/roadmapController.ts @@ -16,7 +16,8 @@ import { responseRoadmapDeleted, responseRoadmapNotFound, responseRoadmapNotRated, - responseRoadmapProgressFound, responseRoadmapProgressNotFound, + responseRoadmapProgressFound, + responseRoadmapProgressNotFound, responseRoadmapProgressUpdated, responseRoadmapRated, responseRoadmapUnrated, @@ -25,13 +26,16 @@ import { import { deleteDBRoadmap, deleteRoadmapLike, + getRoadmapLike, getRoadmapObject, - getRoadmapLike, getRoadmapProgress, + getRoadmapProgress, getUser, insertRoadmap, insertRoadmapLike, + insertRoadmapProgress, updateRoadmap, - updateRoadmapLike, updateRoadmapProgress, insertRoadmapProgress, + updateRoadmapLike, + updateRoadmapProgress, } from '@src/helpers/databaseManagement'; import { RequestWithBody } from '@src/middleware/validators/validateBody'; import { Roadmap, RoadmapTopic } from '@src/types/models/Roadmap'; @@ -40,6 +44,8 @@ import { addRoadmapView } from '@src/util/Views'; import logger from 'jet-logger'; import { RoadmapProgress } from '@src/types/models/RoadmapProgress'; +const profanityCheckEnabled = false; + export async function createRoadmap(req: RequestWithBody, res: Response) { // guaranteed to exist by middleware const { name, description, data, miscData, version } = req.body; @@ -57,7 +63,9 @@ export async function createRoadmap(req: RequestWithBody, res: Response) { if (!topic || !Object.values(RoadmapTopic).includes(topic as RoadmapTopic)) topic = RoadmapTopic.PROGRAMMING; - const isPublic = !Boolean(req.body.isProfane); + const isPublic = profanityCheckEnabled + ? !Boolean(req.body.isprofane) + : true; if (isDraft !== true && isDraft !== false) isDraft = false; if (!isPublic) isDraft = true; @@ -109,8 +117,8 @@ export async function getRoadmap(req: RequestWithSession, res: Response) { 1, ); const isLiked = - userId !== undefined && userId !== null ? - await db.sumWhere( + userId !== undefined && userId !== null + ? await db.sumWhere( 'roadmapLikes', 'value', 'roadmapId', @@ -153,7 +161,9 @@ export async function updateAboutRoadmap(req: RequestWithBody, res: Response) { if (!Object.values(RoadmapTopic).includes(topic as RoadmapTopic)) return responseInvalidBody(res); - const isPublic = !Boolean(req.body.isProfane); + const isPublic = profanityCheckEnabled + ? !Boolean(req.body.isprofane) + : roadmap.isPublic; roadmap.set({ name: name as string, @@ -193,7 +203,9 @@ export async function updateAllRoadmap(req: RequestWithBody, res: Response) { if (!Object.values(RoadmapTopic).includes(topic as RoadmapTopic)) return responseInvalidBody(res); - const isPublic = !Boolean(req.body.isProfane); + const isPublic = profanityCheckEnabled + ? !Boolean(req.body.isprofane) + : roadmap.isPublic; if (!isPublic) isDraft = true; roadmap.set({ @@ -227,7 +239,7 @@ export async function updateNameRoadmap(req: RequestWithBody, res: Response) { if (roadmap.userId !== userId) return responseNotAllowed(res); const { name } = req.body; - const isPublic = !Boolean(req.body.isProfane); + const isPublic = profanityCheckEnabled; if (!name) return responseServerError(res); @@ -261,7 +273,9 @@ export async function updateDescriptionRoadmap( if (roadmap.userId !== userId) return responseNotAllowed(res); const { description } = req.body; - const isPublic = !Boolean(req.body.isProfane); + const isPublic = profanityCheckEnabled + ? !Boolean(req.body.isprofane) + : roadmap.isPublic; if (!description) return responseServerError(res); @@ -292,7 +306,9 @@ export async function updateDataRoadmap(req: RequestWithBody, res: Response) { if (roadmap.userId !== userId) return responseNotAllowed(res); const { data } = req.body; - const isPublic = !Boolean(req.body.isProfane); + const isPublic = profanityCheckEnabled + ? !Boolean(req.body.isprofane) + : roadmap.isPublic; if (!data) return responseServerError(res); From 07a24453176dbdc3763128d91d98231fa2a92f0f Mon Sep 17 00:00:00 2001 From: sopy Date: Wed, 25 Oct 2023 21:42:02 +0300 Subject: [PATCH 4/4] Update package version The package version has been updated from 2.0.9 to 2.0.10. --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 39abd1a..c1bacba 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "navigo-learn-api", - "version": "2.0.9", + "version": "2.0.10", "description": "Navigo Learn API", "repository": "https://github.com/NavigoLearn/API.git", "author": "Navigo",