From bd421cec099affe5dccb0eedeefaa0656282b85f Mon Sep 17 00:00:00 2001 From: Mikael Araya Date: Fri, 27 Oct 2023 19:18:19 +0300 Subject: [PATCH 1/7] Add work private data obfusication --- .../src/module/configureWorkerModule.ts | 48 +++++++++++++++---- packages/core/src/core-index.ts | 1 + packages/types/index.d.ts | 5 +- packages/types/modules.ts | 3 +- packages/types/worker.ts | 4 ++ 5 files changed, 50 insertions(+), 11 deletions(-) diff --git a/packages/core-worker/src/module/configureWorkerModule.ts b/packages/core-worker/src/module/configureWorkerModule.ts index 60f0768bb1..2a2d4163f7 100644 --- a/packages/core-worker/src/module/configureWorkerModule.ts +++ b/packages/core-worker/src/module/configureWorkerModule.ts @@ -1,7 +1,7 @@ import os from 'os'; import { Query } from '@unchainedshop/types/common.js'; import { ModuleInput, ModuleMutations } from '@unchainedshop/types/core.js'; -import { Work, WorkData, WorkerModule } from '@unchainedshop/types/worker.js'; +import { Work, WorkData, WorkerModule, WorkerSettingsOptions } from '@unchainedshop/types/worker.js'; import { createLogger } from '@unchainedshop/logger'; import { generateDbFilterById, generateDbMutations, buildSortOptions } from '@unchainedshop/utils'; import { SortDirection } from '@unchainedshop/types/api.js'; @@ -15,6 +15,35 @@ const { UNCHAINED_WORKER_ID = os.hostname() } = process.env; const logger = createLogger('unchained:core-worker'); +const buildObfuscatedFieldsFilter = (additionalSensitiveFields = []) => { + const defaultObfuscatedFields = ['password', 'token', 'plainPassword', 'authorization', 'secret']; + + const sensitiveFields = [...defaultObfuscatedFields, ...additionalSensitiveFields]; + + const obfuscateSensitiveFields = (data) => { + if (Array.isArray(data)) { + return data.map((item) => obfuscateSensitiveFields(item)); + } + + if (typeof data === 'object' && data !== null) { + const temp = { ...data }; + Object.keys(temp).forEach((key) => { + if (sensitiveFields.includes(key)) { + delete temp[key]; + } else { + temp[key] = obfuscateSensitiveFields(temp[key]); + } + }); + + return temp; // Return the modified copy + } + + return data; // Return unchanged data for non-objects + }; + + return obfuscateSensitiveFields; +}; + export const buildQuerySelector = ({ created, scheduled, @@ -93,9 +122,12 @@ const defaultSort: Array<{ key: string; value: SortDirection }> = [ export const configureWorkerModule = async ({ db, -}: ModuleInput>): Promise => { + options, +}: ModuleInput): Promise => { const WorkQueue = await WorkQueueCollection(db); + const removePrivateFields = buildObfuscatedFieldsFilter(options?.blacklistedVariables); + const mutations = generateDbMutations(WorkQueue, WorkQueueSchema) as ModuleMutations; const allocateWork: WorkerModule['allocateWork'] = async ({ types, worker = UNCHAINED_WORKER_ID }) => { @@ -118,7 +150,7 @@ export const configureWorkerModule = async ({ ); WorkerDirector.events.emit(WorkerEventTypes.ALLOCATED, { - work: result.value, + work: removePrivateFields(result.value), }); return result.value; @@ -168,7 +200,7 @@ export const configureWorkerModule = async ({ } logger.debug(`work details:`, { work }); - WorkerDirector.events.emit(WorkerEventTypes.FINISHED, { work }); + WorkerDirector.events.emit(WorkerEventTypes.FINISHED, { work: removePrivateFields(work) }); return work; }; @@ -340,7 +372,7 @@ export const configureWorkerModule = async ({ const work = await WorkQueue.findOne(generateDbFilterById(workId), {}); - WorkerDirector.events.emit(WorkerEventTypes.ADDED, { work }); + WorkerDirector.events.emit(WorkerEventTypes.ADDED, { work: removePrivateFields(work) }); return work; }, @@ -355,7 +387,7 @@ export const configureWorkerModule = async ({ const work = await WorkQueue.findOne(generateDbFilterById(currentWork._id), {}); WorkerDirector.events.emit(WorkerEventTypes.RESCHEDULED, { - work, + work: removePrivateFields(work), oldScheduled: currentWork.scheduled, }); @@ -413,7 +445,7 @@ export const configureWorkerModule = async ({ }); WorkerDirector.events.emit(WorkerEventTypes.ADDED, { - work: result.value, + work: removePrivateFields(result.value), }); } return result.value; @@ -444,7 +476,7 @@ export const configureWorkerModule = async ({ const work = await WorkQueue.findOne(generateDbFilterById(workId), {}); - WorkerDirector.events.emit(WorkerEventTypes.DELETED, { work }); + WorkerDirector.events.emit(WorkerEventTypes.DELETED, { work: removePrivateFields(work) }); return work; }, diff --git a/packages/core/src/core-index.ts b/packages/core/src/core-index.ts index f4837d875e..6fa5497376 100644 --- a/packages/core/src/core-index.ts +++ b/packages/core/src/core-index.ts @@ -110,6 +110,7 @@ export const initCore = async ({ }); const worker = await configureWorkerModule({ db, + options: options.worker, migrationRepository, }); diff --git a/packages/types/index.d.ts b/packages/types/index.d.ts index 325b593f65..c8d2cbbe7c 100644 --- a/packages/types/index.d.ts +++ b/packages/types/index.d.ts @@ -126,6 +126,7 @@ import { IWorkerDirector, WorkerModule, WorkerSchedule, + WorkerSettingsOptions, WorkStatus as WorkerStatusType, } from './worker.js'; import { UnchainedCoreOptions, ModuleInput } from './core.js'; @@ -395,7 +396,7 @@ declare module '@unchainedshop/core-users' { declare module '@unchainedshop/core-warehousing' { function configureWarehousingModule( - params: ModuleInput>, + params: ModuleInput, ): Promise; const WarehousingDirector: IWarehousingDirector; @@ -405,7 +406,7 @@ declare module '@unchainedshop/core-warehousing' { } declare module '@unchainedshop/core-worker' { - function configureWorkerModule(params: ModuleInput>): Promise; + function configureWorkerModule(params: ModuleInput): Promise; const WorkerDirector: IWorkerDirector; const WorkStatus: typeof WorkerStatusType; diff --git a/packages/types/modules.ts b/packages/types/modules.ts index c556b13a0e..e6e79d7aac 100644 --- a/packages/types/modules.ts +++ b/packages/types/modules.ts @@ -16,7 +16,7 @@ import { ProductsModule, ProductsSettingsOptions } from './products.js'; import { QuotationsModule, QuotationsSettingsOptions } from './quotations.js'; import { UsersModule } from './user.js'; import { WarehousingModule } from './warehousing.js'; -import { WorkerModule } from './worker.js'; +import { WorkerModule, WorkerSettingsOptions } from './worker.js'; export interface Modules { accounts: AccountsModule; @@ -51,4 +51,5 @@ export interface ModuleOptions { quotations?: QuotationsSettingsOptions; files?: FilesSettingsOptions; payment?: PaymentSettingsOptions; + worker?: WorkerSettingsOptions; } diff --git a/packages/types/worker.ts b/packages/types/worker.ts index 1a73f3c5bf..424a178ac5 100644 --- a/packages/types/worker.ts +++ b/packages/types/worker.ts @@ -41,6 +41,10 @@ export type Work = { * Module */ +export interface WorkerSettingsOptions { + blacklistedVariables?: string[]; +} + export type WorkData = Pick< Partial, 'input' | 'originalWorkId' | 'priority' | 'retries' | 'timeout' | 'scheduled' | 'worker' From a7480dd49bc18162c3acfd1a3418d5061458a50a Mon Sep 17 00:00:00 2001 From: Mikael Araya Date: Fri, 27 Oct 2023 21:18:06 +0300 Subject: [PATCH 2/7] Remove private data from worker queries --- packages/core-worker/src/module/configureWorkerModule.ts | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/packages/core-worker/src/module/configureWorkerModule.ts b/packages/core-worker/src/module/configureWorkerModule.ts index 2a2d4163f7..c6eb2c1ee4 100644 --- a/packages/core-worker/src/module/configureWorkerModule.ts +++ b/packages/core-worker/src/module/configureWorkerModule.ts @@ -26,7 +26,7 @@ const buildObfuscatedFieldsFilter = (additionalSensitiveFields = []) => { } if (typeof data === 'object' && data !== null) { - const temp = { ...data }; + const temp = data; Object.keys(temp).forEach((key) => { if (sensitiveFields.includes(key)) { delete temp[key]; @@ -284,7 +284,9 @@ export const configureWorkerModule = async ({ }, findWork: async ({ workId, originalWorkId }) => - WorkQueue.findOne(workId ? generateDbFilterById(workId) : { originalWorkId }, {}), + removePrivateFields( + await WorkQueue.findOne(workId ? generateDbFilterById(workId) : { originalWorkId }, {}), + ), findWorkQueue: async ({ limit, skip, sort, ...selectorOptions }) => { const selector = buildQuerySelector(selectorOptions); @@ -293,8 +295,7 @@ export const configureWorkerModule = async ({ limit, sort: buildSortOptions(sort || defaultSort), }); - - return workQueues.toArray(); + return removePrivateFields(await workQueues.toArray()); }, count: async (query) => { From 53624524f3f340cf3754e6ac42f573f94c2118bd Mon Sep 17 00:00:00 2001 From: Mikael Araya Date: Sat, 28 Oct 2023 20:33:34 +0300 Subject: [PATCH 3/7] Handle duplicate blacklistedVariables entry --- changelog.md | 2 ++ packages/core-worker/src/module/configureWorkerModule.ts | 6 ++++-- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/changelog.md b/changelog.md index 161becd144..9b0673d043 100644 --- a/changelog.md +++ b/changelog.md @@ -1,3 +1,5 @@ +# Next +- Add new worker configuration option `blacklistedVariables` that accepts array of variable names that should be removed from a job log data returned. # Unchained Engine v2.6 ## Minor diff --git a/packages/core-worker/src/module/configureWorkerModule.ts b/packages/core-worker/src/module/configureWorkerModule.ts index c6eb2c1ee4..e6e3473e90 100644 --- a/packages/core-worker/src/module/configureWorkerModule.ts +++ b/packages/core-worker/src/module/configureWorkerModule.ts @@ -15,10 +15,12 @@ const { UNCHAINED_WORKER_ID = os.hostname() } = process.env; const logger = createLogger('unchained:core-worker'); -const buildObfuscatedFieldsFilter = (additionalSensitiveFields = []) => { +const buildObfuscatedFieldsFilter = (additionalSensitiveFields: string[] = []) => { const defaultObfuscatedFields = ['password', 'token', 'plainPassword', 'authorization', 'secret']; - const sensitiveFields = [...defaultObfuscatedFields, ...additionalSensitiveFields]; + const sensitiveFields = Array.from( + new Set([...defaultObfuscatedFields, ...additionalSensitiveFields]), + ); const obfuscateSensitiveFields = (data) => { if (Array.isArray(data)) { From bdd5acb6b18fe1ad730cf4add76c1b636f5835dd Mon Sep 17 00:00:00 2001 From: Mikael Araya Date: Sat, 28 Oct 2023 20:52:22 +0300 Subject: [PATCH 4/7] Remove private fields from all worker module function returns --- .../src/module/configureWorkerModule.ts | 31 +++++++++---------- 1 file changed, 15 insertions(+), 16 deletions(-) diff --git a/packages/core-worker/src/module/configureWorkerModule.ts b/packages/core-worker/src/module/configureWorkerModule.ts index e6e3473e90..019d3d8956 100644 --- a/packages/core-worker/src/module/configureWorkerModule.ts +++ b/packages/core-worker/src/module/configureWorkerModule.ts @@ -201,10 +201,10 @@ export const configureWorkerModule = async ({ }); } logger.debug(`work details:`, { work }); + const cleanedWorkData = removePrivateFields(work); + WorkerDirector.events.emit(WorkerEventTypes.FINISHED, { work: cleanedWorkData }); - WorkerDirector.events.emit(WorkerEventTypes.FINISHED, { work: removePrivateFields(work) }); - - return work; + return cleanedWorkData; }; const processNextWork: WorkerModule['processNextWork'] = async (unchainedAPI, workerId) => { @@ -374,10 +374,10 @@ export const configureWorkerModule = async ({ }); const work = await WorkQueue.findOne(generateDbFilterById(workId), {}); + const cleanedWorkData = removePrivateFields(work); + WorkerDirector.events.emit(WorkerEventTypes.ADDED, { work: cleanedWorkData }); - WorkerDirector.events.emit(WorkerEventTypes.ADDED, { work: removePrivateFields(work) }); - - return work; + return cleanedWorkData; }, rescheduleWork: async (currentWork, scheduled) => { @@ -388,13 +388,13 @@ export const configureWorkerModule = async ({ }); const work = await WorkQueue.findOne(generateDbFilterById(currentWork._id), {}); - + const cleanedWorkData = removePrivateFields(work); WorkerDirector.events.emit(WorkerEventTypes.RESCHEDULED, { - work: removePrivateFields(work), + work: cleanedWorkData, oldScheduled: currentWork.scheduled, }); - return work; + return cleanedWorkData; }, allocateWork, @@ -441,17 +441,16 @@ export const configureWorkerModule = async ({ upsert: true, }, ); - + const cleanedWorkData = removePrivateFields(result.value); if (!result.lastErrorObject.updatedExisting) { logger.info(`${type} auto-scheduled @ ${new Date(scheduled).toISOString()}`, { workId, }); - WorkerDirector.events.emit(WorkerEventTypes.ADDED, { - work: removePrivateFields(result.value), + work: cleanedWorkData, }); } - return result.value; + return cleanedWorkData; } catch (e) { /* TODO: If the findOneAndUpdate call failed because of _id conflict with a DELETED work, @@ -478,10 +477,10 @@ export const configureWorkerModule = async ({ await mutations.delete(workId); const work = await WorkQueue.findOne(generateDbFilterById(workId), {}); + const cleanedWorkData = removePrivateFields(work); + WorkerDirector.events.emit(WorkerEventTypes.DELETED, { work: cleanedWorkData }); - WorkerDirector.events.emit(WorkerEventTypes.DELETED, { work: removePrivateFields(work) }); - - return work; + return cleanedWorkData; }, markOldWorkAsFailed: async ({ types, worker = UNCHAINED_WORKER_ID, referenceDate }) => { From 98aa8f1783c20e3046e89632d8fac70fad7ace21 Mon Sep 17 00:00:00 2001 From: Pascal Kaufmann Date: Wed, 17 Jan 2024 10:18:25 +0100 Subject: [PATCH 5/7] Remove apollo engine --- examples/kitchensink/.env.schema | 1 - packages/api/src/createGraphQLServer.ts | 10 ---------- packages/platform/src/startPlatform.ts | 9 --------- packages/types/api.ts | 15 +-------------- 4 files changed, 1 insertion(+), 34 deletions(-) diff --git a/examples/kitchensink/.env.schema b/examples/kitchensink/.env.schema index 3fc7562170..d59c3cf5a1 100644 --- a/examples/kitchensink/.env.schema +++ b/examples/kitchensink/.env.schema @@ -6,7 +6,6 @@ EMAIL_WEBSITE_NAME= EMAIL_FROM= EMAIL_WEBSITE_URL= EMAIL_ERROR_REPORT_RECIPIENT= -APOLLO_ENGINE_KEY= UNCHAINED_CURRENCY= UNCHAINED_LANG= UNCHAINED_COUNTRY= diff --git a/packages/api/src/createGraphQLServer.ts b/packages/api/src/createGraphQLServer.ts index bb8844a93a..eaf5521c9a 100644 --- a/packages/api/src/createGraphQLServer.ts +++ b/packages/api/src/createGraphQLServer.ts @@ -5,8 +5,6 @@ import { buildDefaultTypeDefs } from './schema/index.js'; import resolvers from './resolvers/index.js'; import { actions } from './roles/index.js'; -const { APOLLO_ENGINE_KEY } = process.env; - const logGraphQLServerError = (error: GraphQLFormattedError) => { try { const { @@ -30,7 +28,6 @@ export default async (options) => { const { typeDefs: additionalTypeDefs = [], resolvers: additionalResolvers = [], - engine = {}, events = [], workTypes = [], ...apolloServerOptions @@ -50,13 +47,6 @@ export default async (options) => { logGraphQLServerError(error); return error; }, - apollo: APOLLO_ENGINE_KEY - ? { - key: APOLLO_ENGINE_KEY, - privateVariables: ['email', 'plainPassword', 'oldPlainPassword', 'newPlainPassword'], - ...engine, - } - : undefined, ...apolloServerOptions, }); diff --git a/packages/platform/src/startPlatform.ts b/packages/platform/src/startPlatform.ts index 0fd0333ece..7d0139dbe2 100644 --- a/packages/platform/src/startPlatform.ts +++ b/packages/platform/src/startPlatform.ts @@ -32,9 +32,6 @@ export type PlatformOptions = { rolesOptions?: UnchainedCoreOptions['roleOptions']; workQueueOptions?: SetupWorkqueueOptions & SetupCartsOptions; introspection?: boolean; - playground?: boolean; - tracing?: boolean; - cacheControl?: any; adminUiConfig?: AdminUiConfig; }; @@ -74,9 +71,6 @@ export const startPlatform = async ({ workQueueOptions, context, introspection, - playground, - tracing, - cacheControl, }: PlatformOptions): Promise<{ unchainedAPI: UnchainedCore; apolloGraphQLServer: any; @@ -129,9 +123,6 @@ export const startPlatform = async ({ cache, context, introspection, - playground, - tracing, - cacheControl, adminUiConfig, }); diff --git a/packages/types/api.ts b/packages/types/api.ts index c2e93d2d6b..8d97a0e8f8 100644 --- a/packages/types/api.ts +++ b/packages/types/api.ts @@ -71,18 +71,5 @@ export type UnchainedServerOptions = { roles?: any; events: Array; workTypes: Array; - typeDefs?: Array; - resolvers?: Record; - context?: any; - tracing?: boolean; - schema?: any; - plugins?: any[]; - cache: any; - cacheControl?: any; - introspection: boolean; - playground: boolean; adminUiConfig?: AdminUiConfig; -} & Omit< - ApolloServerOptions, - 'context' | 'uploads' | 'formatError' | 'typeDefs' | 'resolvers' | 'cors' | 'schema' | 'schemaHash' ->; +} & ApolloServerOptions; From 856c97f3ff32a6ae26bee1f441649584d1492db0 Mon Sep 17 00:00:00 2001 From: Pascal Kaufmann Date: Wed, 17 Jan 2024 11:19:54 +0100 Subject: [PATCH 6/7] Move some event types --- docs/source/write-plugins/event.md | 3 +- .../src/resolvers/queries/events/events.ts | 11 ++--- .../resolvers/queries/events/eventsCounts.ts | 9 +++- .../api/src/resolvers/type/event-types.ts | 2 +- .../resolveDefaultCurrencyCodeService.ts | 6 +-- .../core-events/src/db/EventsCollection.ts | 9 +++- packages/core-events/src/events-index.ts | 2 + .../module/configureEventHistoryAdapter.ts | 4 +- .../src/module/configureEventsModule.ts | 31 ++++++++++-- .../src/director/WorkerDirector.ts | 10 ++-- .../src/director/WorkerEventTypes.ts | 13 ++--- .../src/module/configureWorkerModule.ts | 47 ++++++++----------- .../src/schedulers/FailedRescheduler.ts | 8 ++-- .../src/workers/EventListenerWorker.ts | 9 ++-- packages/events/src/EventDirector.ts | 24 ++++++++-- packages/events/src/events-index.ts | 3 +- .../plugins/src/events/node-event-emitter.ts | 3 +- packages/plugins/src/events/redis.ts | 3 +- packages/plugins/src/worker/export-token.ts | 5 +- .../src/worker/update-token-ownership.ts | 3 +- packages/types/api.ts | 1 + packages/types/events.ts | 13 ----- packages/types/worker.ts | 16 ------- 23 files changed, 126 insertions(+), 109 deletions(-) diff --git a/docs/source/write-plugins/event.md b/docs/source/write-plugins/event.md index 1c01c0a1fb..b779cfd252 100644 --- a/docs/source/write-plugins/event.md +++ b/docs/source/write-plugins/event.md @@ -9,8 +9,7 @@ Here is an example of redis-enabled Unchained Events: ```typescript import { createClient } from '@redis/client'; -import { EmitAdapter } from '@unchainedshop/types/events.js'; -import { setEmitAdapter } from '@unchainedshop/events'; +import { EmitAdapter, setEmitAdapter } from '@unchainedshop/events'; const { REDIS_PORT = 6379, REDIS_HOST = '127.0.0.1' } = process.env; diff --git a/packages/api/src/resolvers/queries/events/events.ts b/packages/api/src/resolvers/queries/events/events.ts index 30a4adaba6..c62aa79c24 100644 --- a/packages/api/src/resolvers/queries/events/events.ts +++ b/packages/api/src/resolvers/queries/events/events.ts @@ -1,14 +1,11 @@ import { log } from '@unchainedshop/logger'; -import { Root, Context, SortOption } from '@unchainedshop/types/api.js'; -import { EventQuery } from '@unchainedshop/types/events.js'; +import { Root, Context } from '@unchainedshop/types/api.js'; + +type FindEventsParams = Parameters['0']; export default async function events( root: Root, - params: EventQuery & { - limit?: number; - offset?: number; - sort: Array; - }, + params: FindEventsParams, { modules, userId }: Context, ) { log( diff --git a/packages/api/src/resolvers/queries/events/eventsCounts.ts b/packages/api/src/resolvers/queries/events/eventsCounts.ts index 7294902469..e8a191e469 100644 --- a/packages/api/src/resolvers/queries/events/eventsCounts.ts +++ b/packages/api/src/resolvers/queries/events/eventsCounts.ts @@ -1,8 +1,13 @@ import { log } from '@unchainedshop/logger'; import { Root, Context } from '@unchainedshop/types/api.js'; -import { EventQuery } from '@unchainedshop/types/events.js'; -export default async function eventsCount(root: Root, params: EventQuery, { modules, userId }: Context) { +type CountEventsParams = Parameters['0']; + +export default async function eventsCount( + root: Root, + params: CountEventsParams, + { modules, userId }: Context, +) { log(`query eventsCount queryString: ${params.queryString} types: ${params.types} ${userId}`, { userId, }); diff --git a/packages/api/src/resolvers/type/event-types.ts b/packages/api/src/resolvers/type/event-types.ts index 90c10a305d..cf38f3d3db 100644 --- a/packages/api/src/resolvers/type/event-types.ts +++ b/packages/api/src/resolvers/type/event-types.ts @@ -1,5 +1,5 @@ import { Context } from '@unchainedshop/types/api.js'; -import { Event as EventType } from '@unchainedshop/types/events.js'; +import { Event as EventType } from '@unchainedshop/core-events'; export type HelperType = (work: EventType, params: P, context: Context) => T; diff --git a/packages/core-countries/src/service/resolveDefaultCurrencyCodeService.ts b/packages/core-countries/src/service/resolveDefaultCurrencyCodeService.ts index dbfb2d9e63..450e1c6e77 100644 --- a/packages/core-countries/src/service/resolveDefaultCurrencyCodeService.ts +++ b/packages/core-countries/src/service/resolveDefaultCurrencyCodeService.ts @@ -1,5 +1,5 @@ +import { Context } from '@unchainedshop/types/api.js'; import { Country, ResolveDefaultCurrencyCodeService } from '@unchainedshop/types/countries.js'; -import { Modules } from '@unchainedshop/types/modules.js'; import * as lruCache from 'lru-cache'; const { NODE_ENV } = process.env; @@ -13,7 +13,7 @@ const currencyCodeCache = new lruCache.LRUCache({ const { UNCHAINED_CURRENCY } = process.env; -const getDefaultCurrency = async (modules: Modules, country?: Country) => { +const getDefaultCurrency = async (modules: Context['modules'], country?: Country) => { if (country?.defaultCurrencyId) { return modules.currencies.findCurrency({ currencyId: country.defaultCurrencyId, @@ -24,7 +24,7 @@ const getDefaultCurrency = async (modules: Modules, country?: Country) => { export const resolveDefaultCurrencyCodeService: ResolveDefaultCurrencyCodeService = async ( { isoCode }, - { modules }, + { modules }: Context, ) => { const currencyCode = currencyCodeCache.get(isoCode) as string; if (currencyCode) return currencyCode; diff --git a/packages/core-events/src/db/EventsCollection.ts b/packages/core-events/src/db/EventsCollection.ts index 1572737121..dc2f9f7a15 100644 --- a/packages/core-events/src/db/EventsCollection.ts +++ b/packages/core-events/src/db/EventsCollection.ts @@ -1,9 +1,16 @@ import type { mongodb } from '@unchainedshop/mongodb'; -import { Event } from '@unchainedshop/types/events.js'; import { buildDbIndexes } from '@unchainedshop/mongodb'; +import { TimestampFields } from '@unchainedshop/types/common.js'; const TWO_DAYS_SEC = 172800; +export type Event = { + _id?: string; + type: string; + context?: Record; + payload?: Record; +} & TimestampFields; + export const EventsCollection = async (db: mongodb.Db) => { const Events = db.collection('events'); diff --git a/packages/core-events/src/events-index.ts b/packages/core-events/src/events-index.ts index 5efd3f886e..d095079c8a 100644 --- a/packages/core-events/src/events-index.ts +++ b/packages/core-events/src/events-index.ts @@ -1 +1,3 @@ +export type { Event } from './db/EventsCollection.js'; + export { configureEventsModule } from './module/configureEventsModule.js'; diff --git a/packages/core-events/src/module/configureEventHistoryAdapter.ts b/packages/core-events/src/module/configureEventHistoryAdapter.ts index 1b42fa50ab..7c9c6b910c 100644 --- a/packages/core-events/src/module/configureEventHistoryAdapter.ts +++ b/packages/core-events/src/module/configureEventHistoryAdapter.ts @@ -1,6 +1,6 @@ import { ModuleMutations } from '@unchainedshop/types/core.js'; -import { Event, EmitAdapter } from '@unchainedshop/types/events.js'; -import { getEmitHistoryAdapter, setEmitHistoryAdapter } from '@unchainedshop/events'; +import { getEmitHistoryAdapter, setEmitHistoryAdapter, EmitAdapter } from '@unchainedshop/events'; +import { Event } from '../db/EventsCollection.js'; export const configureEventHistoryAdapter = (mutations: ModuleMutations) => { if (!getEmitHistoryAdapter()) { diff --git a/packages/core-events/src/module/configureEventsModule.ts b/packages/core-events/src/module/configureEventsModule.ts index 4b72d60560..fcedb1686e 100644 --- a/packages/core-events/src/module/configureEventsModule.ts +++ b/packages/core-events/src/module/configureEventsModule.ts @@ -1,14 +1,19 @@ import type { mongodb } from '@unchainedshop/mongodb'; import { generateDbFilterById, generateDbMutations, buildSortOptions } from '@unchainedshop/mongodb'; -import { Event, EventQuery, EventsModule } from '@unchainedshop/types/events.js'; import { getRegisteredEvents } from '@unchainedshop/events'; import { SortDirection, SortOption } from '@unchainedshop/types/api.js'; -import { ModuleInput, ModuleMutations } from '@unchainedshop/types/core.js'; -import { EventsCollection } from '../db/EventsCollection.js'; +import { ModuleCreateMutation, ModuleInput, ModuleMutations } from '@unchainedshop/types/core.js'; +import { EventsCollection, Event } from '../db/EventsCollection.js'; import { EventsSchema } from '../db/EventsSchema.js'; import { configureEventHistoryAdapter } from './configureEventHistoryAdapter.js'; +export type EventQuery = { + types?: Array; + queryString?: string; + created?: Date; +}; + export const buildFindSelector = ({ types, queryString, created }: EventQuery) => { const selector: { type?: any; $text?: any; created?: any } = {}; @@ -18,6 +23,26 @@ export const buildFindSelector = ({ types, queryString, created }: EventQuery) = return selector; }; +export interface EventsModule extends ModuleCreateMutation { + findEvent: ( + params: mongodb.Filter & { eventId: string }, + options?: mongodb.FindOptions, + ) => Promise; + + findEvents: ( + params: EventQuery & { + limit?: number; + offset?: number; + sort?: Array; + }, + options?: mongodb.FindOptions, + ) => Promise>; + + type: (event: Event) => string; + + count: (query: EventQuery) => Promise; +} + export const configureEventsModule = async ({ db, }: ModuleInput>): Promise => { diff --git a/packages/core-worker/src/director/WorkerDirector.ts b/packages/core-worker/src/director/WorkerDirector.ts index ea6cfc8f7a..c9233b42b6 100644 --- a/packages/core-worker/src/director/WorkerDirector.ts +++ b/packages/core-worker/src/director/WorkerDirector.ts @@ -1,4 +1,3 @@ -import { EventEmitter } from 'events'; import { IWorkerAdapter, IWorkerDirector, @@ -6,6 +5,7 @@ import { } from '@unchainedshop/types/worker.js'; import { log, LogLevel } from '@unchainedshop/logger'; import { BaseDirector } from '@unchainedshop/utils'; +import { emit } from '@unchainedshop/events'; import { WorkerEventTypes } from './WorkerEventTypes.js'; export const DIRECTOR_MARKED_FAILED_ERROR = 'DIRECTOR_MARKED_FAILED'; @@ -19,8 +19,6 @@ const baseDirector = BaseDirector>('WorkerDirector', { export const WorkerDirector: IWorkerDirector = { ...baseDirector, - events: new EventEmitter(), - getActivePluginTypes: ({ external } = {}) => { return WorkerDirector.getAdapters() .filter((adapter) => { @@ -71,7 +69,7 @@ export const WorkerDirector: IWorkerDirector = { try { const output = await adapter.doWork(input, unchainedAPI, workId); - WorkerDirector.events.emit(WorkerEventTypes.DONE, { output }); + emit(WorkerEventTypes.DONE, { input, workId, output }); return output; } catch (error) { // DO not use this as flow control. The adapter should catch expected errors and return status: FAILED @@ -80,8 +78,10 @@ export const WorkerDirector: IWorkerDirector = { const errorOutput = { error, success: false }; - WorkerDirector.events.emit(WorkerEventTypes.DONE, { + emit(WorkerEventTypes.DONE, { output: errorOutput, + input, + workId, }); return errorOutput; diff --git a/packages/core-worker/src/director/WorkerEventTypes.ts b/packages/core-worker/src/director/WorkerEventTypes.ts index 20a3e5e222..26d71d25df 100644 --- a/packages/core-worker/src/director/WorkerEventTypes.ts +++ b/packages/core-worker/src/director/WorkerEventTypes.ts @@ -1,11 +1,12 @@ export enum WorkerEventTypes { - ADDED = 'added', - ALLOCATED = 'allocated', - DONE = 'done', - FINISHED = 'finished', - DELETED = 'deleted', - RESCHEDULED = 'rescheduled', + ADDED = 'WORK_ADDED', + ALLOCATED = 'WORK_ALLOCATED', + DONE = 'WORK_DONE', + FINISHED = 'WORK_FINISHED', + DELETED = 'WORK_DELETED', + RESCHEDULED = 'WORK_RESCHEDULED', } + // The difference between `done` and `finished` is, that work is `done` after // it was computed (no DB write, could be external) and `finished` after // the changes are written to the DB diff --git a/packages/core-worker/src/module/configureWorkerModule.ts b/packages/core-worker/src/module/configureWorkerModule.ts index 215ced75c8..915750e391 100644 --- a/packages/core-worker/src/module/configureWorkerModule.ts +++ b/packages/core-worker/src/module/configureWorkerModule.ts @@ -9,6 +9,7 @@ import { mongodb, } from '@unchainedshop/mongodb'; import { SortDirection } from '@unchainedshop/types/api.js'; +import { emit, registerEvents } from '@unchainedshop/events'; import { WorkQueueCollection } from '../db/WorkQueueCollection.js'; import { WorkQueueSchema } from '../db/WorkQueueSchema.js'; import { DIRECTOR_MARKED_FAILED_ERROR, WorkerDirector } from '../director/WorkerDirector.js'; @@ -131,6 +132,8 @@ export const configureWorkerModule = async ({ db, options, }: ModuleInput): Promise => { + registerEvents(Object.values(WorkerEventTypes)); + const WorkQueue = await WorkQueueCollection(db); const removePrivateFields = buildObfuscatedFieldsFilter(options?.blacklistedVariables); @@ -156,9 +159,9 @@ export const configureWorkerModule = async ({ { sort: buildSortOptions(defaultSort), returnDocument: 'after' }, ); - WorkerDirector.events.emit(WorkerEventTypes.ALLOCATED, { - work: removePrivateFields(result), - }); + if (result) { + emit(WorkerEventTypes.ALLOCATED, removePrivateFields(result)); + } return result; }; @@ -206,10 +209,9 @@ export const configureWorkerModule = async ({ }); } logger.debug(`work details:`, { work }); - const cleanedWorkData = removePrivateFields(work); - WorkerDirector.events.emit(WorkerEventTypes.FINISHED, { work: cleanedWorkData }); + emit(WorkerEventTypes.FINISHED, removePrivateFields(work)); - return cleanedWorkData; + return work; }; const processNextWork: WorkerModule['processNextWork'] = async (unchainedAPI, workerId) => { @@ -291,18 +293,15 @@ export const configureWorkerModule = async ({ }, findWork: async ({ workId, originalWorkId }) => - removePrivateFields( - await WorkQueue.findOne(workId ? generateDbFilterById(workId) : { originalWorkId }, {}), - ), + WorkQueue.findOne(workId ? generateDbFilterById(workId) : { originalWorkId }, {}), findWorkQueue: async ({ limit, skip, sort, ...selectorOptions }) => { const selector = buildQuerySelector(selectorOptions); - const workQueues = WorkQueue.find(selector, { + return WorkQueue.find(selector, { skip, limit, sort: buildSortOptions(sort || defaultSort), - }); - return removePrivateFields(await workQueues.toArray()); + }).toArray(); }, count: async (query) => { @@ -379,10 +378,9 @@ export const configureWorkerModule = async ({ }); const work = await WorkQueue.findOne(generateDbFilterById(workId), {}); - const cleanedWorkData = removePrivateFields(work); - WorkerDirector.events.emit(WorkerEventTypes.ADDED, { work: cleanedWorkData }); + emit(WorkerEventTypes.ADDED, removePrivateFields(work)); - return cleanedWorkData; + return work; }, rescheduleWork: async (currentWork, scheduled) => { @@ -393,13 +391,12 @@ export const configureWorkerModule = async ({ }); const work = await WorkQueue.findOne(generateDbFilterById(currentWork._id), {}); - const cleanedWorkData = removePrivateFields(work); - WorkerDirector.events.emit(WorkerEventTypes.RESCHEDULED, { - work: cleanedWorkData, + emit(WorkerEventTypes.RESCHEDULED, { + work: removePrivateFields(work), oldScheduled: currentWork.scheduled, }); - return cleanedWorkData; + return work; }, allocateWork, @@ -447,16 +444,13 @@ export const configureWorkerModule = async ({ upsert: true, }, ); - const cleanedWorkData = removePrivateFields(result.value); if (!result.lastErrorObject.updatedExisting) { logger.info(`${type} auto-scheduled @ ${new Date(scheduled).toISOString()}`, { workId, }); - WorkerDirector.events.emit(WorkerEventTypes.ADDED, { - work: cleanedWorkData, - }); + emit(WorkerEventTypes.ADDED, removePrivateFields(result.value)); } - return cleanedWorkData; + return result.value; } catch (e) { /* TODO: If the findOneAndUpdate call failed because of _id conflict with a DELETED work, @@ -483,10 +477,9 @@ export const configureWorkerModule = async ({ await mutations.delete(workId); const work = await WorkQueue.findOne(generateDbFilterById(workId), {}); - const cleanedWorkData = removePrivateFields(work); - WorkerDirector.events.emit(WorkerEventTypes.DELETED, { work: cleanedWorkData }); + emit(WorkerEventTypes.DELETED, removePrivateFields(work)); - return cleanedWorkData; + return work; }, markOldWorkAsFailed: async ({ types, worker = UNCHAINED_WORKER_ID, referenceDate }) => { diff --git a/packages/core-worker/src/schedulers/FailedRescheduler.ts b/packages/core-worker/src/schedulers/FailedRescheduler.ts index 83b557c720..8938b3f59e 100644 --- a/packages/core-worker/src/schedulers/FailedRescheduler.ts +++ b/packages/core-worker/src/schedulers/FailedRescheduler.ts @@ -1,6 +1,6 @@ import { IScheduler, Work, WorkData } from '@unchainedshop/types/worker.js'; import { log } from '@unchainedshop/logger'; -import { WorkerDirector } from '../director/WorkerDirector.js'; +import { subscribe } from '@unchainedshop/events'; import { WorkerEventTypes } from '../director/WorkerEventTypes.js'; export interface FailedReschedulerParams { @@ -16,7 +16,7 @@ export const FailedRescheduler: IScheduler = { version: '1.0.0', actions: ({ retryInput }, unchainedAPI) => { - const handleFinishedWork = async ({ work }: { work: Work }) => { + const handleFinishedWork = async ({ payload: work }: { payload: Work }) => { if (!work.success && work.retries > 0) { const now = new Date(); const workDelayMs = work.scheduled.getTime() - work.created.getTime(); @@ -56,11 +56,11 @@ export const FailedRescheduler: IScheduler = { return { start() { - WorkerDirector.events.on(WorkerEventTypes.FINISHED, handleFinishedWork); + subscribe(WorkerEventTypes.FINISHED, handleFinishedWork); }, stop() { - WorkerDirector.events.off(WorkerEventTypes.FINISHED, handleFinishedWork); + /* */ }, }; }, diff --git a/packages/core-worker/src/workers/EventListenerWorker.ts b/packages/core-worker/src/workers/EventListenerWorker.ts index 1bd4d0dcda..df1331c6ca 100644 --- a/packages/core-worker/src/workers/EventListenerWorker.ts +++ b/packages/core-worker/src/workers/EventListenerWorker.ts @@ -1,6 +1,6 @@ import { IWorker } from '@unchainedshop/types/worker.js'; +import { subscribe } from '@unchainedshop/events'; import { WorkerEventTypes } from '../director/WorkerEventTypes.js'; -import { WorkerDirector } from '../director/WorkerDirector.js'; import { BaseWorker } from './BaseWorker.js'; function debounce any>(func: T, wait) { @@ -44,8 +44,8 @@ export const EventListenerWorker: IWorker = { ...baseWorkerActions, start() { - WorkerDirector.events.on(WorkerEventTypes.ADDED, processWorkQueue); - WorkerDirector.events.on(WorkerEventTypes.FINISHED, processWorkQueue); + subscribe(WorkerEventTypes.ADDED, processWorkQueue); + subscribe(WorkerEventTypes.FINISHED, processWorkQueue); setTimeout(async () => { await baseWorkerActions.autorescheduleTypes({ @@ -55,8 +55,7 @@ export const EventListenerWorker: IWorker = { }, stop() { - WorkerDirector.events.off(WorkerEventTypes.ADDED, processWorkQueue); - WorkerDirector.events.off(WorkerEventTypes.FINISHED, processWorkQueue); + /* */ }, }; }, diff --git a/packages/events/src/EventDirector.ts b/packages/events/src/EventDirector.ts index 32bd22f389..27119cfb8e 100644 --- a/packages/events/src/EventDirector.ts +++ b/packages/events/src/EventDirector.ts @@ -1,9 +1,25 @@ import { createLogger } from '@unchainedshop/logger'; -import { EmitAdapter } from '@unchainedshop/types/events.js'; const logger = createLogger('unchained:events'); -export type ContextNormalizerFunction = (context: any) => any; +export type RawPayloadType = { + payload: T; + context: { + userAgent?: string; + language?: string; + country?: string; + remoteAddress?: string; + referer?: string; + origin?: string; + userId?: string; + }; +}; +export interface EmitAdapter { + publish(eventName: string, data: RawPayloadType>): void; + subscribe(eventName: string, callback: (payload: RawPayloadType>) => void): void; +} + +export type ContextNormalizerFunction = (context: any) => RawPayloadType['context']; export const defaultNormalizer: ContextNormalizerFunction = (context) => { return { @@ -52,7 +68,7 @@ export const EventDirector = { getEmitHistoryAdapter: (): EmitAdapter => HistoryAdapter, - emit: async (eventName: string, data?: Record): Promise => { + emit: async (eventName: string, data?: Record): Promise => { const extractedContext = ContextNormalizer(null); if (!RegisteredEventsSet.has(eventName)) @@ -73,7 +89,7 @@ export const EventDirector = { logger.verbose(`EventDirector -> Emitted ${eventName} with ${JSON.stringify(data)}`); }, - subscribe: (eventName: string, callback: (payload?: Record) => void): void => { + subscribe: (eventName: string, callback: (payload: RawPayloadType) => void): void => { const currentSubscription = `${eventName}${callback?.toString()}`; // used to avaoid registering the same event handler callback if (!RegisteredEventsSet.has(eventName)) diff --git a/packages/events/src/events-index.ts b/packages/events/src/events-index.ts index 0226f4fd2c..98604bd749 100644 --- a/packages/events/src/events-index.ts +++ b/packages/events/src/events-index.ts @@ -1,4 +1,4 @@ -import { EventDirector } from './EventDirector.js'; +import { EventDirector, EmitAdapter } from './EventDirector.js'; const { emit, @@ -23,4 +23,5 @@ export { setEmitAdapter, setEmitHistoryAdapter, subscribe, + EmitAdapter, }; diff --git a/packages/plugins/src/events/node-event-emitter.ts b/packages/plugins/src/events/node-event-emitter.ts index d4220f702e..96180d8670 100644 --- a/packages/plugins/src/events/node-event-emitter.ts +++ b/packages/plugins/src/events/node-event-emitter.ts @@ -1,6 +1,5 @@ import { EventEmitter } from 'events'; -import { EmitAdapter } from '@unchainedshop/types/events.js'; -import { setEmitAdapter } from '@unchainedshop/events'; +import { setEmitAdapter, EmitAdapter } from '@unchainedshop/events'; const NodeEventEmitter = (): EmitAdapter => { const eventEmitter = new EventEmitter(); diff --git a/packages/plugins/src/events/redis.ts b/packages/plugins/src/events/redis.ts index 2ff0d830a4..1a6e7a9466 100644 --- a/packages/plugins/src/events/redis.ts +++ b/packages/plugins/src/events/redis.ts @@ -1,6 +1,5 @@ import { createClient } from '@redis/client'; -import { EmitAdapter } from '@unchainedshop/types/events.js'; -import { setEmitAdapter } from '@unchainedshop/events'; +import { setEmitAdapter, EmitAdapter } from '@unchainedshop/events'; const { REDIS_PORT = 6379, REDIS_HOST = '127.0.0.1' } = process.env; diff --git a/packages/plugins/src/worker/export-token.ts b/packages/plugins/src/worker/export-token.ts index 8dd29b1bd6..007450081c 100644 --- a/packages/plugins/src/worker/export-token.ts +++ b/packages/plugins/src/worker/export-token.ts @@ -1,6 +1,7 @@ -import { IWorkerAdapter } from '@unchainedshop/types/worker.js'; +import { IWorkerAdapter, Work } from '@unchainedshop/types/worker.js'; import { WorkerDirector, WorkerAdapter, WorkerEventTypes } from '@unchainedshop/core-worker'; import { UnchainedCore } from '@unchainedshop/types/core.js'; +import { subscribe } from '@unchainedshop/events'; export const ExportTokenWorker: IWorkerAdapter = { ...WorkerAdapter, @@ -17,7 +18,7 @@ export const ExportTokenWorker: IWorkerAdapter = { }; export const configureExportToken = (unchainedAPI: UnchainedCore) => { - WorkerDirector.events.on(WorkerEventTypes.FINISHED, async ({ work }) => { + subscribe(WorkerEventTypes.FINISHED, async ({ payload: work }) => { if (work.type === 'EXPORT_TOKEN' && work.success) { await unchainedAPI.modules.warehousing.updateTokenOwnership({ tokenId: work.input.token._id, diff --git a/packages/plugins/src/worker/update-token-ownership.ts b/packages/plugins/src/worker/update-token-ownership.ts index c282441d48..8dcd1a1091 100644 --- a/packages/plugins/src/worker/update-token-ownership.ts +++ b/packages/plugins/src/worker/update-token-ownership.ts @@ -2,6 +2,7 @@ import { IWorkerAdapter, Work } from '@unchainedshop/types/worker.js'; import { WorkerDirector, WorkerAdapter, WorkerEventTypes } from '@unchainedshop/core-worker'; import { UnchainedCore } from '@unchainedshop/types/core.js'; import later from '@breejs/later'; +import { subscribe } from '@unchainedshop/events'; const everyMinute = later.parse.cron('* * * * *'); @@ -63,7 +64,7 @@ export const RefreshTokens: IWorkerAdapter = { }; export const configureUpdateTokenOwnership = (unchainedAPI: UnchainedCore) => { - WorkerDirector.events.on(WorkerEventTypes.FINISHED, async ({ work }: { work: Work }) => { + subscribe(WorkerEventTypes.FINISHED, async ({ payload: work }) => { if (work.type === 'UPDATE_TOKEN_OWNERSHIP' && work.success) { await Promise.all( (work.result?.tokens || []).map(unchainedAPI.modules.warehousing.updateTokenOwnership), diff --git a/packages/types/api.ts b/packages/types/api.ts index 8d97a0e8f8..72afefceaa 100644 --- a/packages/types/api.ts +++ b/packages/types/api.ts @@ -69,6 +69,7 @@ export type UnchainedContextResolver = (params: UnchainedHTTPServerContext) => P export type UnchainedServerOptions = { unchainedAPI: UnchainedCore; roles?: any; + context?: any; events: Array; workTypes: Array; adminUiConfig?: AdminUiConfig; diff --git a/packages/types/events.ts b/packages/types/events.ts index 973e37aaa9..5dfae83faf 100644 --- a/packages/types/events.ts +++ b/packages/types/events.ts @@ -1,20 +1,7 @@ import type { Filter, FindOptions } from 'mongodb'; import { SortOption } from './api.js'; -import { TimestampFields } from './common.js'; import { ModuleCreateMutation } from './core.js'; -export type Event = { - _id?: string; - type: string; - context?: Record; - payload?: Record; -} & TimestampFields; - -export interface EmitAdapter { - publish(eventName: string, data: Pick): void; - subscribe(eventName: string, callback: (payload?: Record) => void): void; -} - export type EventQuery = { types?: Array; queryString?: string; diff --git a/packages/types/worker.ts b/packages/types/worker.ts index 5d9cba25a3..edd19c181d 100644 --- a/packages/types/worker.ts +++ b/packages/types/worker.ts @@ -1,4 +1,3 @@ -import type { EventEmitter } from 'stream'; import { SortOption } from './api.js'; import { IBaseAdapter, IBaseDirector, TimestampFields } from './common.js'; import { UnchainedCore } from './core.js'; @@ -11,15 +10,6 @@ export enum WorkStatus { DELETED = 'DELETED', } -export enum WorkerEventTypes { - ADDED = 'added', - ALLOCATED = 'allocated', - DONE = 'done', - FINISHED = 'finished', - DELETED = 'deleted', - RESCHEDULED = 'rescheduled', -} - export type Work = { _id?: string; priority: number; @@ -141,12 +131,6 @@ export type IWorkerDirector = IBaseDirector> & { workScheduleConfiguration: WorkScheduleConfiguration, ) => void; getAutoSchedules: () => Array<[string, WorkScheduleConfiguration]>; - - events: EventEmitter; - // emit: (eventName: string, payload: any) => void; - // onEmit: (eventName: string, payload: any) => void; - // offEmit: (eventName: string, payload: any) => void; - doWork: (work: Work, unchainedAPI: UnchainedCore) => Promise>; }; From 93cc8974ab8b5a6b433ee960c8c98c1cfb175a0f Mon Sep 17 00:00:00 2001 From: Pascal Kaufmann Date: Wed, 17 Jan 2024 11:57:29 +0100 Subject: [PATCH 7/7] Add docs --- docs/source/config/worker.md | 29 +++++++++++++ packages/api/src/resolvers/type/work-types.ts | 16 +++++++ .../src/module/configureWorkerModule.ts | 32 +------------- .../src/build-obfuscated-fields-filter.ts | 42 +++++++++++++++++++ packages/utils/src/utils-index.ts | 2 + 5 files changed, 90 insertions(+), 31 deletions(-) create mode 100644 docs/source/config/worker.md create mode 100644 packages/utils/src/build-obfuscated-fields-filter.ts diff --git a/docs/source/config/worker.md b/docs/source/config/worker.md new file mode 100644 index 0000000000..72a8f6dc00 --- /dev/null +++ b/docs/source/config/worker.md @@ -0,0 +1,29 @@ +--- +title: 'Worker' +description: Configure the Worker Module +--- + +- blacklistedVariables: `Array` provide a custom list of blacklisted variables, keys which are part of the blacklist will be obfuscated with `*****` in Work Queue API's and when publishing Events. + +Example custom configuration: + +``` +const options = { + modules: { + worker: { + blacklistedVariables: ['secret-key'] + }, + } +}; +``` + +By default those variables are filtered: +- `password`, +- `plainPassword`, +- `newPlainPassword`, +- `oldPlainPassword`, +- `authorization`, +- `secret`, +- `accesskey`, +- `accesstoken`, +- `token` \ No newline at end of file diff --git a/packages/api/src/resolvers/type/work-types.ts b/packages/api/src/resolvers/type/work-types.ts index 98945ee276..0987dae5d3 100644 --- a/packages/api/src/resolvers/type/work-types.ts +++ b/packages/api/src/resolvers/type/work-types.ts @@ -1,11 +1,15 @@ import { Context } from '@unchainedshop/types/api.js'; import { Work as WorkType } from '@unchainedshop/types/worker.js'; +import { buildObfuscatedFieldsFilter } from '@unchainedshop/utils'; type HelperType = (work: WorkType, params: P, context: Context) => T; export interface WorkHelperTypes { status: HelperType; type: HelperType; + result: HelperType; + input: HelperType; + error: HelperType; original: HelperType>; } @@ -22,4 +26,16 @@ export const Work: WorkHelperTypes = { if (!obj.originalWorkId) return null; return modules.worker.findWork({ workId: obj.originalWorkId }); }, + + input: (obj, _, { options }: Context) => { + return buildObfuscatedFieldsFilter(options.worker?.blacklistedVariables)(obj.input); + }, + + result: (obj, _, { options }: Context) => { + return buildObfuscatedFieldsFilter(options.worker?.blacklistedVariables)(obj.result); + }, + + error: (obj, _, { options }: Context) => { + return buildObfuscatedFieldsFilter(options.worker?.blacklistedVariables)(obj.error); + }, }; diff --git a/packages/core-worker/src/module/configureWorkerModule.ts b/packages/core-worker/src/module/configureWorkerModule.ts index 915750e391..373e960ac1 100644 --- a/packages/core-worker/src/module/configureWorkerModule.ts +++ b/packages/core-worker/src/module/configureWorkerModule.ts @@ -10,6 +10,7 @@ import { } from '@unchainedshop/mongodb'; import { SortDirection } from '@unchainedshop/types/api.js'; import { emit, registerEvents } from '@unchainedshop/events'; +import { buildObfuscatedFieldsFilter } from '@unchainedshop/utils'; import { WorkQueueCollection } from '../db/WorkQueueCollection.js'; import { WorkQueueSchema } from '../db/WorkQueueSchema.js'; import { DIRECTOR_MARKED_FAILED_ERROR, WorkerDirector } from '../director/WorkerDirector.js'; @@ -20,37 +21,6 @@ const { UNCHAINED_WORKER_ID = os.hostname() } = process.env; const logger = createLogger('unchained:core-worker'); -const buildObfuscatedFieldsFilter = (additionalSensitiveFields: string[] = []) => { - const defaultObfuscatedFields = ['password', 'token', 'plainPassword', 'authorization', 'secret']; - - const sensitiveFields = Array.from( - new Set([...defaultObfuscatedFields, ...additionalSensitiveFields]), - ); - - const obfuscateSensitiveFields = (data) => { - if (Array.isArray(data)) { - return data.map((item) => obfuscateSensitiveFields(item)); - } - - if (typeof data === 'object' && data !== null) { - const temp = data; - Object.keys(temp).forEach((key) => { - if (sensitiveFields.includes(key)) { - delete temp[key]; - } else { - temp[key] = obfuscateSensitiveFields(temp[key]); - } - }); - - return temp; // Return the modified copy - } - - return data; // Return unchanged data for non-objects - }; - - return obfuscateSensitiveFields; -}; - export const buildQuerySelector = ({ created, scheduled, diff --git a/packages/utils/src/build-obfuscated-fields-filter.ts b/packages/utils/src/build-obfuscated-fields-filter.ts new file mode 100644 index 0000000000..8ebd253a7e --- /dev/null +++ b/packages/utils/src/build-obfuscated-fields-filter.ts @@ -0,0 +1,42 @@ +const defaultObfuscatedFields = [ + 'password', + 'plainPassword', + 'newPlainPassword', + 'oldPlainPassword', + 'authorization', + 'secret', + 'accesskey', + 'accesstoken', + 'token', +]; + +const buildObfuscatedFieldsFilter = (blacklistedVariables: string[] = []) => { + const sensitiveFields = blacklistedVariables || defaultObfuscatedFields; + + const obfuscateSensitiveFields = (data) => { + if (!data) return data; + + if (Array.isArray(data)) { + return data.map((item) => obfuscateSensitiveFields(item)); + } + + if (typeof data === 'object') { + const temp = data; + Object.keys(temp).forEach((key) => { + if (sensitiveFields.includes(key)) { + temp[key] = '******'; + } else { + temp[key] = obfuscateSensitiveFields(temp[key]); + } + }); + + return temp; // Return the modified copy + } + + return data; // Return unchanged data for non-objects + }; + + return obfuscateSensitiveFields; +}; + +export default buildObfuscatedFieldsFilter; diff --git a/packages/utils/src/utils-index.ts b/packages/utils/src/utils-index.ts index 5e43d6e422..2e61b13655 100644 --- a/packages/utils/src/utils-index.ts +++ b/packages/utils/src/utils-index.ts @@ -7,6 +7,8 @@ export { default as slugify } from './slugify.js'; export { default as pipePromises } from './pipe-promises.js'; export { default as generateRandomHash } from './generate-random-hash.js'; export { default as randomValueHex } from './random-value-hex.js'; +export { default as buildObfuscatedFieldsFilter } from './build-obfuscated-fields-filter.js'; + /* * Schemas */