Merge pull request #579 from unchainedshop/obfusicate-private-data

Remove private information from log

changelog.md

@@ -9,6 +9,7 @@
 - Add new events `PRODUCT_UPDATE_VARIATION_TEXT`, `PRODUCT_UPDATE_TEXT`, `FILTER_UPDATE_TEXT`, `ASSORTMENT_UPDATE_TEXT` (triggered for every locale & product when text changes).
 - The `_CREATE` events for products, filters and assortments are now triggered AFTER creating the initial text objects so that you can safely use both events to update texts in external systems.
 - Allow to configure an "environment" for stripe which allows to drop events coming to the the engine that are intended to land on another engine not causing false negatives in webhooks.
+- Add new worker configuration option `blacklistedVariables` that accepts array of variable names that should be removed from a job log data returned.
 
 ## Patch
 - Fix payment credential signing procedures that depend on a userId

docs/source/config/worker.md

@@ -0,0 +1,29 @@
+---
+title: 'Worker'
+description: Configure the Worker Module
+---
+
+- blacklistedVariables: `Array<string>` provide a custom list of blacklisted variables, keys which are part of the blacklist will be obfuscated with `*****` in Work Queue API's and when publishing Events.
+
+Example custom configuration:
+
+```
+const options = {
+  modules: {
+    worker: {
+      blacklistedVariables: ['secret-key']
+    },
+  }
+};
+```
+
+By default those variables are filtered:
+- `password`,
+- `plainPassword`,
+- `newPlainPassword`,
+- `oldPlainPassword`,
+- `authorization`,
+- `secret`,
+- `accesskey`,
+- `accesstoken`,
+- `token`

docs/source/write-plugins/event.md

@@ -9,8 +9,7 @@ Here is an example of redis-enabled Unchained Events:
 
 ```typescript
 import { createClient } from '@redis/client';
-import { EmitAdapter } from '@unchainedshop/types/events.js';
-import { setEmitAdapter } from '@unchainedshop/events';
+import { EmitAdapter, setEmitAdapter } from '@unchainedshop/events';
 
 const { REDIS_PORT = 6379, REDIS_HOST = '127.0.0.1' } = process.env;
 

examples/kitchensink/.env.schema

@@ -6,7 +6,6 @@ EMAIL_WEBSITE_NAME=
 EMAIL_FROM=
 EMAIL_WEBSITE_URL=
 EMAIL_ERROR_REPORT_RECIPIENT=
-APOLLO_ENGINE_KEY=
 UNCHAINED_CURRENCY=
 UNCHAINED_LANG=
 UNCHAINED_COUNTRY=

packages/api/src/createGraphQLServer.ts

@@ -5,8 +5,6 @@ import { buildDefaultTypeDefs } from './schema/index.js';
 import resolvers from './resolvers/index.js';
 import { actions } from './roles/index.js';
 
-const { APOLLO_ENGINE_KEY } = process.env;
-
 const logGraphQLServerError = (error: GraphQLFormattedError) => {
   try {
     const {
@@ -30,7 +28,6 @@ export default async (options) => {
   const {
     typeDefs: additionalTypeDefs = [],
     resolvers: additionalResolvers = [],
-    engine = {},
     events = [],
     workTypes = [],
     ...apolloServerOptions
@@ -50,13 +47,6 @@ export default async (options) => {
       logGraphQLServerError(error);
       return error;
     },
-    apollo: APOLLO_ENGINE_KEY
-      ? {
-          key: APOLLO_ENGINE_KEY,
-          privateVariables: ['email', 'plainPassword', 'oldPlainPassword', 'newPlainPassword'],
-          ...engine,
-        }
-      : undefined,
     ...apolloServerOptions,
   });
 

packages/api/src/resolvers/queries/events/events.ts

@@ -1,14 +1,11 @@
 import { log } from '@unchainedshop/logger';
-import { Root, Context, SortOption } from '@unchainedshop/types/api.js';
-import { EventQuery } from '@unchainedshop/types/events.js';
+import { Root, Context } from '@unchainedshop/types/api.js';
+
+type FindEventsParams = Parameters<Context['modules']['events']['findEvents']>['0'];
 
 export default async function events(
   root: Root,
-  params: EventQuery & {
-    limit?: number;
-    offset?: number;
-    sort: Array<SortOption>;
-  },
+  params: FindEventsParams,
   { modules, userId }: Context,
 ) {
   log(

packages/api/src/resolvers/queries/events/eventsCounts.ts

@@ -1,8 +1,13 @@
 import { log } from '@unchainedshop/logger';
 import { Root, Context } from '@unchainedshop/types/api.js';
-import { EventQuery } from '@unchainedshop/types/events.js';
 
-export default async function eventsCount(root: Root, params: EventQuery, { modules, userId }: Context) {
+type CountEventsParams = Parameters<Context['modules']['events']['count']>['0'];
+
+export default async function eventsCount(
+  root: Root,
+  params: CountEventsParams,
+  { modules, userId }: Context,
+) {
   log(`query eventsCount  queryString: ${params.queryString}  types: ${params.types}  ${userId}`, {
     userId,
   });

packages/api/src/resolvers/type/event-types.ts

@@ -1,5 +1,5 @@
 import { Context } from '@unchainedshop/types/api.js';
-import { Event as EventType } from '@unchainedshop/types/events.js';
+import { Event as EventType } from '@unchainedshop/core-events';
 
 export type HelperType<P, T> = (work: EventType, params: P, context: Context) => T;
 

packages/api/src/resolvers/type/work-types.ts

@@ -1,11 +1,15 @@
 import { Context } from '@unchainedshop/types/api.js';
 import { Work as WorkType } from '@unchainedshop/types/worker.js';
+import { buildObfuscatedFieldsFilter } from '@unchainedshop/utils';
 
 type HelperType<P, T> = (work: WorkType, params: P, context: Context) => T;
 
 export interface WorkHelperTypes {
   status: HelperType<never, string>;
   type: HelperType<never, string>;
+  result: HelperType<never, any>;
+  input: HelperType<never, any>;
+  error: HelperType<never, any>;
   original: HelperType<never, Promise<WorkType>>;
 }
 
@@ -22,4 +26,16 @@ export const Work: WorkHelperTypes = {
     if (!obj.originalWorkId) return null;
     return modules.worker.findWork({ workId: obj.originalWorkId });
   },
+
+  input: (obj, _, { options }: Context) => {
+    return buildObfuscatedFieldsFilter(options.worker?.blacklistedVariables)(obj.input);
+  },
+
+  result: (obj, _, { options }: Context) => {
+    return buildObfuscatedFieldsFilter(options.worker?.blacklistedVariables)(obj.result);
+  },
+
+  error: (obj, _, { options }: Context) => {
+    return buildObfuscatedFieldsFilter(options.worker?.blacklistedVariables)(obj.error);
+  },
 };

packages/core-countries/src/service/resolveDefaultCurrencyCodeService.ts

@@ -1,5 +1,5 @@
+import { Context } from '@unchainedshop/types/api.js';
 import { Country, ResolveDefaultCurrencyCodeService } from '@unchainedshop/types/countries.js';
-import { Modules } from '@unchainedshop/types/modules.js';
 import * as lruCache from 'lru-cache';
 
 const { NODE_ENV } = process.env;
@@ -13,7 +13,7 @@ const currencyCodeCache = new lruCache.LRUCache({
 
 const { UNCHAINED_CURRENCY } = process.env;
 
-const getDefaultCurrency = async (modules: Modules, country?: Country) => {
+const getDefaultCurrency = async (modules: Context['modules'], country?: Country) => {
   if (country?.defaultCurrencyId) {
     return modules.currencies.findCurrency({
       currencyId: country.defaultCurrencyId,
@@ -24,7 +24,7 @@ const getDefaultCurrency = async (modules: Modules, country?: Country) => {
 
 export const resolveDefaultCurrencyCodeService: ResolveDefaultCurrencyCodeService = async (
   { isoCode },
-  { modules },
+  { modules }: Context,
 ) => {
   const currencyCode = currencyCodeCache.get(isoCode) as string;
   if (currencyCode) return currencyCode;

packages/core-events/src/db/EventsCollection.ts

@@ -1,9 +1,16 @@
 import type { mongodb } from '@unchainedshop/mongodb';
-import { Event } from '@unchainedshop/types/events.js';
 import { buildDbIndexes } from '@unchainedshop/mongodb';
+import { TimestampFields } from '@unchainedshop/types/common.js';
 
 const TWO_DAYS_SEC = 172800;
 
+export type Event = {
+  _id?: string;
+  type: string;
+  context?: Record<string, unknown>;
+  payload?: Record<string, unknown>;
+} & TimestampFields;
+
 export const EventsCollection = async (db: mongodb.Db) => {
   const Events = db.collection<Event>('events');
 

packages/core-events/src/events-index.ts

@@ -1 +1,3 @@
+export type { Event } from './db/EventsCollection.js';
+
 export { configureEventsModule } from './module/configureEventsModule.js';

packages/core-events/src/module/configureEventHistoryAdapter.ts

@@ -1,6 +1,6 @@
 import { ModuleMutations } from '@unchainedshop/types/core.js';
-import { Event, EmitAdapter } from '@unchainedshop/types/events.js';
-import { getEmitHistoryAdapter, setEmitHistoryAdapter } from '@unchainedshop/events';
+import { getEmitHistoryAdapter, setEmitHistoryAdapter, EmitAdapter } from '@unchainedshop/events';
+import { Event } from '../db/EventsCollection.js';
 
 export const configureEventHistoryAdapter = (mutations: ModuleMutations<Event>) => {
   if (!getEmitHistoryAdapter()) {

packages/core-events/src/module/configureEventsModule.ts

@@ -1,14 +1,19 @@
 import type { mongodb } from '@unchainedshop/mongodb';
 
 import { generateDbFilterById, generateDbMutations, buildSortOptions } from '@unchainedshop/mongodb';
-import { Event, EventQuery, EventsModule } from '@unchainedshop/types/events.js';
 import { getRegisteredEvents } from '@unchainedshop/events';
 import { SortDirection, SortOption } from '@unchainedshop/types/api.js';
-import { ModuleInput, ModuleMutations } from '@unchainedshop/types/core.js';
-import { EventsCollection } from '../db/EventsCollection.js';
+import { ModuleCreateMutation, ModuleInput, ModuleMutations } from '@unchainedshop/types/core.js';
+import { EventsCollection, Event } from '../db/EventsCollection.js';
 import { EventsSchema } from '../db/EventsSchema.js';
 import { configureEventHistoryAdapter } from './configureEventHistoryAdapter.js';
 
+export type EventQuery = {
+  types?: Array<string>;
+  queryString?: string;
+  created?: Date;
+};
+
 export const buildFindSelector = ({ types, queryString, created }: EventQuery) => {
   const selector: { type?: any; $text?: any; created?: any } = {};
 
@@ -18,6 +23,26 @@ export const buildFindSelector = ({ types, queryString, created }: EventQuery) =
   return selector;
 };
 
+export interface EventsModule extends ModuleCreateMutation<Event> {
+  findEvent: (
+    params: mongodb.Filter<Event> & { eventId: string },
+    options?: mongodb.FindOptions,
+  ) => Promise<Event>;
+
+  findEvents: (
+    params: EventQuery & {
+      limit?: number;
+      offset?: number;
+      sort?: Array<SortOption>;
+    },
+    options?: mongodb.FindOptions,
+  ) => Promise<Array<Event>>;
+
+  type: (event: Event) => string;
+
+  count: (query: EventQuery) => Promise<number>;
+}
+
 export const configureEventsModule = async ({
   db,
 }: ModuleInput<Record<string, never>>): Promise<EventsModule> => {

packages/core-worker/src/director/WorkerDirector.ts

@@ -1,11 +1,11 @@
-import { EventEmitter } from 'events';
 import {
   IWorkerAdapter,
   IWorkerDirector,
   WorkScheduleConfiguration,
 } from '@unchainedshop/types/worker.js';
 import { log, LogLevel } from '@unchainedshop/logger';
 import { BaseDirector } from '@unchainedshop/utils';
+import { emit } from '@unchainedshop/events';
 import { WorkerEventTypes } from './WorkerEventTypes.js';
 
 export const DIRECTOR_MARKED_FAILED_ERROR = 'DIRECTOR_MARKED_FAILED';
@@ -19,8 +19,6 @@ const baseDirector = BaseDirector<IWorkerAdapter<any, any>>('WorkerDirector', {
 export const WorkerDirector: IWorkerDirector = {
   ...baseDirector,
 
-  events: new EventEmitter(),
-
   getActivePluginTypes: ({ external } = {}) => {
     return WorkerDirector.getAdapters()
       .filter((adapter) => {
@@ -71,7 +69,7 @@ export const WorkerDirector: IWorkerDirector = {
 
     try {
       const output = await adapter.doWork(input, unchainedAPI, workId);
-      WorkerDirector.events.emit(WorkerEventTypes.DONE, { output });
+      emit(WorkerEventTypes.DONE, { input, workId, output });
       return output;
     } catch (error) {
       // DO not use this as flow control. The adapter should catch expected errors and return status: FAILED
@@ -80,8 +78,10 @@ export const WorkerDirector: IWorkerDirector = {
 
       const errorOutput = { error, success: false };
 
-      WorkerDirector.events.emit(WorkerEventTypes.DONE, {
+      emit(WorkerEventTypes.DONE, {
         output: errorOutput,
+        input,
+        workId,
       });
 
       return errorOutput;

packages/core-worker/src/director/WorkerEventTypes.ts

@@ -1,11 +1,12 @@
 export enum WorkerEventTypes {
-  ADDED = 'added',
-  ALLOCATED = 'allocated',
-  DONE = 'done',
-  FINISHED = 'finished',
-  DELETED = 'deleted',
-  RESCHEDULED = 'rescheduled',
+  ADDED = 'WORK_ADDED',
+  ALLOCATED = 'WORK_ALLOCATED',
+  DONE = 'WORK_DONE',
+  FINISHED = 'WORK_FINISHED',
+  DELETED = 'WORK_DELETED',
+  RESCHEDULED = 'WORK_RESCHEDULED',
 }
+
 // The difference between `done` and `finished` is, that work is `done` after
 // it was computed (no DB write, could be external) and `finished` after
 // the changes are written to the DB