Merge pull request #143 from seek-oss/trim-all-serializers

Add truncation to all serializers

.changeset/smooth-ties-look.md

@@ -0,0 +1,9 @@
+---
+'@seek/logger': major
+---
+
+Apply trimming to serializers
+
+Previously, [built-in serializers](https://github.com/seek-oss/logger/tree/54f16e17a9bb94261b9d2e4b77f04f55d5a3ab4c?tab=readme-ov-file#standardised-fields) and custom ones supplied via the [`serializers` option](https://github.com/pinojs/pino/blob/8aafa88139890b97aca0d32601cb5ffdd9bda1eb/docs/api.md#serializers-object) were not subject to [trimming](https://github.com/seek-oss/logger/tree/54f16e17a9bb94261b9d2e4b77f04f55d5a3ab4c?tab=readme-ov-file#trimming). This caused some emitted error logs to be extremely large.
+
+Now, trimming is applied across all serializers by default. If you rely on deeply nested `err` properties to troubleshoot your application, tune the `maxObjectDepth` configured on your logger.

src/formatters/index.ts

@@ -1,6 +1,8 @@
 import { trimmer } from 'dtrim';
 import type { LoggerOptions } from 'pino';
 
+export const DEFAULT_MAX_OBJECT_DEPTH = 4;
+
 export interface FormatterOptions {
   /**
    * Maximum property depth of objects being logged. Default: 4
@@ -17,7 +19,7 @@ export const createFormatters = (
   opts: FormatterOptions & Required<Pick<LoggerOptions, 'serializers'>>,
 ): LoggerOptions['formatters'] => {
   const trim = trimmer({
-    depth: opts.maxObjectDepth ?? 4,
+    depth: opts.maxObjectDepth ?? DEFAULT_MAX_OBJECT_DEPTH,
     retain: new Set(Object.keys(opts.serializers)),
   });
 

src/index.test.ts

@@ -441,10 +441,7 @@ testLog(
       request: {
         _options: {
           method: 'get',
-          headers: {
-            Accept: 'application/json, text/plain, */*',
-            authorization: '[Redacted]',
-          },
+          headers: '[Object]',
         },
         domain: null,
       },
@@ -632,48 +629,36 @@ test('should log customized timestamp if timestamp logger option is supplied', a
   expect(log.timestamp).toBe(mockTimestamp);
 });
 
-class Req {
-  get socket() {
-    return {
-      remoteAddress: 'localhost',
-      remotePort: '4000',
-    };
-  }
-}
 testLog(
-  'should not truncate objects with a non error serializer',
+  'should trim default serializers',
   {
-    req: new Req(),
-    notSerialized: {
+    errWithCause: {
       a: {
         b: {},
       },
+      anyField: 'a'.repeat(555),
+      stack: 'a'.repeat(555),
     },
-  },
-  {
-    req: {
-      remoteAddress: 'localhost',
-      remotePort: '4000',
-    },
-    notSerialized: {
-      a: '[Object]',
-    },
-  },
-  'info',
-  {
-    maxObjectDepth: 2,
-  },
-);
-
-testLog(
-  'should truncate objects with error serializers',
-  {
-    errWithCause: {
+    err: {
       a: {
         b: {},
       },
+      anyField: 'a'.repeat(555),
+      stack: 'a'.repeat(555),
     },
-    err: {
+    req: {
+      method: 'GET',
+      url: 'a'.repeat(555),
+      headers: [],
+      socket: { remoteAddress: 'localhost', remotePort: '4000' },
+    },
+    res: {
+      headers: { Origin: 'a'.repeat(555) },
+      status: 500,
+      foo: 'baz',
+    },
+    headers: {
+      'test-header': 'a'.repeat(555),
       a: {
         b: {},
       },
@@ -684,44 +669,66 @@ testLog(
       a: {
         b: '[Object]',
       },
+      anyField: `${'a'.repeat(512)}...`,
+      stack: 'a'.repeat(555),
     },
     err: {
       a: {
         b: '[Object]',
       },
+      anyField: `${'a'.repeat(512)}...`,
+      stack: 'a'.repeat(555),
+    },
+    req: {
+      method: 'GET',
+      url: `${'a'.repeat(512)}...`,
+      headers: [],
+      remoteAddress: 'localhost',
+      remotePort: '4000',
+    },
+    res: {
+      headers: { Origin: `${'a'.repeat(512)}...` },
+      statusCode: 500,
+    },
+    headers: {
+      'test-header': `${'a'.repeat(512)}...`,
+      a: {
+        b: '[Object]',
+      },
     },
   },
   'info',
   {
-    maxObjectDepth: 2,
+    maxObjectDepth: 3,
   },
 );
 
 testLog(
-  'should truncate strings longer than 512 characters with error serializers',
+  'should trim custom serializer',
   {
-    err: {
-      anyField: {
-        anyField: 'a'.repeat(555),
-      },
-    },
-    errWithCause: {
-      anyField: {
-        anyField: 'a'.repeat(555),
+    serialize: {
+      a: {
+        b: {
+          c: {},
+        },
       },
+      anyField: 'a'.repeat(555),
     },
   },
   {
-    err: {
-      anyField: {
-        anyField: `${'a'.repeat(512)}...`,
+    serialize: {
+      a: {
+        b: '[Object]',
       },
+      anyField: `${'a'.repeat(512)}...`,
     },
-    errWithCause: {
-      anyField: {
-        anyField: `${'a'.repeat(512)}...`,
-      },
+  },
+  'info',
+  {
+    serializers: {
+      serialize: (input: unknown) => input,
     },
+    maxObjectDepth: 3,
   },
 );
 

src/index.ts

@@ -29,10 +29,7 @@ export default (
 ): Logger => {
   opts.redact = redact.addDefaultRedactPathStrings(opts.redact);
 
-  const serializers = {
-    ...createSerializers(opts),
-    ...opts.serializers,
-  };
+  const serializers = createSerializers(opts);
 
   opts.serializers = serializers;
 

src/serializers/index.ts

@@ -1,11 +1,11 @@
 import { trimmer } from 'dtrim';
-import type pino from 'pino';
-import { type SerializedError, err, errWithCause } from 'pino-std-serializers';
+import type { pino } from 'pino';
+import { err, errWithCause } from 'pino-std-serializers';
 
-import type { FormatterOptions } from '../formatters';
+import { DEFAULT_MAX_OBJECT_DEPTH } from '../formatters';
 
 import { createOmitPropertiesSerializer } from './omitPropertiesSerializer';
-import type { SerializerFn } from './types';
+import type { SerializerFn, TrimmerFn } from './types';
 
 export const DEFAULT_OMIT_HEADER_NAMES = Object.freeze([
   'x-envoy-attempt-count',
@@ -32,6 +32,10 @@ export interface SerializerOptions {
    * and can be disabled by supplying an empty array `[]`.
    */
   omitHeaderNames?: readonly string[];
+
+  maxObjectDepth?: number;
+
+  serializers?: pino.LoggerOptions['serializers'];
 }
 
 interface Socket {
@@ -80,30 +84,56 @@ const res = (response: Response) =>
       }
     : response;
 
-const createErrSerializer =
-  (
-    serializer: (error: Error) => SerializedError,
-    opts: FormatterOptions,
-  ): SerializerFn =>
-  (error: unknown): unknown =>
-    trimmer({
-      depth: opts.maxObjectDepth ?? 4,
-    })(serializer(error as Error));
-
-export const createSerializers = (
-  opts: SerializerOptions & FormatterOptions,
-) => {
+export const trimSerializerOutput =
+  (serializer: SerializerFn, trim: TrimmerFn): SerializerFn =>
+  (input) =>
+    trim(serializer(input));
+
+export const createSerializers = (opts: SerializerOptions) => {
   const serializeHeaders = createOmitPropertiesSerializer(
     opts.omitHeaderNames ?? DEFAULT_OMIT_HEADER_NAMES,
   );
 
+  // We are trimming inside one level of property nesting.
+  const depth = Math.max(
+    0,
+    (opts.maxObjectDepth ?? DEFAULT_MAX_OBJECT_DEPTH) - 1,
+  );
+
+  const errSerializers = trimSerializers(
+    {
+      err,
+      errWithCause,
+    },
+    // Retain long stack traces for troubleshooting purposes.
+    trimmer({ depth, retain: new Set(['stack']) }),
+  );
+
+  const restSerializers = trimSerializers(
+    {
+      req: createReqSerializer(serializeHeaders),
+      res,
+      headers: serializeHeaders,
+      ...opts.serializers,
+    },
+    trimmer({ depth }),
+  );
+
   const serializers = {
-    err: createErrSerializer(err, opts),
-    errWithCause: createErrSerializer(errWithCause, opts),
-    req: createReqSerializer(serializeHeaders),
-    res,
-    headers: serializeHeaders,
+    ...errSerializers,
+    ...restSerializers,
   } satisfies pino.LoggerOptions['serializers'];
 
   return serializers;
 };
+
+const trimSerializers = <T extends string>(
+  serializers: Record<T, SerializerFn>,
+  trim: TrimmerFn,
+) =>
+  Object.fromEntries(
+    Object.entries<SerializerFn>(serializers).map(
+      ([property, serializer]) =>
+        [property, trimSerializerOutput(serializer, trim)] as const,
+    ),
+  ) as Record<T, SerializerFn>;

src/serializers/omitPropertiesSerializer.ts

@@ -10,7 +10,7 @@ export const createOmitPropertiesSerializer = (
   const uniquePropertySet = new Set(properties);
 
   if (uniquePropertySet.size === 0) {
-    return (input) => input;
+    return (input): unknown => input;
   }
 
   const uniqueProperties = Array.from(uniquePropertySet);

src/serializers/types.ts

@@ -1 +1,3 @@
-export type SerializerFn = (input: unknown) => unknown;
+export type TrimmerFn = (input: unknown) => unknown;
+// eslint-disable-next-line @typescript-eslint/no-explicit-any
+export type SerializerFn = (input: any) => unknown;