Merge branch 'master' into test-strict-equal

.changeset/moody-bobcats-travel.md

@@ -0,0 +1,37 @@
+---
+'@seek/logger': minor
+---
+
+Omit request headers
+
+`@seek/logger` now omits the following properties from `headers` and `req.headers` by default:
+
+- `x-envoy-attempt-count`
+- `x-envoy-decorator-operation`
+- `x-envoy-expected-rq-timeout-ms`
+- `x-envoy-external-address`
+- `x-envoy-internal`
+- `x-envoy-peer-metadata`
+- `x-envoy-peer-metadata-id`
+- `x-envoy-upstream-service-time`
+
+If you would like to opt out of this behaviours, you can provide an empty list or your own list of request headers to `omitHeaderNames`:
+
+```diff
+const logger = createLogger({
+  name: 'my-app',
++ omitHeaderNames: ['dnt', 'sec-fetch-dest'],
+});
+```
+
+You can also extend the default list like so:
+
+```diff
+- import createLogger from '@seek/logger';
++ import createLogger, { DEFAULT_OMIT_HEADER_NAMES } from '@seek/logger';
+
+const logger = createLogger({
+  name: 'my-app',
++ omitHeaderNames: [...DEFAULT_OMIT_HEADER_NAMES, 'dnt', 'sec-fetch-dest']
+});
+```

README.md

@@ -45,7 +45,7 @@ childLogger.error({ err }, 'Something bad happened');
 
 ### Standardised fields
 
-**@seek/logger** bundles custom `req` and `res` serializers along with [Pino]'s standard set.
+**@seek/logger** bundles custom `req`, `res` and `headers` serializers along with [Pino]'s standard set.
 User-defined serializers will take precedence over predefined ones.
 
 Use the following standardised logging fields to benefit from customised serialization:
@@ -57,12 +57,21 @@ Use the following standardised logging fields to benefit from customised seriali
 
 - `req` for HTTP requests.
 
-  The request object is trimmed to a set of essential fields.
+  The request object is trimmed to a set of essential fields.  
+  Certain headers are omitted by default (e.g. `x-envoy-peer-metadata`).  
+  To opt out of this behavior, set the `omitHeaderNames` logger option to an empty list `[]`
+  or provide your own list.
 
 - `res` for HTTP responses.
 
   The response object is trimmed to a set of essential fields.
 
+- `headers` for tracing headers.
+
+  Certain headers are omitted by default (e.g. `x-envoy-peer-metadata`).  
+  To opt out of this behavior, set the `omitHeaderNames` logger option to an empty list `[]`
+  or provide your own list.
+
 All other fields will be logged directly.
 
 ### Typed fields

src/index.test.ts

@@ -1,5 +1,7 @@
 import split from 'split2';
 
+import { DEFAULT_OMIT_HEADER_NAMES } from './serializers';
+
 import createLogger, { type LoggerOptions } from '.';
 
 const bearerToken =
@@ -586,3 +588,103 @@ testLog(
     maxObjectDepth: 2,
   },
 );
+
+const objectWithDefaultOmitHeaderNameKeys = Object.fromEntries(
+  DEFAULT_OMIT_HEADER_NAMES.map((headerName) => [headerName, 'header value']),
+);
+
+testLog(
+  'should omit defaultOmitHeaderNames by default',
+  {
+    headers: {
+      ['authorization']: bearerToken,
+      ...objectWithDefaultOmitHeaderNameKeys,
+      ['x-request-id']: 'some-uuid',
+    },
+    req: {
+      headers: {
+        ['authorization']: bearerToken,
+        ...objectWithDefaultOmitHeaderNameKeys,
+        ['x-request-id']: 'some-uuid',
+      },
+    },
+  },
+  {
+    headers: {
+      ['authorization']: redactedBearer,
+      ['x-request-id']: 'some-uuid',
+    },
+    req: {
+      headers: {
+        ['authorization']: redactedBearer,
+        ['x-request-id']: 'some-uuid',
+      },
+    },
+  },
+  'info',
+);
+
+testLog(
+  'should keep defaultOmitHeaderNames when omitHeaderNames option is empty',
+  {
+    headers: {
+      ['authorization']: bearerToken,
+      ...objectWithDefaultOmitHeaderNameKeys,
+      ['x-request-id']: 'some-uuid',
+    },
+    req: {
+      headers: {
+        ['authorization']: bearerToken,
+        ...objectWithDefaultOmitHeaderNameKeys,
+        ['x-request-id']: 'some-uuid',
+      },
+    },
+  },
+  {
+    headers: {
+      ['authorization']: redactedBearer,
+      ...objectWithDefaultOmitHeaderNameKeys,
+      ['x-request-id']: 'some-uuid',
+    },
+    req: {
+      headers: {
+        ['authorization']: redactedBearer,
+        ...objectWithDefaultOmitHeaderNameKeys,
+        ['x-request-id']: 'some-uuid',
+      },
+    },
+  },
+  'info',
+  { omitHeaderNames: [] },
+);
+
+test('it merges serializers', async () => {
+  const stream = sink();
+  const logger = createLogger(
+    {
+      name: 'my-app',
+      omitHeaderNames: ['omit'],
+      serializers: {
+        serialize: () => 'serialized',
+      },
+    },
+    stream,
+  );
+
+  logger.info(
+    { req: { headers: { omit: 'raw' } }, serialize: 'raw' },
+    'Test log entry',
+  );
+  const reqLog: any = await once(stream, 'data');
+
+  expect(reqLog).toHaveProperty('serialize', 'serialized');
+  expect(reqLog.req.headers).not.toHaveProperty('omit');
+  expect(reqLog).not.toHaveProperty('headers');
+
+  logger.info({ headers: { omit: 'raw' }, serialize: 'raw' }, 'Test log entry');
+  const rootLog: any = await once(stream, 'data');
+
+  expect(rootLog).toHaveProperty('serialize', 'serialized');
+  expect(rootLog.headers).not.toHaveProperty('omit');
+  expect(rootLog).not.toHaveProperty('req');
+});

src/index.ts

@@ -4,11 +4,15 @@ import base from './base';
 import { withRedaction } from './destination';
 import { type FormatterOptions, createFormatters } from './formatters';
 import * as redact from './redact';
-import serializers from './serializers';
+import { type SerializerOptions, createSerializers } from './serializers';
+
+export { DEFAULT_OMIT_HEADER_NAMES } from './serializers';
 
 export { pino };
 
-export type LoggerOptions = pino.LoggerOptions & FormatterOptions;
+export type LoggerOptions = pino.LoggerOptions &
+  FormatterOptions &
+  SerializerOptions;
 export type Logger = pino.Logger;
 
 /**
@@ -25,10 +29,13 @@ export default (
   }),
 ): Logger => {
   opts.redact = redact.addDefaultRedactPathStrings(opts.redact);
+
+  const serializers = createSerializers(opts);
   opts.serializers = {
     ...serializers,
     ...opts.serializers,
   };
+
   const formatters = createFormatters(opts);
   opts.base = {
     ...base,

src/serializers/index.test.ts

@@ -1,4 +1,6 @@
-import serializers from './index';
+import { DEFAULT_OMIT_HEADER_NAMES, createSerializers } from '.';
+
+const serializers = createSerializers({});
 
 describe('req', () => {
   const remoteAddress = '::ffff:123.45.67.89';
@@ -86,7 +88,49 @@ describe('req', () => {
   `('remoteAddress and remotePort is undefined when $scenario', ({ value }) => {
     const result = serializers.req(value);
 
-    expect(result).toStrictEqual({ ...expectedRequestBase });
+    expect(result).toStrictEqual(expectedRequestBase);
+  });
+
+  const objectWithDefaultOmitHeaderNameKeys = Object.fromEntries(
+    DEFAULT_OMIT_HEADER_NAMES.map((headerName) => [headerName, 'header value']),
+  );
+
+  it('omits defaultOmitHeaderNames by default', () => {
+    const request = {
+      ...requestBase,
+      headers: {
+        ...requestBase.headers,
+        ...objectWithDefaultOmitHeaderNameKeys,
+      },
+    };
+
+    const result = serializers.req(request);
+
+    expect(result).toStrictEqual(expectedRequestBase);
+  });
+
+  it('omits only specified headers when omitHeaderNames is provided', () => {
+    const request = {
+      ...requestBase,
+      headers: {
+        ...requestBase.headers,
+        ['omit-me']: 'header value',
+        ...objectWithDefaultOmitHeaderNameKeys,
+      },
+    };
+
+    const expectedRequest = {
+      ...expectedRequestBase,
+      headers: {
+        ...requestBase.headers,
+        ...objectWithDefaultOmitHeaderNameKeys,
+      },
+    };
+
+    const altSerializers = createSerializers({ omitHeaderNames: ['omit-me'] });
+    const result = altSerializers.req(request);
+
+    expect(result).toStrictEqual(expectedRequest);
   });
 });
 
@@ -124,12 +168,15 @@ describe('res', () => {
 });
 
 describe('serializers', () => {
-  test('it exports only err, errWithCause, req, res', () => {
-    expect(serializers).toStrictEqual({
-      err: expect.any(Function),
-      errWithCause: expect.any(Function),
-      req: expect.any(Function),
-      res: expect.any(Function),
-    });
+  test('it exports the expected properties', () => {
+    expect(serializers).toMatchInlineSnapshot(`
+      {
+        "err": [Function],
+        "errWithCause": [Function],
+        "headers": [Function],
+        "req": [Function],
+        "res": [Function],
+      }
+    `);
   });
 });

src/serializers/index.ts

@@ -1,5 +1,36 @@
+import type pino from 'pino';
 import { err, errWithCause } from 'pino-std-serializers';
 
+import { createOmitPropertiesSerializer } from './omitPropertiesSerializer';
+import type { SerializerFn } from './types';
+
+export const DEFAULT_OMIT_HEADER_NAMES = [
+  'x-envoy-attempt-count',
+  'x-envoy-decorator-operation',
+  'x-envoy-expected-rq-timeout-ms',
+  'x-envoy-external-address',
+  'x-envoy-internal',
+  'x-envoy-peer-metadata',
+  'x-envoy-peer-metadata-id',
+  'x-envoy-upstream-service-time',
+];
+
+export interface SerializerOptions {
+  /**
+   * The request headers to omit from serialized logs.
+   *
+   * The properties listed will be removed under `headers` and `req.headers`.
+   * Matching is currently case sensitive.
+   * You will typically express the header names in lowercase,
+   * as server frameworks normalise incoming headers.
+   *
+   * You can use this option to reduce logging costs.
+   * Defaults to `DEFAULT_OMIT_HEADER_NAMES`,
+   * and can be disabled by supplying an empty array `[]`.
+   */
+  omitHeaderNames?: string[];
+}
+
 interface Socket {
   remoteAddress?: string;
   remotePort?: string;
@@ -8,7 +39,7 @@ interface Request extends Record<string, unknown> {
   method: string;
   url: string;
   headers: Record<string, string>;
-  socket: Socket;
+  socket?: Socket;
 }
 
 interface Response extends Record<string, unknown> {
@@ -26,16 +57,17 @@ const isObject = (value: unknown): boolean => {
   return value != null && (type === 'object' || type === 'function');
 };
 
-const req = (request: Request) =>
-  isObject(request)
-    ? {
-        method: request.method,
-        url: request.url,
-        headers: request.headers,
-        remoteAddress: request?.socket?.remoteAddress,
-        remotePort: request?.socket?.remotePort,
-      }
-    : request;
+const createReqSerializer =
+  (serializeHeaders: SerializerFn) => (request: Request) =>
+    isObject(request)
+      ? {
+          method: request.method,
+          url: request.url,
+          headers: serializeHeaders(request.headers),
+          remoteAddress: request?.socket?.remoteAddress,
+          remotePort: request?.socket?.remotePort,
+        }
+      : request;
 
 const res = (response: Response) =>
   isObject(response)
@@ -45,9 +77,18 @@ const res = (response: Response) =>
       }
     : response;
 
-export default {
-  err,
-  errWithCause,
-  res,
-  req,
+export const createSerializers = (opts: SerializerOptions) => {
+  const serializeHeaders = createOmitPropertiesSerializer(
+    opts.omitHeaderNames ?? DEFAULT_OMIT_HEADER_NAMES,
+  );
+
+  const serializers = {
+    err,
+    errWithCause,
+    req: createReqSerializer(serializeHeaders),
+    res,
+    headers: serializeHeaders,
+  } satisfies pino.LoggerOptions['serializers'];
+
+  return serializers;
 };