feat: capture network payloads (internal alpha) (#886)

rrweb has network payloads queued up as a feature... but it's taking a while.

The easiest way to test it is to adopt it ourselves.

This adds a copy of the plugin proposed for rrweb, and uses it to wrap xhr and fetch.

We can match performance timings and these new NetworkRequests based on URL and timings

used by PostHog/posthog#18562

for now this can only be enabled via decide response, which allows header and body capture to be configured separately, that config is only enabled via flag while we test internally

cypress/e2e/session-recording.cy.js

@@ -43,8 +43,7 @@ describe('Session recording', () => {
                     cy.phCaptures({ full: true }).then((captures) => {
                         // should be a pageview and a $snapshot
                         expect(captures.map((c) => c.event)).to.deep.equal(['$pageview', '$snapshot'])
-                        // the amount of captured data should be deterministic
-                        // but of course that would be too easy
+
                         expect(captures[1]['properties']['$snapshot_data']).to.have.length.above(33).and.below(38)
                         // a meta and then a full snapshot
                         expect(captures[1]['properties']['$snapshot_data'][0].type).to.equal(4) // meta

src/__tests__/extensions/replay/config.test.ts

@@ -0,0 +1,80 @@
+import { defaultConfig } from '../../../posthog-core'
+import { buildNetworkRequestOptions } from '../../../extensions/replay/config'
+
+describe('config', () => {
+    describe('network request options', () => {
+        describe('maskRequestFn', () => {
+            it('can enable header recording remotely', () => {
+                const networkOptions = buildNetworkRequestOptions(defaultConfig(), { recordHeaders: true })
+                expect(networkOptions.recordHeaders).toBe(true)
+                expect(networkOptions.recordBody).toBe(undefined)
+            })
+
+            it('can enable body recording remotely', () => {
+                const networkOptions = buildNetworkRequestOptions(defaultConfig(), { recordBody: true })
+                expect(networkOptions.recordHeaders).toBe(undefined)
+                expect(networkOptions.recordBody).toBe(true)
+            })
+
+            it('client can force disable recording', () => {
+                const posthogConfig = defaultConfig()
+                posthogConfig.session_recording.recordHeaders = false
+                posthogConfig.session_recording.recordBody = false
+                const networkOptions = buildNetworkRequestOptions(posthogConfig, {
+                    recordHeaders: true,
+                    recordBody: true,
+                })
+                expect(networkOptions.recordHeaders).toBe(false)
+                expect(networkOptions.recordBody).toBe(false)
+            })
+
+            it('should remove the Authorization header from requests even if no other config is set', () => {
+                const networkOptions = buildNetworkRequestOptions(defaultConfig(), {})
+                const cleaned = networkOptions.maskRequestFn!({
+                    url: 'something',
+                    requestHeaders: {
+                        Authorization: 'Bearer 123',
+                        'content-type': 'application/json',
+                    },
+                })
+                expect(cleaned?.requestHeaders).toEqual({
+                    'content-type': 'application/json',
+                })
+            })
+
+            it('should cope with no headers when even if no other config is set', () => {
+                const networkOptions = buildNetworkRequestOptions(defaultConfig(), {})
+                const cleaned = networkOptions.maskRequestFn!({
+                    url: 'something',
+                    requestHeaders: undefined,
+                })
+                expect(cleaned?.requestHeaders).toBeUndefined()
+            })
+
+            it('should remove the Authorization header from requests even when a mask request fn is set', () => {
+                const posthogConfig = defaultConfig()
+                posthogConfig.session_recording.maskNetworkRequestFn = (data) => {
+                    return {
+                        ...data,
+                        requestHeaders: {
+                            ...(data.requestHeaders ? data.requestHeaders : {}),
+                            'content-type': 'edited',
+                        },
+                    }
+                }
+                const networkOptions = buildNetworkRequestOptions(posthogConfig, {})
+
+                const cleaned = networkOptions.maskRequestFn!({
+                    url: 'something',
+                    requestHeaders: {
+                        Authorization: 'Bearer 123',
+                        'content-type': 'application/json',
+                    },
+                })
+                expect(cleaned?.requestHeaders).toEqual({
+                    'content-type': 'edited',
+                })
+            })
+        })
+    })
+})

src/extensions/replay/config.ts

@@ -0,0 +1,71 @@
+import { NetworkRecordOptions, NetworkRequest, PostHogConfig } from '../../types'
+import { _isFunction } from '../../utils/type-utils'
+
+export const defaultNetworkOptions: NetworkRecordOptions = {
+    initiatorTypes: [
+        'audio',
+        'beacon',
+        'body',
+        'css',
+        'early-hint',
+        'embed',
+        'fetch',
+        'frame',
+        'iframe',
+        'icon',
+        'image',
+        'img',
+        'input',
+        'link',
+        'navigation',
+        'object',
+        'ping',
+        'script',
+        'track',
+        'video',
+        'xmlhttprequest',
+    ],
+    maskRequestFn: (data: NetworkRequest) => data,
+    recordHeaders: false,
+    recordBody: false,
+    recordInitialRequests: false,
+}
+
+const removeAuthorizationHeader = (data: NetworkRequest): NetworkRequest => {
+    delete data.requestHeaders?.['Authorization']
+    return data
+}
+
+/**
+ *  whether a maskRequestFn is provided or not,
+ *  we ensure that we remove the Authorization header from requests
+ *  we _never_ want to record that header by accident
+ *  if someone complains then we'll add an opt-in to let them override it
+ */
+export const buildNetworkRequestOptions = (
+    instanceConfig: PostHogConfig,
+    remoteNetworkOptions: Pick<NetworkRecordOptions, 'recordHeaders' | 'recordBody'>
+): NetworkRecordOptions => {
+    const config = instanceConfig.session_recording as NetworkRecordOptions
+    // client can always disable despite remote options
+    const canRecordHeaders = config.recordHeaders === false ? false : remoteNetworkOptions.recordHeaders
+    const canRecordBody = config.recordBody === false ? false : remoteNetworkOptions.recordBody
+
+    config.maskRequestFn = _isFunction(instanceConfig.session_recording.maskNetworkRequestFn)
+        ? (data) => {
+              const cleanedRequest = removeAuthorizationHeader(data)
+              return instanceConfig.session_recording.maskNetworkRequestFn?.(cleanedRequest) ?? undefined
+          }
+        : undefined
+
+    if (!config.maskRequestFn) {
+        config.maskRequestFn = removeAuthorizationHeader
+    }
+
+    return {
+        ...defaultNetworkOptions,
+        ...config,
+        recordHeaders: canRecordHeaders,
+        recordBody: canRecordBody,
+    }
+}

src/extensions/replay/sessionrecording.ts

@@ -14,13 +14,15 @@ import {
     truncateLargeConsoleLogs,
 } from './sessionrecording-utils'
 import { PostHog } from '../../posthog-core'
-import { DecideResponse, NetworkRequest, Properties } from '../../types'
+import { DecideResponse, NetworkRecordOptions, NetworkRequest, Properties } from '../../types'
 import { EventType, type eventWithTime, type listenerHandler } from '@rrweb/types'
 import Config from '../../config'
 import { _timestamp, loadScript } from '../../utils'
 
-import { _isBoolean, _isNull, _isNumber, _isObject, _isString, _isUndefined } from '../../utils/type-utils'
+import { _isBoolean, _isFunction, _isNull, _isNumber, _isObject, _isString, _isUndefined } from '../../utils/type-utils'
 import { logger } from '../../utils/logger'
+import { window } from '../../utils/globals'
+import { buildNetworkRequestOptions } from './config'
 
 const BASE_ENDPOINT = '/s/'
 
@@ -90,6 +92,7 @@ export class SessionRecording {
     private receivedDecide: boolean
     private rrwebRecord: rrwebRecord | undefined
     private isIdle = false
+    private _networkPayloadCapture: Pick<NetworkRecordOptions, 'recordHeaders' | 'recordBody'> | undefined = undefined
 
     private _linkedFlagSeen: boolean = false
     private _lastActivityTimestamp: number = Date.now()
@@ -252,6 +255,8 @@ export class SessionRecording {
             })
         }
 
+        this._networkPayloadCapture = response.sessionRecording?.networkPayloadCapture
+
         const receivedSampleRate = response.sessionRecording?.sampleRate
         this._sampleRate =
             _isUndefined(receivedSampleRate) || _isNull(receivedSampleRate) ? null : parseFloat(receivedSampleRate)
@@ -462,14 +467,26 @@ export class SessionRecording {
                 },
             })
 
+        const plugins = []
+
+        if ((window as any).rrwebConsoleRecord && this.isConsoleLogCaptureEnabled) {
+            plugins.push((window as any).rrwebConsoleRecord.getRecordConsolePlugin())
+        }
+        if (this._networkPayloadCapture) {
+            if (_isFunction((window as any).getRecordNetworkPlugin)) {
+                plugins.push(
+                    (window as any).getRecordNetworkPlugin(
+                        buildNetworkRequestOptions(this.instance.config, this._networkPayloadCapture)
+                    )
+                )
+            }
+        }
+
         this.stopRrweb = this.rrwebRecord({
             emit: (event) => {
                 this.onRRwebEmit(event)
             },
-            plugins:
-                (window as any).rrwebConsoleRecord && this.isConsoleLogCaptureEnabled
-                    ? [(window as any).rrwebConsoleRecord.getRecordConsolePlugin()]
-                    : [],
+            plugins,
             ...sessionRecordingOptions,
         })
 
@@ -550,6 +567,8 @@ export class SessionRecording {
                 url,
             }
 
+            // TODO we should deprecate this and use the same function for this masking and the rrweb/network plugin
+            // TODO or deprecate this and provide a new clearer name so this would be `maskURLPerformanceFn` or similar
             networkRequest = userSessionRecordingOptions.maskNetworkRequestFn(networkRequest)
 
             return networkRequest?.url