feat: add recording url blocklist (#1500)

* Add blocked urls to the state

* rename function to indicate new purpose

* exit early when paused or disabled

* refactor: move matching to a function

* extend status to return paused

* feat: start stop the recording when arriving/leaving from blocked url

* fix: use same name for property

* fix: don't fully stop/restart sessions

* fix: more property typos

* fix: trigger should be disabled even when no decide response was received as well

* fix: early exit only for paused state

* fix: guard condition for pause/resume

* fix: more reliable way of pausing, lose less data

* test: add a new test for url blocking functionality

* oops

* add/remove capture blocking tag

* test the document classname changes

* remove some comments, these cannot be tested as-is

src/__tests__/extensions/replay/sessionrecording.test.ts

@@ -1,5 +1,7 @@
 /// <reference lib="dom" />
 
+import '@testing-library/jest-dom'
+
 import { PostHogPersistence } from '../../../posthog-persistence'
 import {
     CONSOLE_LOG_RECORDING_ENABLED_SERVER_SIDE,
@@ -45,6 +47,7 @@ import {
 } from '@rrweb/types'
 import Mock = jest.Mock
 import { ConsentManager } from '../../../consent'
+import { waitFor } from '@testing-library/preact'
 
 // Type and source defined here designate a non-user-generated recording event
 
@@ -2178,4 +2181,73 @@ describe('SessionRecording', () => {
             )
         })
     })
+
+    describe('URL blocking', () => {
+        beforeEach(() => {
+            sessionRecording.startIfEnabledOrStop()
+            sessionRecording.afterDecideResponse(
+                makeDecideResponse({
+                    sessionRecording: {
+                        endpoint: '/s/',
+                        urlBlocklist: [
+                            {
+                                matching: 'regex',
+                                url: '/blocked',
+                            },
+                        ],
+                    },
+                })
+            )
+        })
+
+        it('flushes buffer and includes pause event when hitting blocked URL', async () => {
+            // Emit some events before hitting blocked URL
+            _emit(createIncrementalSnapshot({ data: { source: 1 } }))
+            _emit(createIncrementalSnapshot({ data: { source: 2 } }))
+
+            // Simulate URL change to blocked URL
+            fakeNavigateTo('https://test.com/blocked')
+            _emit(createIncrementalSnapshot({ data: { source: 3 } }))
+            expect(document.body).toHaveClass('ph-no-capture')
+
+            await waitFor(() => {
+                // Verify the buffer was flushed with all events including pause
+                expect(posthog.capture).toHaveBeenCalledWith(
+                    '$snapshot',
+                    {
+                        $session_id: sessionId,
+                        $window_id: 'windowId',
+                        $snapshot_bytes: expect.any(Number),
+                        $snapshot_data: [
+                            { type: 3, data: { source: 1 } },
+                            { type: 3, data: { source: 2 } },
+                        ],
+                    },
+                    expect.any(Object)
+                )
+            })
+
+            // Verify subsequent events are not captured while on blocked URL
+            _emit(createIncrementalSnapshot({ data: { source: 4 } }))
+            expect(sessionRecording['buffer'].data).toHaveLength(0)
+
+            // Simulate URL change to allowed URL
+            fakeNavigateTo('https://test.com/allowed')
+
+            // Verify recording resumes with resume event
+            _emit(createIncrementalSnapshot({ data: { source: 5 } }))
+
+            expect(document.body).not.toHaveClass('ph-no-capture')
+
+            expect(sessionRecording['buffer'].data).toStrictEqual([
+                expect.objectContaining({
+                    type: 2,
+                }),
+                expect.objectContaining({
+                    type: 3,
+                    data: { source: 5 },
+                }),
+            ])
+        })
+    })
 })

src/extensions/replay/sessionrecording.ts

@@ -84,7 +84,7 @@ type TriggerStatus = typeof TRIGGER_STATUSES[number]
  * When sampled that means a sample rate is set and the last time the session id was rotated
  * the sample rate determined this session should be sent to the server.
  */
-type SessionRecordingStatus = 'disabled' | 'sampled' | 'active' | 'buffering'
+type SessionRecordingStatus = 'disabled' | 'sampled' | 'active' | 'buffering' | 'paused'
 
 export interface SnapshotBuffer {
     size: number
@@ -211,6 +211,24 @@ function isSessionIdleEvent(e: eventWithTime): e is eventWithTime & customEvent
     return e.type === EventType.Custom && e.data.tag === 'sessionIdle'
 }
 
+function sessionRecordingUrlTriggerMatches(url: string, triggers: SessionRecordingUrlTrigger[]) {
+    return triggers.some((trigger) => {
+        switch (trigger.matching) {
+            case 'regex':
+                return new RegExp(trigger.url).test(url)
+            default:
+                return false
+        }
+    })
+}
+
+/** When we put the recording into a paused state, we add a custom event.
+ *  However in the paused state, events are dropped, and never make it to the buffer,
+ *  so we need to manually let this one through */
+function isRecordingPausedEvent(e: eventWithTime) {
+    return e.type === EventType.Custom && e.data.tag === 'recording paused'
+}
+
 export class SessionRecording {
     private _endpoint: string
     private flushBufferTimer?: any
@@ -244,6 +262,9 @@ export class SessionRecording {
     private _lastHref?: string
 
     private _urlTriggers: SessionRecordingUrlTrigger[] = []
+    private _urlBlocklist: SessionRecordingUrlTrigger[] = []
+
+    private _urlBlocked: boolean = false
 
     // Util to help developers working on this feature manually override
     _forceAllowLocalhostNetworkCapture = false
@@ -372,6 +393,10 @@ export class SessionRecording {
             return 'buffering'
         }
 
+        if (this._urlBlocked) {
+            return 'paused'
+        }
+
         if (isBoolean(this.isSampled)) {
             return this.isSampled ? 'sampled' : 'disabled'
         } else {
@@ -380,7 +405,7 @@ export class SessionRecording {
     }
 
     private get urlTriggerStatus(): TriggerStatus {
-        if (this.receivedDecide && this._urlTriggers.length === 0) {
+        if (this._urlTriggers.length === 0) {
             return 'trigger_disabled'
         }
 
@@ -615,6 +640,10 @@ export class SessionRecording {
             this._urlTriggers = response.sessionRecording.urlTriggers
         }
 
+        if (response.sessionRecording?.urlBlocklist) {
+            this._urlBlocklist = response.sessionRecording.urlBlocklist
+        }
+
         this.receivedDecide = true
         this.startIfEnabledOrStop()
     }
@@ -983,7 +1012,11 @@ export class SessionRecording {
         }
 
         // Check if the URL matches any trigger patterns
-        this._checkUrlTrigger()
+        this._checkTriggerConditions()
+
+        if (this.status === 'paused' && !isRecordingPausedEvent(rawEvent)) {
+            return
+        }
 
         // we're processing a full snapshot, so we should reset the timer
         if (rawEvent.type === EventType.FullSnapshot) {
@@ -1036,11 +1069,12 @@ export class SessionRecording {
             $window_id: this.windowId,
         }
 
-        if (this.status !== 'disabled') {
-            this._captureSnapshotBuffered(properties)
-        } else {
+        if (this.status === 'disabled') {
             this.clearBuffer()
+            return
         }
+
+        this._captureSnapshotBuffered(properties)
     }
 
     private _pageViewFallBack() {
@@ -1171,23 +1205,23 @@ export class SessionRecording {
         })
     }
 
-    private _checkUrlTrigger() {
+    private _checkTriggerConditions() {
         if (typeof window === 'undefined' || !window.location.href) {
             return
         }
 
         const url = window.location.href
 
-        if (
-            this._urlTriggers.some((trigger) => {
-                switch (trigger.matching) {
-                    case 'regex':
-                        return new RegExp(trigger.url).test(url)
-                    default:
-                        return false
-                }
-            })
-        ) {
+        const wasBlocked = this.status === 'paused'
+        const isNowBlocked = sessionRecordingUrlTriggerMatches(url, this._urlBlocklist)
+
+        if (isNowBlocked && !wasBlocked) {
+            this._pauseRecording()
+        } else if (!isNowBlocked && wasBlocked) {
+            this._resumeRecording()
+        }
+
+        if (sessionRecordingUrlTriggerMatches(url, this._urlTriggers)) {
             this._activateUrlTrigger()
         }
     }
@@ -1201,6 +1235,41 @@ export class SessionRecording {
         }
     }
 
+    private _pauseRecording() {
+        if (this.status === 'paused') {
+            return
+        }
+        logger.info(LOGGER_PREFIX + ' recording paused due to URL blocker')
+
+        this._tryAddCustomEvent('recording paused', { reason: 'url blocker' })
+
+        this._urlBlocked = true
+        document?.body?.classList?.add('ph-no-capture')
+
+        // Clear the snapshot timer since we don't want new snapshots while paused
+        clearInterval(this._fullSnapshotTimer)
+
+        // Running this in a timeout to ensure we can
+        setTimeout(() => {
+            this._flushBuffer()
+        }, 100)
+    }
+
+    private _resumeRecording() {
+        if (this.status !== 'paused') {
+            return
+        }
+
+        this._urlBlocked = false
+        document?.body?.classList?.remove('ph-no-capture')
+
+        this._tryTakeFullSnapshot()
+
+        this._scheduleFullSnapshot()
+        this._tryAddCustomEvent('recording resumed', { reason: 'left blocked url' })
+        logger.info(LOGGER_PREFIX + ' recording resumed')
+    }
+
     /**
      * this ignores the linked flag config and causes capture to start
      * (if recording would have started had the flag been received i.e. it does not override other config).

src/types.ts

@@ -428,6 +428,7 @@ export interface DecideResponse {
         linkedFlag?: string | FlagVariant | null
         networkPayloadCapture?: Pick<NetworkRecordOptions, 'recordBody' | 'recordHeaders'>
         urlTriggers?: SessionRecordingUrlTrigger[]
+        urlBlocklist?: SessionRecordingUrlTrigger[]
     }
     surveys?: boolean
     toolbarParams: ToolbarParams
@@ -646,6 +647,7 @@ export interface ErrorConversions {
 }
 
 export interface SessionRecordingUrlTrigger {
+    urlBlockList?: SessionRecordingUrlTrigger[]
     url: string
     matching: 'regex'
 }