From 5a3e62d00485aea5e464ae781a73989eb35fa2c3 Mon Sep 17 00:00:00 2001 From: Paul D'Ambra Date: Wed, 3 Apr 2024 18:34:16 +0100 Subject: [PATCH] fix: block bytespider bot (#1113) --- src/__tests__/utils.test.ts | 3 +++ src/utils/blocked-uas.ts | 10 ++++------ 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/src/__tests__/utils.test.ts b/src/__tests__/utils.test.ts index 84ef4a7e7..f3467abf7 100644 --- a/src/__tests__/utils.test.ts +++ b/src/__tests__/utils.test.ts @@ -161,6 +161,9 @@ describe('utils', () => { ], ['AhrefsSiteAudit (Desktop) - Mozilla/5.0 (compatible; AhrefsSiteAudit/6.1; +http://ahrefs.com/robot/)'], ['Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; GPTBot/1.0; +https://openai.com/gptbot)'], + [ + 'Mozilla/5.0 (Linux; Android 5.0) AppleWebKit/537.36 (KHTML, like Gecko) Mobile Safari/537.36 (compatible; Bytespider; spider-feedback@bytedance.com)', + ], ])('blocks based on user agent', (botString) => { expect(_isBlockedUA(botString, [])).toBe(true) expect(_isBlockedUA(botString.toLowerCase(), [])).toBe(true) diff --git a/src/utils/blocked-uas.ts b/src/utils/blocked-uas.ts index b823d395e..a02e9d767 100644 --- a/src/utils/blocked-uas.ts +++ b/src/utils/blocked-uas.ts @@ -46,6 +46,7 @@ export const DEFAULT_BLOCKED_UA_STRS = [ 'googleweblight', 'mediapartners-google', 'storebot-google', + 'Bytespider;', ] /** @@ -58,11 +59,8 @@ export const _isBlockedUA = function (ua: string, customBlockedUserAgents: strin const uaLower = ua.toLowerCase() return DEFAULT_BLOCKED_UA_STRS.concat(customBlockedUserAgents || []).some((blockedUA) => { const blockedUaLower = blockedUA.toLowerCase() - if (uaLower.includes) { - return uaLower.includes(blockedUaLower) - } else { - // IE 11 :/ - return uaLower.indexOf(blockedUaLower) !== -1 - } + + // can't use includes because IE 11 :/ + return uaLower.indexOf(blockedUaLower) !== -1 }) }