From d3330213b40bb8251201011fbf9953891d1a990d Mon Sep 17 00:00:00 2001 From: EldarHuman <137180725+EldarHuman@users.noreply.github.com> Date: Thu, 11 Jan 2024 20:16:45 +0200 Subject: [PATCH 1/3] Release/3.14.0 (#299) * Feature/sdknew 3382 decode url chars (#297) - Support url decoded chars in url - Updated CHANGELOG.md - Some fixes to url - Align Unit-Tests - PR fix - PR fix - PR fix - make request object handling more generic so it can support both node express and lambda edge --------- Co-authored-by: eldarlandman * - release version 3.14.0 * - release version 3.14.0 - Fix getFullUrlFromRequest and use req.get('host') * - Fix unit-tests to support req.get * - release version 3.14.0 (#298) * - release version 3.14.0 * - release version 3.14.0 - Fix getFullUrlFromRequest and use req.get('host') * - Fix unit-tests to support req.get --------- Co-authored-by: chen-zimmer-px <101555597+chen-zimmer-px@users.noreply.github.com> Co-authored-by: Guy Eisenbach <117652045+guyeisenbach@users.noreply.github.com> Co-authored-by: DanBezalelpx <121791022+DanBezalelpx@users.noreply.github.com> Co-authored-by: eldarlandman --------- Co-authored-by: chen-zimmer-px <101555597+chen-zimmer-px@users.noreply.github.com> Co-authored-by: eldarlandman Co-authored-by: Guy Eisenbach <117652045+guyeisenbach@users.noreply.github.com> Co-authored-by: DanBezalelpx <121791022+DanBezalelpx@users.noreply.github.com> --- CHANGELOG.md | 5 +++++ README.md | 2 +- lib/pxapi.js | 3 +++ lib/pxconfig.js | 6 ++++-- lib/pxcontext.js | 4 +++- lib/pxenforcer.js | 42 +++++++++++++++++++++++++++++++---------- lib/pxutil.js | 8 +++++++- package-lock.json | 29 +++++++++++++++------------- package.json | 2 +- test/pxcors.test.js | 4 ++++ test/pxenforcer.test.js | 3 +++ 11 files changed, 79 insertions(+), 29 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 80ba7e7..3080962 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,11 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](http://keepachangelog.com/) and this project adheres to [Semantic Versioning](http://semver.org/). +## [3.14.0] - 2024-01-11 + +### Added +- Support for url decode reserved characters feature + ## [3.13.0] - 2023-12-21 ### Added diff --git a/README.md b/README.md index 6a55b46..63edc48 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,7 @@ [PerimeterX](http://www.perimeterx.com) Shared base for NodeJS enforcers ============================================================= -> Latest stable version: [v3.13.0](https://www.npmjs.com/package/perimeterx-node-core) +> Latest stable version: [v3.14.0](https://www.npmjs.com/package/perimeterx-node-core) This is a shared base implementation for PerimeterX Express enforcer and future NodeJS enforcers. For a fully functioning implementation example, see the [Node-Express enforcer](https://github.com/PerimeterX/perimeterx-node-express/) implementation. diff --git a/lib/pxapi.js b/lib/pxapi.js index 781d269..ea6bfb1 100644 --- a/lib/pxapi.js +++ b/lib/pxapi.js @@ -75,6 +75,9 @@ function buildRequestData(ctx, config) { if (ctx.serverInfoRegion) { data.additional['server_info_region'] = ctx.serverInfoRegion; } + if (ctx.isRawUrlDifferentFromNormalizedUrl) { + data.additional['raw_url'] = ctx.rawUrl; + } if (ctx.additionalFields && ctx.additionalFields.loginCredentials) { const { loginCredentials } = ctx.additionalFields; diff --git a/lib/pxconfig.js b/lib/pxconfig.js index 8dec463..b5e38c2 100644 --- a/lib/pxconfig.js +++ b/lib/pxconfig.js @@ -104,7 +104,8 @@ class PxConfig { ['JWT_HEADER_ADDITIONAL_FIELD_NAMES', 'px_jwt_header_additional_field_names'], ['CUSTOM_IS_SENSITIVE_REQUEST', 'px_custom_is_sensitive_request'], ['LOGGER_AUTH_TOKEN', 'px_logger_auth_token'], - ['FIRST_PARTY_TIMEOUT_MS', 'px_first_party_timeout_ms'] + ['FIRST_PARTY_TIMEOUT_MS', 'px_first_party_timeout_ms'], + ['URL_DECODE_RESERVED_CHARACTERS', 'px_url_decode_reserved_characters'] ]; configKeyMapping.forEach(([targetKey, sourceKey]) => { @@ -365,7 +366,8 @@ function pxDefaultConfig() { JWT_HEADER_ADDITIONAL_FIELD_NAMES: [], CUSTOM_IS_SENSITIVE_REQUEST: '', LOGGER_AUTH_TOKEN: '', - FIRST_PARTY_TIMEOUT_MS: 4000 + FIRST_PARTY_TIMEOUT_MS: 4000, + URL_DECODE_RESERVED_CHARACTERS: false }; } diff --git a/lib/pxcontext.js b/lib/pxcontext.js index 5e9c63c..208134d 100644 --- a/lib/pxcontext.js +++ b/lib/pxcontext.js @@ -20,7 +20,9 @@ class PxContext { this.hostname = req.hostname || req.get('host'); this.userAgent = userAgent; this.uri = req.originalUrl || '/'; - this.fullUrl = req.protocol + '://' + req.get('host') + req.originalUrl; + this.rawUrl = req.originalFullUrl; + this.isRawUrlDifferentFromNormalizedUrl = req.originalFullUrl !== req.requestNormalizedUrl; + this.fullUrl = req.requestNormalizedUrl; this.originalRequest = req.originalRequest || req; this.httpVersion = req.httpVersion || ''; this.httpMethod = req.method || ''; diff --git a/lib/pxenforcer.js b/lib/pxenforcer.js index 9e7907b..0c30296 100644 --- a/lib/pxenforcer.js +++ b/lib/pxenforcer.js @@ -64,32 +64,36 @@ class PxEnforcer { } enforce(req, res, cb) { - const requestUrl = req.originalUrl; + if (!this._config.ENABLE_MODULE) { + this.logger.debug('Request will not be verified, module is disabled'); + return cb(); + } + const fullUrlFromRequest = pxUtil.getFullUrlFromRequest(req); + req.originalFullUrl = req.originalFullUrl || fullUrlFromRequest; + const normalizedUrl = this._normalizeUrl(fullUrlFromRequest); + + req['requestNormalizedUrl'] = normalizedUrl.href; + const userAgent = req.get('user-agent') || ''; const ipAddress = pxUtil.extractIP(this._config, req); this.logger.debug('Starting request verification'); - if (requestUrl.startsWith(`/${this.reversePrefix}${this._config.FIRST_PARTY_VENDOR_PATH}`)) { + if (req.originalUrl.startsWith(`/${this.reversePrefix}${this._config.FIRST_PARTY_VENDOR_PATH}`)) { // reverse proxy client return pxProxy.getClient(req, this._config, ipAddress, cb); } - if (requestUrl.startsWith(`/${this.reversePrefix}${this._config.FIRST_PARTY_XHR_PATH}`)) { + if (req.originalUrl.startsWith(`/${this.reversePrefix}${this._config.FIRST_PARTY_XHR_PATH}`)) { //reverse proxy xhr return pxProxy.sendXHR(req, this._config, ipAddress, this.reversePrefix, cb); } - if (requestUrl.startsWith(`/${this.reversePrefix}${this._config.FIRST_PARTY_CAPTCHA_PATH}`)) { + if (req.originalUrl.startsWith(`/${this.reversePrefix}${this._config.FIRST_PARTY_CAPTCHA_PATH}`)) { // reverse proxy captcha return pxProxy.getCaptcha(req, this._config, ipAddress, this.reversePrefix, cb); } - if (!this._config.ENABLE_MODULE) { - this.logger.debug('Request will not be verified, module is disabled'); - return cb(); - } - if (this._config.FILTER_BY_METHOD.includes(req.method.toUpperCase())) { this.logger.debug(`Skipping verification for filtered method ${req.method}`); return cb(); @@ -153,6 +157,19 @@ class PxEnforcer { } } + _normalizeUrl(originalUrl) { + let normalizedUrl = new URL(originalUrl); + if (this._config.URL_DECODE_RESERVED_CHARACTERS) { + try { + normalizedUrl = new URL(`${normalizedUrl.origin}${decodeURIComponent(normalizedUrl.pathname)}${normalizedUrl.search}`); + } catch (e) { + this.logger.debug(`unable to URL decode reserved characters: ${e}`); + } + } + normalizedUrl.pathname = normalizedUrl.pathname.replace(/\/+$/, '').replace(/\/+/g, '/'); + return normalizedUrl; + } + _tryModifyContext(ctx, req) { if (this._config.MODIFY_CONTEXT && typeof this._config.MODIFY_CONTEXT === 'function') { try { @@ -409,7 +426,7 @@ class PxEnforcer { } getActivityDetails(ctx) { - return { + const details = { client_uuid: ctx.uuid, http_version: ctx.httpVersion, risk_rtt: ctx.riskRtt, @@ -419,6 +436,11 @@ class PxEnforcer { request_cookie_names: ctx.requestCookieNames, enforcer_start_time: ctx.enforcerStartTime, }; + + if (ctx.isRawUrlDifferentFromNormalizedUrl) { + details.raw_url = ctx.rawUrl; + } + return details; } /** diff --git a/lib/pxutil.js b/lib/pxutil.js index 300e71f..e032990 100644 --- a/lib/pxutil.js +++ b/lib/pxutil.js @@ -19,6 +19,11 @@ const { EMAIL_ADDRESS_REGEX, HASH_ALGORITHM } = require('./utils/constants'); * @param {Object} headers - request headers in key value format. * @return {Array} request headers an array format. */ + +function getFullUrlFromRequest(req) { + return `${req.protocol}://${req.get('host')}${req.originalUrl}`; +} + function formatHeaders(headers, sensitiveHeaders) { const retval = []; try { @@ -407,5 +412,6 @@ module.exports = { isEmailAddress, isGraphql, tryOrNull, - appendContentType + appendContentType, + getFullUrlFromRequest }; diff --git a/package-lock.json b/package-lock.json index 28186bb..88c7e76 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "perimeterx-node-core", - "version": "3.13.0", + "version": "3.14.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "perimeterx-node-core", - "version": "3.13.0", + "version": "3.14.0", "license": "ISC", "dependencies": { "agent-phin": "^1.0.4", @@ -1100,9 +1100,9 @@ } }, "node_modules/import-fresh": { - "version": "3.2.1", - "resolved": "https://registry.npmjs.org/import-fresh/-/import-fresh-3.2.1.tgz", - "integrity": "sha512-6e1q1cnWP2RXD9/keSkxHScg508CdXqXWgWBaETNhyuBFz+kUZlKboh+ISK+bU++DmbHimVBrOz/zzPe0sZ3sQ==", + "version": "3.3.0", + "resolved": "https://registry.npmjs.org/import-fresh/-/import-fresh-3.3.0.tgz", + "integrity": "sha512-veYYhQa+D1QBKznvhUHxb8faxlrwUnxseDAbAp457E0wLNio2bOSKnjYDhMj+YiAq61xrMGhQk9iXVk5FzgQMw==", "dev": true, "dependencies": { "parent-module": "^1.0.0", @@ -1110,6 +1110,9 @@ }, "engines": { "node": ">=6" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" } }, "node_modules/imurmurhash": { @@ -2203,26 +2206,26 @@ } }, "node_modules/string-width": { - "version": "4.2.0", - "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.0.tgz", - "integrity": "sha512-zUz5JD+tgqtuDjMhwIg5uFVV3dtqZ9yQJlZVfq4I01/K5Paj5UHj7VyrQOJvzawSVlKpObApbfD0Ed6yJc+1eg==", + "version": "4.2.3", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", + "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", "dev": true, "dependencies": { "emoji-regex": "^8.0.0", "is-fullwidth-code-point": "^3.0.0", - "strip-ansi": "^6.0.0" + "strip-ansi": "^6.0.1" }, "engines": { "node": ">=8" } }, "node_modules/string-width/node_modules/strip-ansi": { - "version": "6.0.0", - "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.0.tgz", - "integrity": "sha512-AuvKTrTfQNYNIctbR1K/YGTR1756GycPsg7b9bdV9Duqur4gv6aKqHXah67Z8ImS7WEz5QVcOtlfW2rZEugt6w==", + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", + "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", "dev": true, "dependencies": { - "ansi-regex": "^5.0.0" + "ansi-regex": "^5.0.1" }, "engines": { "node": ">=8" diff --git a/package.json b/package.json index 536b288..6bdf02d 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "perimeterx-node-core", - "version": "3.13.0", + "version": "3.14.0", "description": "PerimeterX NodeJS shared core for various applications to monitor and block traffic according to PerimeterX risk score", "main": "index.js", "scripts": { diff --git a/test/pxcors.test.js b/test/pxcors.test.js index ba1c0e1..f3218d2 100644 --- a/test/pxcors.test.js +++ b/test/pxcors.test.js @@ -37,7 +37,11 @@ describe('PX Cors - pxCors.js', () => { req.protocol = 'http'; req.ip = '1.2.3.4'; req.hostname = 'example.com'; + req.host = 'example.com'; req.get = (key) => { + if (key === 'host') { + return req.host; + } return req.headers[key] || ''; }; diff --git a/test/pxenforcer.test.js b/test/pxenforcer.test.js index 50ac461..5c43868 100644 --- a/test/pxenforcer.test.js +++ b/test/pxenforcer.test.js @@ -40,6 +40,9 @@ describe('PX Enforcer - pxenforcer.js', () => { req.ip = '1.2.3.4'; req.hostname = 'example.com'; req.get = (key) => { + if (key === 'host') { + return req.host; + } return req.headers[key] || ''; }; From bb73d12d8eadbe38b208983f25369bcace36e537 Mon Sep 17 00:00:00 2001 From: chen-zimmer-px Date: Wed, 17 Jan 2024 23:03:26 +0200 Subject: [PATCH 2/3] added eslint disabled --- lib/pxenforcer.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/pxenforcer.js b/lib/pxenforcer.js index 8b20555..d45eead 100644 --- a/lib/pxenforcer.js +++ b/lib/pxenforcer.js @@ -640,7 +640,8 @@ class PxEnforcer { }); } - sendHeaderBasedLogs(pxCtx, config, req) { + // @ + sendHeaderBasedLogs(pxCtx, config, req) { // eslint-disable-line // Feature has been removed, function definition for backwards compatibility. } } From 127023beb3cce088426b7c8c8a88eac53f5d8b5e Mon Sep 17 00:00:00 2001 From: chen-zimmer-px Date: Wed, 17 Jan 2024 23:06:42 +0200 Subject: [PATCH 3/3] fix --- lib/pxenforcer.js | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/lib/pxenforcer.js b/lib/pxenforcer.js index d45eead..c15a902 100644 --- a/lib/pxenforcer.js +++ b/lib/pxenforcer.js @@ -639,8 +639,7 @@ class PxEnforcer { cb(htmlTemplate); }); } - - // @ + sendHeaderBasedLogs(pxCtx, config, req) { // eslint-disable-line // Feature has been removed, function definition for backwards compatibility. }