From af9067e1ff795aa1ad25a844abbf69165e921e5c Mon Sep 17 00:00:00 2001 From: EldarHuman <137180725+EldarHuman@users.noreply.github.com> Date: Thu, 11 Jan 2024 13:39:05 +0200 Subject: [PATCH] Feature/sdknew 3382 decode url chars (#297) - Support url decoded chars in url - Updated CHANGELOG.md - Some fixes to url - Align Unit-Tests - PR fix - PR fix - PR fix - make request object handling more generic so it can support both node express and lambda edge --------- Co-authored-by: eldarlandman --- CHANGELOG.md | 5 +++++ lib/pxapi.js | 3 +++ lib/pxconfig.js | 6 ++++-- lib/pxcontext.js | 4 +++- lib/pxenforcer.js | 42 ++++++++++++++++++++++++++++++++---------- lib/pxutil.js | 8 +++++++- package-lock.json | 25 ++++++++++++++----------- 7 files changed, 68 insertions(+), 25 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 80ba7e7..a9c4d07 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,11 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](http://keepachangelog.com/) and this project adheres to [Semantic Versioning](http://semver.org/). +## [x.x.x] - xxx-x-x + +### Added +- Support for url decode reserved characters feature + ## [3.13.0] - 2023-12-21 ### Added diff --git a/lib/pxapi.js b/lib/pxapi.js index 781d269..ea6bfb1 100644 --- a/lib/pxapi.js +++ b/lib/pxapi.js @@ -75,6 +75,9 @@ function buildRequestData(ctx, config) { if (ctx.serverInfoRegion) { data.additional['server_info_region'] = ctx.serverInfoRegion; } + if (ctx.isRawUrlDifferentFromNormalizedUrl) { + data.additional['raw_url'] = ctx.rawUrl; + } if (ctx.additionalFields && ctx.additionalFields.loginCredentials) { const { loginCredentials } = ctx.additionalFields; diff --git a/lib/pxconfig.js b/lib/pxconfig.js index 8dec463..b5e38c2 100644 --- a/lib/pxconfig.js +++ b/lib/pxconfig.js @@ -104,7 +104,8 @@ class PxConfig { ['JWT_HEADER_ADDITIONAL_FIELD_NAMES', 'px_jwt_header_additional_field_names'], ['CUSTOM_IS_SENSITIVE_REQUEST', 'px_custom_is_sensitive_request'], ['LOGGER_AUTH_TOKEN', 'px_logger_auth_token'], - ['FIRST_PARTY_TIMEOUT_MS', 'px_first_party_timeout_ms'] + ['FIRST_PARTY_TIMEOUT_MS', 'px_first_party_timeout_ms'], + ['URL_DECODE_RESERVED_CHARACTERS', 'px_url_decode_reserved_characters'] ]; configKeyMapping.forEach(([targetKey, sourceKey]) => { @@ -365,7 +366,8 @@ function pxDefaultConfig() { JWT_HEADER_ADDITIONAL_FIELD_NAMES: [], CUSTOM_IS_SENSITIVE_REQUEST: '', LOGGER_AUTH_TOKEN: '', - FIRST_PARTY_TIMEOUT_MS: 4000 + FIRST_PARTY_TIMEOUT_MS: 4000, + URL_DECODE_RESERVED_CHARACTERS: false }; } diff --git a/lib/pxcontext.js b/lib/pxcontext.js index 5e9c63c..208134d 100644 --- a/lib/pxcontext.js +++ b/lib/pxcontext.js @@ -20,7 +20,9 @@ class PxContext { this.hostname = req.hostname || req.get('host'); this.userAgent = userAgent; this.uri = req.originalUrl || '/'; - this.fullUrl = req.protocol + '://' + req.get('host') + req.originalUrl; + this.rawUrl = req.originalFullUrl; + this.isRawUrlDifferentFromNormalizedUrl = req.originalFullUrl !== req.requestNormalizedUrl; + this.fullUrl = req.requestNormalizedUrl; this.originalRequest = req.originalRequest || req; this.httpVersion = req.httpVersion || ''; this.httpMethod = req.method || ''; diff --git a/lib/pxenforcer.js b/lib/pxenforcer.js index 9e7907b..0c30296 100644 --- a/lib/pxenforcer.js +++ b/lib/pxenforcer.js @@ -64,32 +64,36 @@ class PxEnforcer { } enforce(req, res, cb) { - const requestUrl = req.originalUrl; + if (!this._config.ENABLE_MODULE) { + this.logger.debug('Request will not be verified, module is disabled'); + return cb(); + } + const fullUrlFromRequest = pxUtil.getFullUrlFromRequest(req); + req.originalFullUrl = req.originalFullUrl || fullUrlFromRequest; + const normalizedUrl = this._normalizeUrl(fullUrlFromRequest); + + req['requestNormalizedUrl'] = normalizedUrl.href; + const userAgent = req.get('user-agent') || ''; const ipAddress = pxUtil.extractIP(this._config, req); this.logger.debug('Starting request verification'); - if (requestUrl.startsWith(`/${this.reversePrefix}${this._config.FIRST_PARTY_VENDOR_PATH}`)) { + if (req.originalUrl.startsWith(`/${this.reversePrefix}${this._config.FIRST_PARTY_VENDOR_PATH}`)) { // reverse proxy client return pxProxy.getClient(req, this._config, ipAddress, cb); } - if (requestUrl.startsWith(`/${this.reversePrefix}${this._config.FIRST_PARTY_XHR_PATH}`)) { + if (req.originalUrl.startsWith(`/${this.reversePrefix}${this._config.FIRST_PARTY_XHR_PATH}`)) { //reverse proxy xhr return pxProxy.sendXHR(req, this._config, ipAddress, this.reversePrefix, cb); } - if (requestUrl.startsWith(`/${this.reversePrefix}${this._config.FIRST_PARTY_CAPTCHA_PATH}`)) { + if (req.originalUrl.startsWith(`/${this.reversePrefix}${this._config.FIRST_PARTY_CAPTCHA_PATH}`)) { // reverse proxy captcha return pxProxy.getCaptcha(req, this._config, ipAddress, this.reversePrefix, cb); } - if (!this._config.ENABLE_MODULE) { - this.logger.debug('Request will not be verified, module is disabled'); - return cb(); - } - if (this._config.FILTER_BY_METHOD.includes(req.method.toUpperCase())) { this.logger.debug(`Skipping verification for filtered method ${req.method}`); return cb(); @@ -153,6 +157,19 @@ class PxEnforcer { } } + _normalizeUrl(originalUrl) { + let normalizedUrl = new URL(originalUrl); + if (this._config.URL_DECODE_RESERVED_CHARACTERS) { + try { + normalizedUrl = new URL(`${normalizedUrl.origin}${decodeURIComponent(normalizedUrl.pathname)}${normalizedUrl.search}`); + } catch (e) { + this.logger.debug(`unable to URL decode reserved characters: ${e}`); + } + } + normalizedUrl.pathname = normalizedUrl.pathname.replace(/\/+$/, '').replace(/\/+/g, '/'); + return normalizedUrl; + } + _tryModifyContext(ctx, req) { if (this._config.MODIFY_CONTEXT && typeof this._config.MODIFY_CONTEXT === 'function') { try { @@ -409,7 +426,7 @@ class PxEnforcer { } getActivityDetails(ctx) { - return { + const details = { client_uuid: ctx.uuid, http_version: ctx.httpVersion, risk_rtt: ctx.riskRtt, @@ -419,6 +436,11 @@ class PxEnforcer { request_cookie_names: ctx.requestCookieNames, enforcer_start_time: ctx.enforcerStartTime, }; + + if (ctx.isRawUrlDifferentFromNormalizedUrl) { + details.raw_url = ctx.rawUrl; + } + return details; } /** diff --git a/lib/pxutil.js b/lib/pxutil.js index 300e71f..6cd7592 100644 --- a/lib/pxutil.js +++ b/lib/pxutil.js @@ -19,6 +19,11 @@ const { EMAIL_ADDRESS_REGEX, HASH_ALGORITHM } = require('./utils/constants'); * @param {Object} headers - request headers in key value format. * @return {Array} request headers an array format. */ + +function getFullUrlFromRequest(req) { + return `${req.protocol}://${req.hostname}${req.originalUrl}`; +} + function formatHeaders(headers, sensitiveHeaders) { const retval = []; try { @@ -407,5 +412,6 @@ module.exports = { isEmailAddress, isGraphql, tryOrNull, - appendContentType + appendContentType, + getFullUrlFromRequest }; diff --git a/package-lock.json b/package-lock.json index 28186bb..1947f49 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1100,9 +1100,9 @@ } }, "node_modules/import-fresh": { - "version": "3.2.1", - "resolved": "https://registry.npmjs.org/import-fresh/-/import-fresh-3.2.1.tgz", - "integrity": "sha512-6e1q1cnWP2RXD9/keSkxHScg508CdXqXWgWBaETNhyuBFz+kUZlKboh+ISK+bU++DmbHimVBrOz/zzPe0sZ3sQ==", + "version": "3.3.0", + "resolved": "https://registry.npmjs.org/import-fresh/-/import-fresh-3.3.0.tgz", + "integrity": "sha512-veYYhQa+D1QBKznvhUHxb8faxlrwUnxseDAbAp457E0wLNio2bOSKnjYDhMj+YiAq61xrMGhQk9iXVk5FzgQMw==", "dev": true, "dependencies": { "parent-module": "^1.0.0", @@ -1110,6 +1110,9 @@ }, "engines": { "node": ">=6" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" } }, "node_modules/imurmurhash": { @@ -2203,26 +2206,26 @@ } }, "node_modules/string-width": { - "version": "4.2.0", - "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.0.tgz", - "integrity": "sha512-zUz5JD+tgqtuDjMhwIg5uFVV3dtqZ9yQJlZVfq4I01/K5Paj5UHj7VyrQOJvzawSVlKpObApbfD0Ed6yJc+1eg==", + "version": "4.2.3", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", + "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", "dev": true, "dependencies": { "emoji-regex": "^8.0.0", "is-fullwidth-code-point": "^3.0.0", - "strip-ansi": "^6.0.0" + "strip-ansi": "^6.0.1" }, "engines": { "node": ">=8" } }, "node_modules/string-width/node_modules/strip-ansi": { - "version": "6.0.0", - "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.0.tgz", - "integrity": "sha512-AuvKTrTfQNYNIctbR1K/YGTR1756GycPsg7b9bdV9Duqur4gv6aKqHXah67Z8ImS7WEz5QVcOtlfW2rZEugt6w==", + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", + "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", "dev": true, "dependencies": { - "ansi-regex": "^5.0.0" + "ansi-regex": "^5.0.1" }, "engines": { "node": ">=8"