Feature/sdknew 3382 decode url chars (#297)

- Support url decoded chars in url - Updated CHANGELOG.md - Some fixes to url - Align Unit-Tests - PR fix - PR fix - PR fix - make request object handling more generic so it can support both node express and lambda edge --------- Co-authored-by: eldarlandman <[email protected]>
PerimeterX · Jan 11, 2024 · af9067e · af9067e
1 parent a0cfb6e
commit af9067e
Show file tree

Hide file tree

Showing 7 changed files with 68 additions and 25 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -5,6 +5,11 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](http://keepachangelog.com/)
 and this project adheres to [Semantic Versioning](http://semver.org/).
 
+## [x.x.x] - xxx-x-x
+
+### Added
+- Support for url decode reserved characters feature
+
 ## [3.13.0] - 2023-12-21
 
 ### Added

diff --git a/lib/pxapi.js b/lib/pxapi.js
@@ -75,6 +75,9 @@ function buildRequestData(ctx, config) {
     if (ctx.serverInfoRegion) {
         data.additional['server_info_region'] = ctx.serverInfoRegion;
     }
+    if (ctx.isRawUrlDifferentFromNormalizedUrl) {
+        data.additional['raw_url'] = ctx.rawUrl;
+    }
 
     if (ctx.additionalFields && ctx.additionalFields.loginCredentials) {
         const { loginCredentials } = ctx.additionalFields;

diff --git a/lib/pxconfig.js b/lib/pxconfig.js
@@ -104,7 +104,8 @@ class PxConfig {
             ['JWT_HEADER_ADDITIONAL_FIELD_NAMES', 'px_jwt_header_additional_field_names'],
             ['CUSTOM_IS_SENSITIVE_REQUEST', 'px_custom_is_sensitive_request'],
             ['LOGGER_AUTH_TOKEN', 'px_logger_auth_token'],
-            ['FIRST_PARTY_TIMEOUT_MS', 'px_first_party_timeout_ms']
+            ['FIRST_PARTY_TIMEOUT_MS', 'px_first_party_timeout_ms'],
+            ['URL_DECODE_RESERVED_CHARACTERS', 'px_url_decode_reserved_characters']
         ];
 
         configKeyMapping.forEach(([targetKey, sourceKey]) => {
@@ -365,7 +366,8 @@ function pxDefaultConfig() {
         JWT_HEADER_ADDITIONAL_FIELD_NAMES: [],
         CUSTOM_IS_SENSITIVE_REQUEST: '',
         LOGGER_AUTH_TOKEN: '',
-        FIRST_PARTY_TIMEOUT_MS: 4000
+        FIRST_PARTY_TIMEOUT_MS: 4000,
+        URL_DECODE_RESERVED_CHARACTERS: false
     };
 }
 

diff --git a/lib/pxcontext.js b/lib/pxcontext.js
@@ -20,7 +20,9 @@ class PxContext {
         this.hostname = req.hostname || req.get('host');
         this.userAgent = userAgent;
         this.uri = req.originalUrl || '/';
-        this.fullUrl = req.protocol + '://' + req.get('host') + req.originalUrl;
+        this.rawUrl = req.originalFullUrl;
+        this.isRawUrlDifferentFromNormalizedUrl = req.originalFullUrl !== req.requestNormalizedUrl;
+        this.fullUrl = req.requestNormalizedUrl;
         this.originalRequest = req.originalRequest || req;
         this.httpVersion = req.httpVersion || '';
         this.httpMethod = req.method || '';

diff --git a/lib/pxenforcer.js b/lib/pxenforcer.js
@@ -64,32 +64,36 @@ class PxEnforcer {
     }
 
     enforce(req, res, cb) {
-        const requestUrl = req.originalUrl;
+        if (!this._config.ENABLE_MODULE) {
+            this.logger.debug('Request will not be verified, module is disabled');
+            return cb();
+        }
+        const fullUrlFromRequest = pxUtil.getFullUrlFromRequest(req);
+        req.originalFullUrl = req.originalFullUrl || fullUrlFromRequest;
+        const normalizedUrl = this._normalizeUrl(fullUrlFromRequest);
+
+        req['requestNormalizedUrl'] = normalizedUrl.href;
+
         const userAgent = req.get('user-agent') || '';
         const ipAddress = pxUtil.extractIP(this._config, req);
 
         this.logger.debug('Starting request verification');
 
-        if (requestUrl.startsWith(`/${this.reversePrefix}${this._config.FIRST_PARTY_VENDOR_PATH}`)) {
+        if (req.originalUrl.startsWith(`/${this.reversePrefix}${this._config.FIRST_PARTY_VENDOR_PATH}`)) {
             // reverse proxy client
             return pxProxy.getClient(req, this._config, ipAddress, cb);
         }
 
-        if (requestUrl.startsWith(`/${this.reversePrefix}${this._config.FIRST_PARTY_XHR_PATH}`)) {
+        if (req.originalUrl.startsWith(`/${this.reversePrefix}${this._config.FIRST_PARTY_XHR_PATH}`)) {
             //reverse proxy xhr
             return pxProxy.sendXHR(req, this._config, ipAddress, this.reversePrefix, cb);
         }
 
-        if (requestUrl.startsWith(`/${this.reversePrefix}${this._config.FIRST_PARTY_CAPTCHA_PATH}`)) {
+        if (req.originalUrl.startsWith(`/${this.reversePrefix}${this._config.FIRST_PARTY_CAPTCHA_PATH}`)) {
             // reverse proxy captcha
             return pxProxy.getCaptcha(req, this._config, ipAddress, this.reversePrefix, cb);
         }
 
-        if (!this._config.ENABLE_MODULE) {
-            this.logger.debug('Request will not be verified, module is disabled');
-            return cb();
-        }
-
         if (this._config.FILTER_BY_METHOD.includes(req.method.toUpperCase())) {
             this.logger.debug(`Skipping verification for filtered method ${req.method}`);
             return cb();
@@ -153,6 +157,19 @@ class PxEnforcer {
         }
     }
 
+    _normalizeUrl(originalUrl) {
+        let normalizedUrl = new URL(originalUrl);
+        if (this._config.URL_DECODE_RESERVED_CHARACTERS) {
+            try {
+                normalizedUrl = new URL(`${normalizedUrl.origin}${decodeURIComponent(normalizedUrl.pathname)}${normalizedUrl.search}`);
+            } catch (e) {
+                this.logger.debug(`unable to URL decode reserved characters: ${e}`);
+            }
+        }
+        normalizedUrl.pathname = normalizedUrl.pathname.replace(/\/+$/, '').replace(/\/+/g, '/');
+        return normalizedUrl;
+    }
+
     _tryModifyContext(ctx, req) {
         if (this._config.MODIFY_CONTEXT && typeof this._config.MODIFY_CONTEXT === 'function') {
             try {
@@ -409,7 +426,7 @@ class PxEnforcer {
     }
 
     getActivityDetails(ctx) {
-        return {
+        const details = {
             client_uuid: ctx.uuid,
             http_version: ctx.httpVersion,
             risk_rtt: ctx.riskRtt,
@@ -419,6 +436,11 @@ class PxEnforcer {
             request_cookie_names: ctx.requestCookieNames,
             enforcer_start_time: ctx.enforcerStartTime,
         };
+
+        if (ctx.isRawUrlDifferentFromNormalizedUrl) {
+            details.raw_url = ctx.rawUrl;
+        }
+        return details;
     }
 
     /**

diff --git a/lib/pxutil.js b/lib/pxutil.js
@@ -19,6 +19,11 @@ const { EMAIL_ADDRESS_REGEX, HASH_ALGORITHM } = require('./utils/constants');
  * @param {Object} headers - request headers in key value format.
  * @return {Array} request headers an array format.
  */
+
+function getFullUrlFromRequest(req) {
+    return `${req.protocol}://${req.hostname}${req.originalUrl}`;
+}
+
 function formatHeaders(headers, sensitiveHeaders) {
     const retval = [];
     try {
@@ -407,5 +412,6 @@ module.exports = {
     isEmailAddress,
     isGraphql,
     tryOrNull,
-    appendContentType
+    appendContentType,
+    getFullUrlFromRequest
 };
diff --git a/package-lock.json b/package-lock.json