From 229ce473af1ddd62d946bea1b018c882b142a5ef Mon Sep 17 00:00:00 2001 From: AASHISH MALIK Date: Tue, 12 Mar 2024 17:53:51 +0530 Subject: [PATCH 1/4] fix: send proper status to server in cm360 (#3127) --- .../campaign_manager/networkHandler.js | 4 ++-- .../campaign_manager/dataDelivery/oauth.ts | 24 +++++++++---------- .../campaign_manager/dataDelivery/other.ts | 8 +++---- 3 files changed, 18 insertions(+), 18 deletions(-) diff --git a/src/v1/destinations/campaign_manager/networkHandler.js b/src/v1/destinations/campaign_manager/networkHandler.js index 79f7e7f93b..300b5f9676 100644 --- a/src/v1/destinations/campaign_manager/networkHandler.js +++ b/src/v1/destinations/campaign_manager/networkHandler.js @@ -69,7 +69,7 @@ const responseHandler = (responseParams) => { const errorMessage = response.error?.message || 'unknown error format'; for (const metadata of rudderJobMetadata) { responseWithIndividualEvents.push({ - statusCode: 500, + statusCode: status, metadata, error: errorMessage, }); @@ -77,7 +77,7 @@ const responseHandler = (responseParams) => { throw new TransformerProxyError( `Campaign Manager: Error transformer proxy v1 during CAMPAIGN_MANAGER response transformation`, - 500, + status, { [tags.TAG_NAMES.ERROR_TYPE]: getDynamicErrorType(status), }, diff --git a/test/integrations/destinations/campaign_manager/dataDelivery/oauth.ts b/test/integrations/destinations/campaign_manager/dataDelivery/oauth.ts index 929af485d8..288a06bfe6 100644 --- a/test/integrations/destinations/campaign_manager/dataDelivery/oauth.ts +++ b/test/integrations/destinations/campaign_manager/dataDelivery/oauth.ts @@ -326,14 +326,14 @@ export const v1oauthScenarios: ProxyV1TestData[] = [ }, output: { response: { - status: 500, + status: 401, body: { output: { response: [ { error: '{"error":{"code":401,"message":"Request is missing required authentication credential. Expected OAuth 2 access token, login cookie or other valid authentication credential. See https://developers.google.com/identity/sign-in/web/devconsole-project.","errors":[{"message":"Login Required.","domain":"global","reason":"required","location":"Authorization","locationType":"header"}],"status":"UNAUTHENTICATED","details":[{"@type":"type.googleapis.com/google.rpc.ErrorInfo","reason":"CREDENTIALS_MISSING","domain":"googleapis.com","metadata":{"method":"google.ads.xfa.op.v4.DfareportingConversions.Batchinsert","service":"googleapis.com"}}]}}', - statusCode: 500, + statusCode: 401, metadata: { jobId: 1, attemptNum: 1, @@ -361,7 +361,7 @@ export const v1oauthScenarios: ProxyV1TestData[] = [ authErrorCategory: 'REFRESH_TOKEN', message: 'Campaign Manager: Error transformer proxy v1 during CAMPAIGN_MANAGER response transformation', - status: 500, + status: 401, }, }, }, @@ -389,14 +389,14 @@ export const v1oauthScenarios: ProxyV1TestData[] = [ }, output: { response: { - status: 500, + status: 403, body: { output: { response: [ { error: '{"error":{"code":403,"message":"Request had insufficient authentication scopes.","errors":[{"message":"Insufficient Permission","domain":"global","reason":"insufficientPermissions"}],"status":"PERMISSION_DENIED","details":[{"@type":"type.googleapis.com/google.rpc.ErrorInfo","reason":"ACCESS_TOKEN_SCOPE_INSUFFICIENT","domain":"googleapis.com","metadata":{"service":"gmail.googleapis.com","method":"caribou.api.proto.MailboxService.GetProfile"}}]}}', - statusCode: 500, + statusCode: 403, metadata: { jobId: 1, attemptNum: 1, @@ -424,7 +424,7 @@ export const v1oauthScenarios: ProxyV1TestData[] = [ authErrorCategory: 'AUTH_STATUS_INACTIVE', message: 'Campaign Manager: Error transformer proxy v1 during CAMPAIGN_MANAGER response transformation', - status: 500, + status: 403, }, }, }, @@ -452,14 +452,14 @@ export const v1oauthScenarios: ProxyV1TestData[] = [ }, output: { response: { - status: 500, + status: 403, body: { output: { response: [ { error: '{"error":{"code":403,"message":"invalid_grant","error_description":"Bad accesss"}}', - statusCode: 500, + statusCode: 403, metadata: { jobId: 1, attemptNum: 1, @@ -487,7 +487,7 @@ export const v1oauthScenarios: ProxyV1TestData[] = [ authErrorCategory: 'AUTH_STATUS_INACTIVE', message: 'Campaign Manager: Error transformer proxy v1 during CAMPAIGN_MANAGER response transformation', - status: 500, + status: 403, }, }, }, @@ -514,14 +514,14 @@ export const v1oauthScenarios: ProxyV1TestData[] = [ }, output: { response: { - status: 500, + status: 401, body: { output: { response: [ { error: '{"error":"unauthorized","error_description":"Access token expired: 2020-10-20T12:00:00.000Z"}', - statusCode: 500, + statusCode: 401, metadata: { jobId: 1, attemptNum: 1, @@ -549,7 +549,7 @@ export const v1oauthScenarios: ProxyV1TestData[] = [ authErrorCategory: 'REFRESH_TOKEN', message: 'Campaign Manager: Error transformer proxy v1 during CAMPAIGN_MANAGER response transformation', - status: 500, + status: 401, }, }, }, diff --git a/test/integrations/destinations/campaign_manager/dataDelivery/other.ts b/test/integrations/destinations/campaign_manager/dataDelivery/other.ts index 709f55a4c0..1c0c45728c 100644 --- a/test/integrations/destinations/campaign_manager/dataDelivery/other.ts +++ b/test/integrations/destinations/campaign_manager/dataDelivery/other.ts @@ -260,7 +260,7 @@ export const otherScenariosV1: ProxyV1TestData[] = [ { error: '{"error":{"message":"Service Unavailable","description":"The server is currently unable to handle the request due to temporary overloading or maintenance of the server. Please try again later."}}', - statusCode: 500, + statusCode: 503, metadata: { jobId: 1, attemptNum: 1, @@ -287,7 +287,7 @@ export const otherScenariosV1: ProxyV1TestData[] = [ }, message: 'Campaign Manager: Error transformer proxy v1 during CAMPAIGN_MANAGER response transformation', - status: 500, + status: 503, }, }, }, @@ -376,7 +376,7 @@ export const otherScenariosV1: ProxyV1TestData[] = [ response: [ { error: '"Gateway Timeout"', - statusCode: 500, + statusCode: 504, metadata: { jobId: 1, attemptNum: 1, @@ -403,7 +403,7 @@ export const otherScenariosV1: ProxyV1TestData[] = [ }, message: 'Campaign Manager: Error transformer proxy v1 during CAMPAIGN_MANAGER response transformation', - status: 500, + status: 504, }, }, }, From 5dd5142a5d55a5eca5517c75d5356df416308afc Mon Sep 17 00:00:00 2001 From: shrouti1507 <60211312+shrouti1507@users.noreply.github.com> Date: Tue, 12 Mar 2024 17:56:56 +0530 Subject: [PATCH 2/4] chore: adding proxy test cases for salesforce oauth (#3170) * chore: add initial business tests * chore: cleanup * chore: add other scenario test, refactor * chore: address commentsx1 * chore: move test to other * chore: address commentsx2 * chore: adding proxy test cases for salesforce oauth access token refresh * code review suggestion Co-authored-by: Yashasvi Bajpai <33063622+yashasvibajpai@users.noreply.github.com> * fix: review comments address --------- Co-authored-by: Yashasvi Bajpai <33063622+yashasvibajpai@users.noreply.github.com> --- .../salesforce_oauth/dataDelivery/data.ts | 3 + .../salesforce_oauth/dataDelivery/oauth.ts | 174 ++++++++++++++++++ .../destinations/salesforce_oauth/network.ts | 51 +++++ 3 files changed, 228 insertions(+) create mode 100644 test/integrations/destinations/salesforce_oauth/dataDelivery/data.ts create mode 100644 test/integrations/destinations/salesforce_oauth/dataDelivery/oauth.ts create mode 100644 test/integrations/destinations/salesforce_oauth/network.ts diff --git a/test/integrations/destinations/salesforce_oauth/dataDelivery/data.ts b/test/integrations/destinations/salesforce_oauth/dataDelivery/data.ts new file mode 100644 index 0000000000..bed8eec8db --- /dev/null +++ b/test/integrations/destinations/salesforce_oauth/dataDelivery/data.ts @@ -0,0 +1,3 @@ +import { testScenariosForV1API } from './oauth'; + +export const data = [...testScenariosForV1API]; diff --git a/test/integrations/destinations/salesforce_oauth/dataDelivery/oauth.ts b/test/integrations/destinations/salesforce_oauth/dataDelivery/oauth.ts new file mode 100644 index 0000000000..55eaa9cca1 --- /dev/null +++ b/test/integrations/destinations/salesforce_oauth/dataDelivery/oauth.ts @@ -0,0 +1,174 @@ +import { ProxyMetdata } from '../../../../../src/types'; +import { ProxyV1TestData } from '../../../testTypes'; +import { generateProxyV1Payload } from '../../../testUtils'; + +const commonHeadersForWrongToken = { + Authorization: 'Bearer expiredAccessToken', + 'Content-Type': 'application/json', +}; + +const commonHeadersForRightToken = { + Authorization: 'Bearer correctAccessToken', + 'Content-Type': 'application/json', +}; +const params = { destination: 'salesforce_oauth' }; + +const users = [ + { + Email: 'danis.archurav@sbermarket.ru', + Company: 'itus.ru', + LastName: 'Danis', + FirstName: 'Archurav', + LeadSource: 'App Signup', + account_type__c: 'free_trial', + }, +]; + +const statTags = { + retryable: { + destType: 'SALESFORCE_OAUTH', + destinationId: 'dummyDestinationId', + errorCategory: 'network', + errorType: 'retryable', + feature: 'dataDelivery', + implementation: 'native', + module: 'destination', + workspaceId: 'dummyWorkspaceId', + }, +}; + +const commonRequestParametersWithWrongToken = { + headers: commonHeadersForWrongToken, + JSON: users[0], + params, +}; + +const commonRequestParametersWithRightToken = { + headers: commonHeadersForRightToken, + JSON: users[0], + params, +}; + +export const proxyMetdataWithSecretWithWrongAccessToken: ProxyMetdata = { + jobId: 1, + attemptNum: 1, + userId: 'dummyUserId', + sourceId: 'dummySourceId', + destinationId: 'dummyDestinationId', + workspaceId: 'dummyWorkspaceId', + secret: { + access_token: 'expiredAccessToken', + instanceUrl: 'https://rudderstack.my.salesforce_oauth.com', + }, + destInfo: { authKey: 'dummyDestinationId' }, + dontBatch: false, +}; + +export const proxyMetdataWithSecretWithRightAccessToken: ProxyMetdata = { + jobId: 1, + attemptNum: 1, + userId: 'dummyUserId', + sourceId: 'dummySourceId', + destinationId: 'dummyDestinationId', + workspaceId: 'dummyWorkspaceId', + secret: { + access_token: 'expiredRightToken', + instanceUrl: 'https://rudderstack.my.salesforce_oauth.com', + }, + destInfo: { authKey: 'dummyDestinationId' }, + dontBatch: false, +}; + +export const reqMetadataArrayWithWrongSecret = [proxyMetdataWithSecretWithWrongAccessToken]; +export const reqMetadataArray = [proxyMetdataWithSecretWithRightAccessToken]; + +export const testScenariosForV1API: ProxyV1TestData[] = [ + { + id: 'salesforce_v1_scenario_1', + name: 'salesforce_oauth', + description: '[Proxy v1 API] :: Test with expired access token scenario', + successCriteria: + 'Should return 5XX with error category REFRESH_TOKEN and Session expired or invalid, INVALID_SESSION_ID', + scenario: 'Business', + feature: 'dataDelivery', + module: 'destination', + version: 'v1', + input: { + request: { + body: generateProxyV1Payload( + { + ...commonRequestParametersWithWrongToken, + endpoint: + 'https://rudderstack.my.salesforce_oauth.com/services/data/v50.0/sobjects/Lead/20', + }, + reqMetadataArrayWithWrongSecret, + ), + method: 'POST', + }, + }, + output: { + response: { + status: 500, + body: { + output: { + status: 500, + authErrorCategory: 'REFRESH_TOKEN', + message: + 'Salesforce Request Failed - due to "INVALID_SESSION_ID", (Retryable) during Salesforce Response Handling', + response: [ + { + error: + '[{"message":"Session expired or invalid","errorCode":"INVALID_SESSION_ID"}]', + metadata: proxyMetdataWithSecretWithWrongAccessToken, + statusCode: 500, + }, + ], + statTags: statTags.retryable, + }, + }, + }, + }, + }, + { + id: 'salesforce_v1_scenario_2', + name: 'salesforce', + description: + '[Proxy v1 API] :: Test for a valid request - Lead creation with existing unchanged leadId and unchanged data', + successCriteria: 'Should return 200 with no error with destination response', + scenario: 'Business', + feature: 'dataDelivery', + module: 'destination', + version: 'v1', + input: { + request: { + body: generateProxyV1Payload( + { + ...commonRequestParametersWithRightToken, + endpoint: + 'https://rudderstack.my.salesforce.com/services/data/v50.0/sobjects/Lead/existing_unchanged_leadId', + }, + reqMetadataArray, + ), + method: 'POST', + }, + }, + output: { + response: { + status: 200, + body: { + output: { + status: 200, + message: 'Request for destination: salesforce Processed Successfully', + response: [ + { + error: '{"statusText":"No Content"}', + metadata: proxyMetdataWithSecretWithRightAccessToken, + statusCode: 200, + }, + ], + }, + }, + }, + }, + }, +]; diff --git a/test/integrations/destinations/salesforce_oauth/network.ts b/test/integrations/destinations/salesforce_oauth/network.ts new file mode 100644 index 0000000000..ae5f9d3fe4 --- /dev/null +++ b/test/integrations/destinations/salesforce_oauth/network.ts @@ -0,0 +1,51 @@ +const headerWithWrongAccessToken = { + Authorization: 'Bearer expiredAccessToken', + 'Content-Type': 'application/json', +}; + +const headerWithRightAccessToken = { + Authorization: 'Bearer correctAccessToken', + 'Content-Type': 'application/json', +}; + +const dataValue = { + Email: 'danis.archurav@sbermarket.ru', + Company: 'itus.ru', + LastName: 'Danis', + FirstName: 'Archurav', + LeadSource: 'App Signup', + account_type__c: 'free_trial', +}; + +const businessMockData = [ + { + description: 'Mock response from destination depicting an expired access token', + httpReq: { + method: 'post', + url: 'https://rudderstack.my.salesforce_oauth.com/services/data/v50.0/sobjects/Lead/20', + headers: headerWithWrongAccessToken, + data: dataValue, + params: { destination: 'salesforce_oauth' }, + }, + httpRes: { + data: [{ message: 'Session expired or invalid', errorCode: 'INVALID_SESSION_ID' }], + status: 401, + }, + }, + { + description: + 'Mock response from destination depicting a valid lead request, with no changed data', + httpReq: { + method: 'post', + url: 'https://rudderstack.my.salesforce.com/services/data/v50.0/sobjects/Lead/existing_unchanged_leadId', + data: dataValue, + headers: headerWithRightAccessToken, + }, + httpRes: { + data: { statusText: 'No Content' }, + status: 204, + }, + }, +]; + +export const networkCallsData = [...businessMockData]; From d2eba21191dc4f7b610414158af68e5533016014 Mon Sep 17 00:00:00 2001 From: Sankeerth Date: Tue, 12 Mar 2024 22:26:07 +0530 Subject: [PATCH 3/4] fix: upload js coverage to codecov (#3179) --- .github/workflows/dt-test-and-report-code-coverage.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/dt-test-and-report-code-coverage.yml b/.github/workflows/dt-test-and-report-code-coverage.yml index 51a9f8c9ee..4375b3383e 100644 --- a/.github/workflows/dt-test-and-report-code-coverage.yml +++ b/.github/workflows/dt-test-and-report-code-coverage.yml @@ -41,6 +41,8 @@ jobs: - name: Upload Coverage Reports to Codecov uses: codecov/codecov-action@v4.0.1 + env: + CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} with: directory: ./reports/coverage From 67fcdd3f4e8c60b545e7bf7164dde0d1df7b6e2c Mon Sep 17 00:00:00 2001 From: Sandeep Digumarty Date: Wed, 13 Mar 2024 07:33:22 +0530 Subject: [PATCH 4/4] chore: resolve code injection vulnerabilities (#3164) --- src/controllers/bulkUpload.ts | 10 +++---- src/util/fetchDestinationHandlers.ts | 42 ++++++++++++++++++++++++++++ 2 files changed, 47 insertions(+), 5 deletions(-) create mode 100644 src/util/fetchDestinationHandlers.ts diff --git a/src/controllers/bulkUpload.ts b/src/controllers/bulkUpload.ts index babb8b6db1..dbd77dc07f 100644 --- a/src/controllers/bulkUpload.ts +++ b/src/controllers/bulkUpload.ts @@ -1,14 +1,14 @@ /* eslint-disable global-require, import/no-dynamic-require, @typescript-eslint/no-unused-vars */ import { client as errNotificationClient } from '../util/errorNotifier'; import logger from '../logger'; +import { + getJobStatusHandler, + getPollStatusHandler, + getDestFileUploadHandler, +} from '../util/fetchDestinationHandlers'; import { CatchErr, ContextBodySimple } from '../util/types'; // TODO: To be refactored and redisgned -const getDestFileUploadHandler = (version, dest) => - require(`../${version}/destinations/${dest}/fileUpload`); -const getPollStatusHandler = (version, dest) => require(`../${version}/destinations/${dest}/poll`); -const getJobStatusHandler = (version, dest) => - require(`../${version}/destinations/${dest}/fetchJobStatus`); const ERROR_MESSAGE_PROCESSOR_STRING = 'Error occurred while processing payload.'; const getCommonMetadata = (ctx) => diff --git a/src/util/fetchDestinationHandlers.ts b/src/util/fetchDestinationHandlers.ts new file mode 100644 index 0000000000..2661ef2e68 --- /dev/null +++ b/src/util/fetchDestinationHandlers.ts @@ -0,0 +1,42 @@ +import * as V0MarketoBulkUploadFileUpload from '../v0/destinations/marketo_bulk_upload/fileUpload'; +import * as V0MarketoBulkUploadPollStatus from '../v0/destinations/marketo_bulk_upload/poll'; +import * as V0MarketoBulkUploadJobStatus from '../v0/destinations/marketo_bulk_upload/fetchJobStatus'; + +const fileUploadHandlers = { + v0: { + marketo_bulk_upload: V0MarketoBulkUploadFileUpload, + }, +}; + +const pollStatusHandlers = { + v0: { + marketo_bulk_upload: V0MarketoBulkUploadPollStatus, + }, +}; + +const jobStatusHandlers = { + v0: { + marketo_bulk_upload: V0MarketoBulkUploadJobStatus, + }, +}; + +export const getDestFileUploadHandler = (version, dest) => { + if (fileUploadHandlers[version] && fileUploadHandlers[version][dest]) { + return fileUploadHandlers[version][dest]; + } + return undefined; +}; + +export const getPollStatusHandler = (version, dest) => { + if (pollStatusHandlers[version] && pollStatusHandlers[version][dest]) { + return pollStatusHandlers[version][dest]; + } + return undefined; +}; + +export const getJobStatusHandler = (version, dest) => { + if (jobStatusHandlers[version] && jobStatusHandlers[version][dest]) { + return jobStatusHandlers[version][dest]; + } + return undefined; +};