diff --git a/.github/workflows/build-push-docker-image.yml b/.github/workflows/build-push-docker-image.yml index 7ddae0a3ae..0d3494e8d1 100644 --- a/.github/workflows/build-push-docker-image.yml +++ b/.github/workflows/build-push-docker-image.yml @@ -38,6 +38,7 @@ jobs: - name: Checkout uses: actions/checkout@v4.1.1 with: + ref: ${{ github.event.pull_request.head.sha }} fetch-depth: 1 - name: Setup Docker Buildx @@ -88,6 +89,7 @@ jobs: - name: Checkout uses: actions/checkout@v4.1.1 with: + ref: ${{ github.event.pull_request.head.sha }} fetch-depth: 1 - name: Setup Docker Buildx diff --git a/CHANGELOG.md b/CHANGELOG.md index 9c04835cd4..3ba14c6e92 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,27 @@ All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines. +## [1.68.0](https://github.com/rudderlabs/rudder-transformer/compare/v1.67.0...v1.68.0) (2024-05-27) + + +### Features + +* add json-data type support in redis ([#3336](https://github.com/rudderlabs/rudder-transformer/issues/3336)) ([0196f20](https://github.com/rudderlabs/rudder-transformer/commit/0196f20cc79e1f470d96a649dd9404c3c9284329)) +* facebook custom audience app secret support ([#3357](https://github.com/rudderlabs/rudder-transformer/issues/3357)) ([fce4ef9](https://github.com/rudderlabs/rudder-transformer/commit/fce4ef973500411c7ad812e7949bb1b73dabc3ba)) +* filtering unknown events in awin ([#3392](https://github.com/rudderlabs/rudder-transformer/issues/3392)) ([d842da8](https://github.com/rudderlabs/rudder-transformer/commit/d842da87a34cb63023eba288e0c5258e29997dcf)) +* **ga4:** component test refactor ([#3220](https://github.com/rudderlabs/rudder-transformer/issues/3220)) ([3ff9a5e](https://github.com/rudderlabs/rudder-transformer/commit/3ff9a5e8e955b929a1b04a89dcf0ccbc49e18648)) +* **integrations/auth0:** include Auth0 event type in Rudderstack message ([#3370](https://github.com/rudderlabs/rudder-transformer/issues/3370)) ([e9409fd](https://github.com/rudderlabs/rudder-transformer/commit/e9409fde6063d7eaa8558396b85b5fdf99f964e1)) +* onboard koddi destination ([#3359](https://github.com/rudderlabs/rudder-transformer/issues/3359)) ([f74c4a0](https://github.com/rudderlabs/rudder-transformer/commit/f74c4a0bc92ae6ccb0c00ac5b21745e496a015bc)) +* onboarding adjust source ([#3395](https://github.com/rudderlabs/rudder-transformer/issues/3395)) ([668d331](https://github.com/rudderlabs/rudder-transformer/commit/668d3311aadacbb92b1873bf43919db7d341afbb)) + + +### Bug Fixes + +* fb custom audience html response ([#3402](https://github.com/rudderlabs/rudder-transformer/issues/3402)) ([d1a2bd6](https://github.com/rudder +* standardise hashing for all CAPI integrations ([#3379](https://github.com/rudderlabs/rudder-transformer/issues/3379)) ([c249a69](https://github.com/rudderlabs/rudder-transformer/commit/c249a694d735f6d241a35b6e21f493c54890ac84)) +* tiktok_v2 remove default value for content-type for custom events ([#3383](https://github.com/rudderlabs/rudder-transformer/issues/3383)) ([6e7b5a0](https://github.com/rudderlabs/rudder-transformer/commit/6e7b5a0d8bf2c859dfb15b9cad7ed6070bd0892b)) +* added step for reconciling openfaas functions for python transformations ([#3420](https://github.com/rudderlabs/rudder-transformer/issues/3420)) ([7a2ab63](https://github.com/rudderlabs/rudder-transformer/commit/7a2ab63674d40870af4d16f0673a2a2594c899e9)) + ## [1.67.0](https://github.com/rudderlabs/rudder-transformer/compare/v1.66.1...v1.67.0) (2024-05-23) diff --git a/package-lock.json b/package-lock.json index bfeb00963a..b8e2c81a29 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "rudder-transformer", - "version": "1.67.0", + "version": "1.68.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "rudder-transformer", - "version": "1.67.0", + "version": "1.68.0", "license": "ISC", "dependencies": { "@amplitude/ua-parser-js": "0.7.24", diff --git a/package.json b/package.json index 7fa3a7330c..ed15683d4f 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "rudder-transformer", - "version": "1.67.0", + "version": "1.68.0", "description": "", "homepage": "https://github.com/rudderlabs/rudder-transformer#readme", "bugs": { diff --git a/src/cdk/v2/destinations/koddi/config.js b/src/cdk/v2/destinations/koddi/config.js new file mode 100644 index 0000000000..927e1858fc --- /dev/null +++ b/src/cdk/v2/destinations/koddi/config.js @@ -0,0 +1,39 @@ +const { getMappingConfig } = require('../../../../v0/util'); + +/** + * ref :- https://developers.koddi.com/reference/winning-ads + * impressions - https://developers.koddi.com/reference/impressions-1 + * clicks - https://developers.koddi.com/reference/clicks-1 + * conversions - https://developers.koddi.com/reference/conversions-1 + */ +const EVENT_TYPES = { + IMPRESSIONS: 'impressions', + CLICKS: 'clicks', + CONVERSIONS: 'conversions', +}; + +const CONFIG_CATEGORIES = { + IMPRESSIONS: { + type: 'track', + name: 'ImpressionsConfig', + }, + CLICKS: { + type: 'track', + name: 'ClicksConfig', + }, + CONVERSIONS: { + type: 'track', + name: 'ConversionsConfig', + }, +}; + +const MAPPING_CONFIG = getMappingConfig(CONFIG_CATEGORIES, __dirname); + +module.exports = { + EVENT_TYPES, + CONFIG_CATEGORIES, + MAPPING_CONFIG, + IMPRESSIONS_CONFIG: MAPPING_CONFIG[CONFIG_CATEGORIES.IMPRESSIONS.name], + CLICKS_CONFIG: MAPPING_CONFIG[CONFIG_CATEGORIES.CLICKS.name], + CONVERSIONS_CONFIG: MAPPING_CONFIG[CONFIG_CATEGORIES.CONVERSIONS.name], +}; diff --git a/src/cdk/v2/destinations/koddi/data/ClicksConfig.json b/src/cdk/v2/destinations/koddi/data/ClicksConfig.json new file mode 100644 index 0000000000..96ab27b2ae --- /dev/null +++ b/src/cdk/v2/destinations/koddi/data/ClicksConfig.json @@ -0,0 +1,35 @@ +[ + { + "sourceKeys": "properties.tracking_data", + "required": true, + "destKey": "trackingData" + }, + { + "sourceKeys": "properties.rank", + "required": true, + "destKey": "rank" + }, + { + "sourceKeys": "properties.beacon_issued", + "required": true, + "destKey": "beaconIssued" + }, + { + "sourceKeys": "userId", + "sourceFromGenericMap": true, + "required": true, + "destKey": "userGuid" + }, + { + "sourceKeys": "properties.test_version_override", + "destKey": "testVersionOverride" + }, + { + "sourceKeys": "properties.destination_url", + "destKey": "destinationUrl" + }, + { + "sourceKeys": "properties.overrides", + "destKey": "overrides" + } +] diff --git a/src/cdk/v2/destinations/koddi/data/ConversionsConfig.json b/src/cdk/v2/destinations/koddi/data/ConversionsConfig.json new file mode 100644 index 0000000000..495574f198 --- /dev/null +++ b/src/cdk/v2/destinations/koddi/data/ConversionsConfig.json @@ -0,0 +1,53 @@ +[ + { + "sourceKeys": "context.page.referring_domain", + "destKey": "domain" + }, + { + "sourceKeys": "context.locale", + "required": true, + "destKey": "culture" + }, + { + "sourceKeys": "properties.currency", + "required": true, + "destKey": "currency" + }, + { + "sourceKeys": ["context.ip", "request_ip"], + "destKey": "user_ip" + }, + { + "sourceKeys": "context.userAgent", + "destKey": "user_agent" + }, + { + "sourceKeys": "userId", + "sourceFromGenericMap": true, + "required": true, + "destKey": "user_guid" + }, + { + "sourceKeys": "context.device.type", + "destKey": "device_type" + }, + { + "sourceKeys": ["properties.order_id", "properties.transaction_id"], + "required": true, + "destKey": "transaction_id" + }, + { + "sourceKeys": "properties.conversion_source", + "destKey": "conversion_source" + }, + { + "sourceKeys": "timestamp", + "sourceFromGenericMap": true, + "destKey": "unixtime" + }, + { + "sourceKeys": "properties.bidders", + "required": true, + "destKey": "bidders" + } +] diff --git a/src/cdk/v2/destinations/koddi/data/ImpressionsConfig.json b/src/cdk/v2/destinations/koddi/data/ImpressionsConfig.json new file mode 100644 index 0000000000..de53703b32 --- /dev/null +++ b/src/cdk/v2/destinations/koddi/data/ImpressionsConfig.json @@ -0,0 +1,22 @@ +[ + { + "sourceKeys": "properties.tracking_data", + "required": true, + "destKey": "trackingData" + }, + { + "sourceKeys": "properties.rank", + "required": true, + "destKey": "rank" + }, + { + "sourceKeys": "properties.beacon_issued", + "required": true, + "destKey": "beaconIssued" + }, + { + "sourceKeys": "timestamp", + "sourceFromGenericMap": true, + "destKey": "ts" + } +] diff --git a/src/cdk/v2/destinations/koddi/procWorkflow.yaml b/src/cdk/v2/destinations/koddi/procWorkflow.yaml new file mode 100644 index 0000000000..cc3f0166dc --- /dev/null +++ b/src/cdk/v2/destinations/koddi/procWorkflow.yaml @@ -0,0 +1,33 @@ +bindings: + - name: EventType + path: ../../../../constants + - path: ../../bindings/jsontemplate + - name: removeUndefinedAndNullValues + path: ../../../../v0/util + - path: ./utils + - path: ./config + +steps: + - name: messageType + template: | + .message.type.toLowerCase(); + - name: eventType + template: | + .message.integrations.koddi.eventType.toLowerCase(); + - name: validateInput + template: | + let messageType = $.outputs.messageType; + let eventType = $.outputs.eventType; + $.assert(messageType, "message Type is not present. Aborting message."); + $.assert(messageType in {{$.EventType.([.TRACK])}}, "message type " + messageType + " is not supported"); + $.assert(eventType in {{$.EVENT_TYPES.([.IMPRESSIONS, .CLICKS, .CONVERSIONS])}}, "event type " + eventType + " is not supported"); + $.assertConfig(.destination.Config.apiBaseUrl, "API Base URL is not present. Aborting"); + $.assertConfig(.destination.Config.clientName, "Client Name is not present. Aborting"); + - name: preparePayload + template: | + const payload = $.constructFullPayload($.outputs.eventType, .message, .destination.Config); + $.context.payload = $.removeUndefinedAndNullValues(payload); + - name: buildResponse + template: | + const response = $.constructResponse($.outputs.eventType, .destination.Config, $.context.payload); + response diff --git a/src/cdk/v2/destinations/koddi/rtWorkflow.yaml b/src/cdk/v2/destinations/koddi/rtWorkflow.yaml new file mode 100644 index 0000000000..dd438a911c --- /dev/null +++ b/src/cdk/v2/destinations/koddi/rtWorkflow.yaml @@ -0,0 +1,31 @@ +bindings: + - name: handleRtTfSingleEventError + path: ../../../../v0/util/index + +steps: + - name: validateInput + template: | + $.assert(Array.isArray(^) && ^.length > 0, "Invalid event array") + + - name: transform + externalWorkflow: + path: ./procWorkflow.yaml + loopOverInput: true + + - name: successfulEvents + template: | + $.outputs.transform#idx.output.({ + "batchedRequest": ., + "batched": false, + "destination": ^[idx].destination, + "metadata": ^[idx].metadata[], + "statusCode": 200 + })[] + - name: failedEvents + template: | + $.outputs.transform#idx.error.( + $.handleRtTfSingleEventError(^[idx], .originalError ?? ., {}) + )[] + - name: finalPayload + template: | + [...$.outputs.successfulEvents, ...$.outputs.failedEvents] diff --git a/src/cdk/v2/destinations/koddi/utils.js b/src/cdk/v2/destinations/koddi/utils.js new file mode 100644 index 0000000000..13014e2e7c --- /dev/null +++ b/src/cdk/v2/destinations/koddi/utils.js @@ -0,0 +1,116 @@ +const { InstrumentationError } = require('@rudderstack/integrations-lib'); +const { EVENT_TYPES, IMPRESSIONS_CONFIG, CLICKS_CONFIG, CONVERSIONS_CONFIG } = require('./config'); +const { + constructPayload, + defaultRequestConfig, + toUnixTimestamp, + stripTrailingSlash, +} = require('../../../../v0/util'); + +const validateBidders = (bidders) => { + if (!Array.isArray(bidders)) { + throw new InstrumentationError('properties.bidders should be an array of objects. Aborting.'); + } + if (bidders.length === 0) { + throw new InstrumentationError( + 'properties.bidders should contains at least one bidder. Aborting.', + ); + } + bidders.forEach((bidder) => { + if (!(bidder.bidder || bidder.alternate_bidder)) { + throw new InstrumentationError('bidder or alternate_bidder is not present. Aborting.'); + } + if (!bidder.count) { + throw new InstrumentationError('count is not present. Aborting.'); + } + if (!bidder.base_price) { + throw new InstrumentationError('base_price is not present. Aborting.'); + } + }); +}; + +/** + * This function constructs payloads based upon mappingConfig for all calls. + * @param {*} eventType + * @param {*} message + * @param {*} Config + * @returns + */ +const constructFullPayload = (eventType, message, Config) => { + let payload; + switch (eventType) { + case EVENT_TYPES.IMPRESSIONS: + payload = constructPayload(message, IMPRESSIONS_CONFIG); + payload.clientName = Config.clientName; + break; + case EVENT_TYPES.CLICKS: + payload = constructPayload(message, CLICKS_CONFIG); + payload.clientName = Config.clientName; + if (!Config.testVersionOverride) { + payload.testVersionOverride = null; + } + if (!Config.overrides) { + payload.overrides = null; + } + break; + case EVENT_TYPES.CONVERSIONS: + payload = constructPayload(message, CONVERSIONS_CONFIG); + payload.client_name = Config.clientName; + payload.unixtime = toUnixTimestamp(payload.unixtime); + validateBidders(payload.bidders); + break; + default: + throw new InstrumentationError(`event type ${eventType} is not supported.`); + } + return payload; +}; + +const getEndpoint = (eventType, Config) => { + let endpoint = stripTrailingSlash(Config.apiBaseUrl); + switch (eventType) { + case EVENT_TYPES.IMPRESSIONS: + endpoint += '?action=impression'; + break; + case EVENT_TYPES.CLICKS: + endpoint += '?action=click'; + break; + case EVENT_TYPES.CONVERSIONS: + endpoint += '/conversion'; + break; + default: + throw new InstrumentationError(`event type ${eventType} is not supported.`); + } + return endpoint; +}; + +/** + * This function constructs response based upon event. + * @param {*} eventType + * @param {*} Config + * @param {*} payload + * @returns + */ +const constructResponse = (eventType, Config, payload) => { + if (!Object.values(EVENT_TYPES).includes(eventType)) { + throw new InstrumentationError(`event type ${eventType} is not supported.`); + } + const response = defaultRequestConfig(); + response.endpoint = getEndpoint(eventType, Config); + response.headers = { + accept: 'application/json', + }; + if (eventType === EVENT_TYPES.CONVERSIONS) { + response.body.JSON = payload; + response.method = 'POST'; + response.headers = { + ...response.headers, + 'content-type': 'application/json', + }; + } else { + response.params = payload; + response.method = 'GET'; + } + return response; +}; + +module.exports = { getEndpoint, validateBidders, constructFullPayload, constructResponse }; diff --git a/src/cdk/v2/destinations/koddi/utils.test.js b/src/cdk/v2/destinations/koddi/utils.test.js new file mode 100644 index 0000000000..2c1f660f70 --- /dev/null +++ b/src/cdk/v2/destinations/koddi/utils.test.js @@ -0,0 +1,421 @@ +const { + getEndpoint, + validateBidders, + constructFullPayload, + constructResponse, +} = require('./utils'); +const { InstrumentationError } = require('@rudderstack/integrations-lib'); + +describe('getEndpoint', () => { + it('returns the correct endpoint for IMPRESSIONS event', () => { + const eventType = 'impressions'; + const Config = { + apiBaseUrl: 'https://www.test-client.com/', + clientName: 'test-client', + }; + const result = getEndpoint(eventType, Config); + expect(result).toEqual('https://www.test-client.com?action=impression'); + }); + + it('returns the correct endpoint for CLICKS event', () => { + const eventType = 'clicks'; + const Config = { + apiBaseUrl: 'https://www.test-client.com', + clientName: 'test-client', + }; + const result = getEndpoint(eventType, Config); + expect(result).toEqual('https://www.test-client.com?action=click'); + }); + + it('returns the correct endpoint for IMPRESSIONS event', () => { + const eventType = 'conversions'; + const Config = { + apiBaseUrl: 'https://www.test-client.com', + clientName: 'test-client', + }; + const result = getEndpoint(eventType, Config); + expect(result).toEqual('https://www.test-client.com/conversion'); + }); + + it('should throw error for unsupported event', () => { + const eventType = 'test'; + const Config = { + apiBaseUrl: 'https://www.test-client.com', + clientName: 'test-client', + }; + expect(() => getEndpoint(eventType, Config)).toThrow(InstrumentationError); + expect(() => getEndpoint(eventType, Config)).toThrow('event type test is not supported.'); + }); +}); + +describe('validateBidders', () => { + it('should throw error if bidders is not an array', () => { + const bidders = {}; + expect(() => validateBidders(bidders)).toThrow(InstrumentationError); + expect(() => validateBidders(bidders)).toThrow( + 'properties.bidders should be an array of objects. Aborting.', + ); + }); + + it('should throw error if bidders is an empty array', () => { + const bidders = []; + expect(() => validateBidders(bidders)).toThrow(InstrumentationError); + expect(() => validateBidders(bidders)).toThrow( + 'properties.bidders should contains at least one bidder. Aborting.', + ); + }); + + it('should throw error if bidder or alternate_bidder is not present', () => { + const bidders = [ + { count: 1, base_price: 100 }, + { bidder: 'bidder1', count: 2, base_price: 200 }, + { alternate_bidder: 'alternate1', count: 3, base_price: 300 }, + ]; + expect(() => validateBidders(bidders)).toThrow(InstrumentationError); + expect(() => validateBidders(bidders)).toThrow( + 'bidder or alternate_bidder is not present. Aborting.', + ); + }); + + it('should throw error if count is not present', () => { + const bidders = [{ bidder: 'bidder1', alternate_bidder: 'alternate1', base_price: 100 }]; + expect(() => validateBidders(bidders)).toThrow(InstrumentationError); + expect(() => validateBidders(bidders)).toThrow('count is not present. Aborting.'); + }); + + it('should throw error if base_price is not present', () => { + const bidders = [{ bidder: 'bidder1', alternate_bidder: 'alternate1', count: 1 }]; + expect(() => validateBidders(bidders)).toThrow(InstrumentationError); + expect(() => validateBidders(bidders)).toThrow('base_price is not present. Aborting.'); + }); + + it('should not throw error if all required fields are present for all bidders', () => { + const bidders = [ + { bidder: 'bidder1', alternate_bidder: 'alternate1', count: 1, base_price: 100 }, + { bidder: 'bidder2', alternate_bidder: 'alternate2', count: 2, base_price: 200 }, + ]; + expect(() => validateBidders(bidders)).not.toThrow(); + }); +}); + +describe('constructFullPayload', () => { + it('should construct payload for IMPRESSIONS event', () => { + const eventType = 'impressions'; + const message = { + type: 'track', + event: 'Impressions Event', + properties: { + tracking_data: 'dummy-tracking-data', + rank: 1, + beacon_issued: '2024-03-04T15:32:56.409Z', + }, + timestamp: '2024-03-03T00:29:12.117+05:30', + }; + const Config = { + apiBaseUrl: 'https://www.test-client.com', + clientName: 'test-client', + }; + const expectedPayload = { + beaconIssued: '2024-03-04T15:32:56.409Z', + clientName: 'test-client', + rank: 1, + trackingData: 'dummy-tracking-data', + ts: '2024-03-03T00:29:12.117+05:30', + }; + const payload = constructFullPayload(eventType, message, Config); + expect(payload).toEqual(expectedPayload); + }); + it('should throw error if required value is missing for IMPRESSIONS event', () => { + const eventType = 'impressions'; + const message = { + type: 'track', + event: 'Impressions Event', + properties: { + tracking_data: '', + rank: 1, + beacon_issued: '2024-03-04T15:32:56.409Z', + }, + timestamp: '2024-03-03T00:29:12.117+05:30', + }; + const Config = { + apiBaseUrl: 'https://www.test-client.com', + clientName: 'test-client', + }; + try { + const payload = constructFullPayload(eventType, message, Config); + } catch (error) { + expect(error.message).toEqual('Missing required value from "properties.tracking_data"'); + } + }); + + it('should construct payload for CLICKS event', () => { + const eventType = 'clicks'; + const message = { + type: 'track', + event: 'Clicks Event', + properties: { + tracking_data: 'dummy-tracking-data', + rank: 1, + beacon_issued: '2024-03-04T15:32:56.409Z', + }, + anonymousId: '1234', + }; + const Config = { + apiBaseUrl: 'https://www.test-client.com', + clientName: 'test-client', + }; + const expectedPayload = { + beaconIssued: '2024-03-04T15:32:56.409Z', + clientName: 'test-client', + rank: 1, + trackingData: 'dummy-tracking-data', + userGuid: '1234', + overrides: null, + testVersionOverride: null, + }; + const payload = constructFullPayload(eventType, message, Config); + expect(payload).toEqual(expectedPayload); + }); + it('should construct payload with non-null value if overrides and testVersionOverride are enable and values for these are provided for CLICKS event ', () => { + const eventType = 'clicks'; + const message = { + type: 'track', + event: 'Clicks Event', + properties: { + tracking_data: 'dummy-tracking-data', + rank: 1, + beacon_issued: '2024-03-04T15:32:56.409Z', + overrides: 'overridden-value', + testVersionOverride: 1, + }, + anonymousId: '1234', + }; + const Config = { + apiBaseUrl: 'https://www.test-client.com', + clientName: 'test-client', + overrides: true, + testVersionOverride: false, + }; + const expectedPayload = { + beaconIssued: '2024-03-04T15:32:56.409Z', + clientName: 'test-client', + rank: 1, + trackingData: 'dummy-tracking-data', + userGuid: '1234', + overrides: 'overridden-value', + testVersionOverride: null, + }; + const payload = constructFullPayload(eventType, message, Config); + expect(payload).toEqual(expectedPayload); + }); + it('should throw error if required value is missing for CLICKS event', () => { + const eventType = 'clicks'; + const message = { + type: 'track', + event: 'Clicks Event', + properties: { + tracking_data: 'dummy-tracking-data', + rank: 1, + beacon_issued: '2024-03-04T15:32:56.409Z', + }, + }; + const Config = { + apiBaseUrl: 'https://www.test-client.com', + clientName: 'test-client', + }; + try { + const payload = constructFullPayload(eventType, message, Config); + } catch (error) { + expect(error.message).toEqual('Missing required value from "userId"'); + } + }); + + it('should construct payload for CONVERSIONS event', () => { + const eventType = 'conversions'; + const message = { + type: 'track', + event: 'Conversions Event', + properties: { + currency: 'USD', + order_id: '123', + bidders: [ + { + bidder: 'dummy-bidder-id', + count: 1, + base_price: 100.1, + }, + ], + }, + context: { + locale: 'en-US', + ip: '127.0.0.1', + }, + timestamp: '2024-03-03T00:29:12.117+05:30', + anonymousId: '1234', + }; + const Config = { + apiBaseUrl: 'https://www.test-client.com', + clientName: 'test-client', + }; + const expectedPayload = { + client_name: 'test-client', + culture: 'en-US', + currency: 'USD', + transaction_id: '123', + unixtime: 1709405952, + user_guid: '1234', + user_ip: '127.0.0.1', + bidders: [ + { + bidder: 'dummy-bidder-id', + count: 1, + base_price: 100.1, + }, + ], + }; + const payload = constructFullPayload(eventType, message, Config); + expect(payload).toEqual(expectedPayload); + }); + it('should throw error if required value is missing for CONVERSIONS event', () => { + const eventType = 'conversions'; + const message = { + type: 'track', + event: 'Conversions Event', + properties: { + currency: 'USD', + order_id: '123', + bidders: [ + { + bidder: 'dummy-bidder-id', + count: 1, + base_price: 100.1, + }, + ], + }, + context: { + ip: '127.0.0.1', + }, + timestamp: '2024-03-03T00:29:12.117+05:30', + anonymousId: '1234', + }; + const Config = { + apiBaseUrl: 'https://www.test-client.com', + clientName: 'test-client', + }; + try { + const payload = constructFullPayload(eventType, message, Config); + } catch (error) { + expect(error.message).toEqual('Missing required value from "context.locale"'); + } + }); + + it('should throw error for unsupported event', () => { + const eventType = 'test'; + const message = {}; + const Config = {}; + expect(() => constructFullPayload(eventType, message, Config)).toThrow(InstrumentationError); + expect(() => constructFullPayload(eventType, message, Config)).toThrow( + 'event type test is not supported.', + ); + }); +}); + +describe('constructResponse', () => { + it('should construct response for IMPRESSIONS event', () => { + const eventType = 'impressions'; + const Config = { + apiBaseUrl: 'https://www.test-client.com', + clientName: 'test-client', + }; + const payload = { + beaconIssued: '2024-03-04T15:32:56.409Z', + clientName: 'test-client', + rank: 1, + trackingData: 'dummy-tracking-data', + ts: '2024-03-03T00:29:12.117+05:30', + }; + const expectedResponse = { + endpoint: 'https://www.test-client.com?action=impression', + headers: { + accept: 'application/json', + }, + method: 'GET', + params: payload, + }; + const response = constructResponse(eventType, Config, payload); + expect(response).toMatchObject(expectedResponse); + }); + + it('should construct response for CLICKS event', () => { + const eventType = 'clicks'; + const Config = { + apiBaseUrl: 'https://www.test-client.com', + clientName: 'test-client', + }; + const payload = { + beaconIssued: '2024-03-04T15:32:56.409Z', + clientName: 'test-client', + rank: 1, + trackingData: 'dummy-tracking-data', + userGuid: '1234', + }; + const expectedResponse = { + endpoint: 'https://www.test-client.com?action=click', + headers: { + accept: 'application/json', + }, + method: 'GET', + params: payload, + }; + const response = constructResponse(eventType, Config, payload); + expect(response).toMatchObject(expectedResponse); + }); + + it('should construct response for CONVERSIONS event', () => { + const eventType = 'conversions'; + const Config = { + apiBaseUrl: 'https://www.test-client.com', + clientName: 'test-client', + }; + const payload = { + client_name: 'test-client', + culture: 'en-US', + currency: 'USD', + transaction_id: '123', + unixtime: 1709405952, + userGuid: '1234', + user_ip: '127.0.0.1', + bidders: [ + { + bidder: 'dummy-bidder-id', + count: 1, + base_price: 100.1, + }, + ], + }; + + const expectedResponse = { + endpoint: 'https://www.test-client.com/conversion', + headers: { + accept: 'application/json', + 'content-type': 'application/json', + }, + method: 'POST', + body: { + JSON: payload, + }, + }; + const response = constructResponse(eventType, Config, payload); + expect(response).toMatchObject(expectedResponse); + }); + + it('should throw error for unsupported event', () => { + const eventType = 'test'; + const Config = {}; + const payload = {}; + expect(() => constructResponse(eventType, Config, payload)).toThrow(InstrumentationError); + expect(() => constructResponse(eventType, Config, payload)).toThrow( + 'event type test is not supported.', + ); + }); +}); diff --git a/src/cdk/v2/destinations/reddit/procWorkflow.yaml b/src/cdk/v2/destinations/reddit/procWorkflow.yaml index 7b989f15e4..06d2c95f25 100644 --- a/src/cdk/v2/destinations/reddit/procWorkflow.yaml +++ b/src/cdk/v2/destinations/reddit/procWorkflow.yaml @@ -36,13 +36,13 @@ steps: const os = (.message.context.os.name)? .message.context.os.name.toLowerCase(): null; const hashData = .destination.Config.hashData; let user = .message.().({ - "email": hashData ? $.SHA256({{{{$.getGenericPaths("email")}}}}) : ({{{{$.getGenericPaths("email")}}}}), - "external_id": hashData ? $.SHA256({{{{$.getGenericPaths("userId")}}}}) : ({{{{$.getGenericPaths("userId")}}}}), - "ip_address": hashData? $.SHA256(.context.ip || .request_ip) : (.context.ip || .request_ip), + "email": hashData ? $.SHA256({{{{$.getGenericPaths("email")}}}}.trim()) : ({{{{$.getGenericPaths("email")}}}}), + "external_id": hashData ? $.SHA256({{{{$.getGenericPaths("userId")}}}}.trim()) : ({{{{$.getGenericPaths("userId")}}}}), + "ip_address": hashData? $.SHA256(.context.ip.trim() || .request_ip.trim()) : (.context.ip || .request_ip), "uuid": .properties.uuid, "user_agent": .context.userAgent, - "idfa": $.isAppleFamily(os)? (hashData? $.SHA256(.context.device.advertisingId): .context.device.advertisingId): null, - "aaid": os === "android" && .context.device ? (hashData? $.SHA256(.context.device.advertisingId): .context.device.advertisingId): null, + "idfa": $.isAppleFamily(os)? (hashData? $.SHA256(.context.device.advertisingId.trim()): .context.device.advertisingId): null, + "aaid": os === "android" && .context.device ? (hashData? $.SHA256(.context.device.advertisingId.trim()): .context.device.advertisingId): null, "opt_out": .properties.optOut, "screen_dimensions": {"width": .context.screen.width, "height": .context.screen.height}, }); diff --git a/src/features.json b/src/features.json index a7e4b70109..58af795a77 100644 --- a/src/features.json +++ b/src/features.json @@ -71,7 +71,8 @@ "LINKEDIN_ADS": true, "BLOOMREACH": true, "MOVABLE_INK": true, - "EMARSYS": true + "EMARSYS": true, + "KODDI": true }, "regulations": [ "BRAZE", diff --git a/src/util/customTransformer-faas.js b/src/util/customTransformer-faas.js index 2c0bbfd8c0..9ac9804097 100644 --- a/src/util/customTransformer-faas.js +++ b/src/util/customTransformer-faas.js @@ -11,9 +11,10 @@ const { } = require('./openfaas'); const { getLibraryCodeV1 } = require('./customTransforrmationsStore-v1'); +const HASH_SECRET = process.env.OPENFAAS_FN_HASH_SECRET || ''; const libVersionIdsCache = new NodeCache(); -function generateFunctionName(userTransformation, libraryVersionIds, testMode) { +function generateFunctionName(userTransformation, libraryVersionIds, testMode, hashSecret = '') { if (userTransformation.versionId === FAAS_AST_VID) return FAAS_AST_FN_NAME; if (testMode) { @@ -21,10 +22,15 @@ function generateFunctionName(userTransformation, libraryVersionIds, testMode) { return funcName.substring(0, 63).toLowerCase(); } - const ids = [userTransformation.workspaceId, userTransformation.versionId].concat( + let ids = [userTransformation.workspaceId, userTransformation.versionId].concat( (libraryVersionIds || []).sort(), ); + if (hashSecret !== '') { + ids = ids.concat([hashSecret]); + } + + // FIXME: Why the id's are sorted ?! const hash = crypto.createHash('md5').update(`${ids}`).digest('hex'); return `fn-${userTransformation.workspaceId}-${hash}`.substring(0, 63).toLowerCase(); } @@ -90,7 +96,13 @@ async function setOpenFaasUserTransform( testMode, }; const functionName = - pregeneratedFnName || generateFunctionName(userTransformation, libraryVersionIds, testMode); + pregeneratedFnName || + generateFunctionName( + userTransformation, + libraryVersionIds, + testMode, + process.env.OPENFAAS_FN_HASH_SECRET, + ); const setupTime = new Date(); await setupFaasFunction( @@ -130,7 +142,13 @@ async function runOpenFaasUserTransform( const trMetadata = events[0].metadata ? getTransformationMetadata(events[0].metadata) : {}; // check and deploy faas function if not exists - const functionName = generateFunctionName(userTransformation, libraryVersionIds, testMode); + const functionName = generateFunctionName( + userTransformation, + libraryVersionIds, + testMode, + process.env.OPENFAAS_FN_HASH_SECRET, + ); + if (testMode) { await setOpenFaasUserTransform( userTransformation, diff --git a/src/util/openfaas/faasApi.js b/src/util/openfaas/faasApi.js index f8f830f6e4..b932b70032 100644 --- a/src/util/openfaas/faasApi.js +++ b/src/util/openfaas/faasApi.js @@ -1,6 +1,8 @@ const axios = require('axios'); const { RespStatusError, RetryRequestError } = require('../utils'); +const logger = require('../../logger'); + const OPENFAAS_GATEWAY_URL = process.env.OPENFAAS_GATEWAY_URL || 'http://localhost:8080'; const OPENFAAS_GATEWAY_USERNAME = process.env.OPENFAAS_GATEWAY_USERNAME || ''; const OPENFAAS_GATEWAY_PASSWORD = process.env.OPENFAAS_GATEWAY_PASSWORD || ''; @@ -12,7 +14,7 @@ const basicAuth = { const parseAxiosError = (error) => { if (error.response) { - const status = error.response.status || 400; + const status = error.response.status || 500; const errorData = error.response?.data; const message = (errorData && (errorData.message || errorData.error || errorData)) || error.message; @@ -61,6 +63,8 @@ const invokeFunction = async (functionName, payload) => }); const checkFunctionHealth = async (functionName) => { + logger.debug(`Checking function health: ${functionName}`); + return new Promise((resolve, reject) => { const url = `${OPENFAAS_GATEWAY_URL}/function/${functionName}`; axios @@ -76,8 +80,10 @@ const checkFunctionHealth = async (functionName) => { }); }; -const deployFunction = async (payload) => - new Promise((resolve, reject) => { +const deployFunction = async (payload) => { + logger.debug(`Deploying function: ${payload?.name}`); + + return new Promise((resolve, reject) => { const url = `${OPENFAAS_GATEWAY_URL}/system/functions`; axios .post(url, payload, { auth: basicAuth }) @@ -86,6 +92,21 @@ const deployFunction = async (payload) => reject(parseAxiosError(err)); }); }); +}; + +const updateFunction = async (fnName, payload) => { + logger.debug(`Updating function: ${fnName}`); + + return new Promise((resolve, reject) => { + const url = `${OPENFAAS_GATEWAY_URL}/system/functions`; + axios + .put(url, payload, { auth: basicAuth }) + .then((resp) => resolve(resp.data)) + .catch((err) => { + reject(parseAxiosError(err)); + }); + }); +}; module.exports = { deleteFunction, @@ -94,4 +115,5 @@ module.exports = { getFunctionList, invokeFunction, checkFunctionHealth, + updateFunction, }; diff --git a/src/util/openfaas/index.js b/src/util/openfaas/index.js index 3cf3525e6f..c0369deb81 100644 --- a/src/util/openfaas/index.js +++ b/src/util/openfaas/index.js @@ -4,6 +4,7 @@ const { deployFunction, invokeFunction, checkFunctionHealth, + updateFunction, } = require('./faasApi'); const logger = require('../../logger'); const { RetryRequestError, RespStatusError } = require('../utils'); @@ -33,6 +34,7 @@ const FAAS_AST_FN_NAME = 'fn-ast'; const CUSTOM_NETWORK_POLICY_WORKSPACE_IDS = process.env.CUSTOM_NETWORK_POLICY_WORKSPACE_IDS || ''; const customNetworkPolicyWorkspaceIds = CUSTOM_NETWORK_POLICY_WORKSPACE_IDS.split(','); const CUSTOMER_TIER = process.env.CUSTOMER_TIER || 'shared'; +const DISABLE_RECONCILE_FN = process.env.DISABLE_RECONCILE_FN == 'true' || false; // Initialise node cache const functionListCache = new NodeCache(); @@ -67,6 +69,8 @@ const awaitFunctionReadiness = async ( maxWaitInMs = 22000, waitBetweenIntervalsInMs = 250, ) => { + logger.debug(`Awaiting function readiness: ${functionName}`); + const executionPromise = new Promise(async (resolve) => { try { await callWithRetry( @@ -121,7 +125,7 @@ const invalidateFnCache = () => { functionListCache.set(FUNC_LIST_KEY, []); }; -const deployFaasFunction = async ( +const updateFaasFunction = async ( functionName, code, versionId, @@ -130,73 +134,50 @@ const deployFaasFunction = async ( trMetadata = {}, ) => { try { - logger.debug(`[Faas] Deploying a faas function: ${functionName}`); - let envProcess = 'python index.py'; - - const lvidsString = libraryVersionIDs.join(','); + logger.debug(`Updating faas fn: ${functionName}`); - if (!testMode) { - envProcess = `${envProcess} --vid ${versionId} --config-backend-url ${CONFIG_BACKEND_URL} --lvids "${lvidsString}"`; - } else { - envProcess = `${envProcess} --code "${code}" --config-backend-url ${CONFIG_BACKEND_URL} --lvids "${lvidsString}"`; - } - - const envVars = {}; - if (FAAS_ENABLE_WATCHDOG_ENV_VARS.trim().toLowerCase() === 'true') { - envVars.max_inflight = FAAS_MAX_INFLIGHT; - envVars.exec_timeout = FAAS_EXEC_TIMEOUT; - } - if (GEOLOCATION_URL) { - envVars.geolocation_url = GEOLOCATION_URL; - } - // labels - const labels = { - 'openfaas-fn': 'true', - 'parent-component': 'openfaas', - 'com.openfaas.scale.max': FAAS_MAX_PODS_IN_TEXT, - 'com.openfaas.scale.min': FAAS_MIN_PODS_IN_TEXT, - 'com.openfaas.scale.zero': FAAS_SCALE_ZERO, - 'com.openfaas.scale.zero-duration': FAAS_SCALE_ZERO_DURATION, - 'com.openfaas.scale.target': FAAS_SCALE_TARGET, - 'com.openfaas.scale.target-proportion': FAAS_SCALE_TARGET_PROPORTION, - 'com.openfaas.scale.type': FAAS_SCALE_TYPE, - transformationId: trMetadata.transformationId, - workspaceId: trMetadata.workspaceId, - team: 'data-management', - service: 'openfaas-fn', - customer: 'shared', - 'customer-tier': CUSTOMER_TIER, - }; - if ( - trMetadata.workspaceId && - customNetworkPolicyWorkspaceIds.includes(trMetadata.workspaceId) - ) { - labels['custom-network-policy'] = 'true'; + const payload = buildOpenfaasFn( + functionName, + code, + versionId, + libraryVersionIDs, + testMode, + trMetadata, + ); + await updateFunction(functionName, payload); + // wait for function to be ready and then set it in cache + await awaitFunctionReadiness(functionName); + setFunctionInCache(functionName); + } catch (error) { + // 404 is statuscode returned from openfaas community edition + // when the function don't exist, so we can safely ignore this error + // and let the function be created in the next step. + if (error.statusCode !== 404) { + throw error; } + } +}; - // TODO: investigate and add more required labels and annotations - const payload = { - service: functionName, - name: functionName, - image: FAAS_BASE_IMG, - envProcess, - envVars, - labels, - annotations: { - 'prometheus.io.scrape': 'true', - }, - limits: { - memory: FAAS_LIMITS_MEMORY, - cpu: FAAS_LIMITS_CPU, - }, - requests: { - memory: FAAS_REQUESTS_MEMORY, - cpu: FAAS_REQUESTS_CPU, - }, - }; +const deployFaasFunction = async ( + functionName, + code, + versionId, + libraryVersionIDs, + testMode, + trMetadata = {}, +) => { + try { + logger.debug(`Deploying faas fn: ${functionName}`); + const payload = buildOpenfaasFn( + functionName, + code, + versionId, + libraryVersionIDs, + testMode, + trMetadata, + ); await deployFunction(payload); - logger.debug('[Faas] Deployed a faas function'); } catch (error) { logger.error(`[Faas] Error while deploying ${functionName}: ${error.message}`); // To handle concurrent create requests, @@ -246,6 +227,95 @@ async function setupFaasFunction( } } +// reconcileFn runs everytime the service boot's up +// trying to update the functions which are not in cache to the +// latest label and envVars +const reconcileFn = async (name, versionId, libraryVersionIDs, trMetadata) => { + if (DISABLE_RECONCILE_FN) { + return; + } + + logger.debug(`Reconciling faas function: ${name}`); + try { + if (isFunctionDeployed(name)) { + return; + } + await updateFaasFunction(name, null, versionId, libraryVersionIDs, false, trMetadata); + } catch (error) { + logger.error( + `unexpected error occurred when reconciling the function ${name}: ${error.message}`, + ); + throw error; + } +}; + +// buildOpenfaasFn is helper function to build openfaas fn CRUD payload +function buildOpenfaasFn(name, code, versionId, libraryVersionIDs, testMode, trMetadata = {}) { + logger.debug(`Building faas fn: ${name}`); + + let envProcess = 'python index.py'; + const lvidsString = libraryVersionIDs.join(','); + + if (!testMode) { + envProcess = `${envProcess} --vid ${versionId} --config-backend-url ${CONFIG_BACKEND_URL} --lvids "${lvidsString}"`; + } else { + envProcess = `${envProcess} --code "${code}" --config-backend-url ${CONFIG_BACKEND_URL} --lvids "${lvidsString}"`; + } + + const envVars = {}; + + if (FAAS_ENABLE_WATCHDOG_ENV_VARS.trim().toLowerCase() === 'true') { + envVars.max_inflight = FAAS_MAX_INFLIGHT; + envVars.exec_timeout = FAAS_EXEC_TIMEOUT; + } + + if (GEOLOCATION_URL) { + envVars.geolocation_url = GEOLOCATION_URL; + } + + const labels = { + 'openfaas-fn': 'true', + 'parent-component': 'openfaas', + 'com.openfaas.scale.max': FAAS_MAX_PODS_IN_TEXT, + 'com.openfaas.scale.min': FAAS_MIN_PODS_IN_TEXT, + 'com.openfaas.scale.zero': FAAS_SCALE_ZERO, + 'com.openfaas.scale.zero-duration': FAAS_SCALE_ZERO_DURATION, + 'com.openfaas.scale.target': FAAS_SCALE_TARGET, + 'com.openfaas.scale.target-proportion': FAAS_SCALE_TARGET_PROPORTION, + 'com.openfaas.scale.type': FAAS_SCALE_TYPE, + transformationId: trMetadata.transformationId, + workspaceId: trMetadata.workspaceId, + team: 'data-management', + service: 'openfaas-fn', + customer: 'shared', + 'customer-tier': CUSTOMER_TIER, + }; + + if (trMetadata.workspaceId && customNetworkPolicyWorkspaceIds.includes(trMetadata.workspaceId)) { + labels['custom-network-policy'] = 'true'; + } + + return { + service: name, + name: name, + image: FAAS_BASE_IMG, + envProcess, + envVars, + labels, + annotations: { + 'prometheus.io.scrape': 'true', + }, + limits: { + memory: FAAS_LIMITS_MEMORY, + cpu: FAAS_LIMITS_CPU, + }, + requests: { + memory: FAAS_REQUESTS_MEMORY, + cpu: FAAS_REQUESTS_CPU, + }, + }; +} + const executeFaasFunction = async ( name, events, @@ -260,7 +330,11 @@ const executeFaasFunction = async ( let errorRaised; try { - if (testMode) await awaitFunctionReadiness(name); + if (testMode) { + await awaitFunctionReadiness(name); + } else { + await reconcileFn(name, versionId, libraryVersionIDs, trMetadata); + } return await invokeFunction(name, events); } catch (error) { logger.error(`Error while invoking ${name}: ${error.message}`); @@ -268,6 +342,7 @@ const executeFaasFunction = async ( if (error.statusCode === 404 && error.message.includes(`error finding function ${name}`)) { removeFunctionFromCache(name); + await setupFaasFunction(name, null, versionId, libraryVersionIDs, testMode, trMetadata); throw new RetryRequestError(`${name} not found`); } @@ -314,6 +389,8 @@ module.exports = { executeFaasFunction, setupFaasFunction, invalidateFnCache, + buildOpenfaasFn, FAAS_AST_VID, FAAS_AST_FN_NAME, + setFunctionInCache, }; diff --git a/src/util/prometheus.js b/src/util/prometheus.js index a618d35068..78d32c9cb9 100644 --- a/src/util/prometheus.js +++ b/src/util/prometheus.js @@ -613,12 +613,6 @@ class Prometheus { type: 'gauge', labelNames: ['destination_id'], }, - { - name: 'mixpanel_batch_track_pack_size', - help: 'mixpanel_batch_track_pack_size', - type: 'gauge', - labelNames: ['destination_id'], - }, { name: 'mixpanel_batch_import_pack_size', help: 'mixpanel_batch_import_pack_size', diff --git a/src/v0/destinations/awin/transform.js b/src/v0/destinations/awin/transform.js index 68dd9d62e1..0e1e220548 100644 --- a/src/v0/destinations/awin/transform.js +++ b/src/v0/destinations/awin/transform.js @@ -3,6 +3,7 @@ const { BASE_URL, ConfigCategory, mappingConfig } = require('./config'); const { defaultRequestConfig, constructPayload, simpleProcessRouterDest } = require('../../util'); const { getParams, trackProduct, populateCustomTransactionProperties } = require('./utils'); +const { FilteredEventsError } = require('../../util/errorTypes'); const responseBuilder = (message, { Config }) => { const { advertiserId, eventsToTrack, customFieldMap } = Config; @@ -33,9 +34,9 @@ const responseBuilder = (message, { Config }) => { ...customTransactionProperties, }; } else { - throw new InstrumentationError( - "Event is not present in 'Events to Track' list. Aborting message.", - 400, + throw new FilteredEventsError( + "Event is not present in 'Events to Track' list. Dropping the event.", + 298, ); } } diff --git a/src/v0/destinations/facebook_offline_conversions/utils.js b/src/v0/destinations/facebook_offline_conversions/utils.js index c48de4e0b9..460ef71176 100644 --- a/src/v0/destinations/facebook_offline_conversions/utils.js +++ b/src/v0/destinations/facebook_offline_conversions/utils.js @@ -396,7 +396,7 @@ const preparePayload = (facebookOfflineConversionsPayload, destination) => { const keys = Object.keys(facebookOfflineConversionsPayload); keys.forEach((key) => { if (isHashRequired && HASHING_REQUIRED_KEYS.includes(key)) { - payload[key] = sha256(facebookOfflineConversionsPayload[key]); + payload[key] = sha256(facebookOfflineConversionsPayload[key].trim()); } else { payload[key] = facebookOfflineConversionsPayload[key]; } @@ -407,8 +407,8 @@ const preparePayload = (facebookOfflineConversionsPayload, destination) => { ? facebookOfflineConversionsPayload.name.split(' ') : null; if (split !== null && Array.isArray(split) && split.length === 2) { - payload.fn = isHashRequired ? sha256(split[0]) : split[0]; - payload.ln = isHashRequired ? sha256(split[1]) : split[1]; + payload.fn = isHashRequired ? sha256(split[0].trim()) : split[0]; + payload.ln = isHashRequired ? sha256(split[1].trim()) : split[1]; } delete payload.name; } diff --git a/src/v0/destinations/fb/transform.js b/src/v0/destinations/fb/transform.js index e6f8e986cf..1160cef407 100644 --- a/src/v0/destinations/fb/transform.js +++ b/src/v0/destinations/fb/transform.js @@ -90,7 +90,7 @@ function sanityCheckPayloadForTypesAndModifications(updatedEvent) { clonedUpdatedEvent[prop] !== '' ) { isUDSet = true; - clonedUpdatedEvent[prop] = sha256(clonedUpdatedEvent[prop].toLowerCase()); + clonedUpdatedEvent[prop] = sha256(clonedUpdatedEvent[prop].trim().toLowerCase()); } break; case 'ud[zp]': @@ -113,7 +113,7 @@ function sanityCheckPayloadForTypesAndModifications(updatedEvent) { } else { isUDSet = true; clonedUpdatedEvent[prop] = sha256( - clonedUpdatedEvent[prop].toLowerCase() === 'female' ? 'f' : 'm', + clonedUpdatedEvent[prop].trim().toLowerCase() === 'female' ? 'f' : 'm', ); } } @@ -128,7 +128,7 @@ function sanityCheckPayloadForTypesAndModifications(updatedEvent) { if (clonedUpdatedEvent[prop] && clonedUpdatedEvent[prop] !== '') { isUDSet = true; clonedUpdatedEvent[prop] = sha256( - clonedUpdatedEvent[prop].toLowerCase().replace(/ /g, ''), + clonedUpdatedEvent[prop].trim().toLowerCase().replace(/ /g, ''), ); } break; diff --git a/src/v0/destinations/fb_custom_audience/transform.js b/src/v0/destinations/fb_custom_audience/transform.js index 9320a3476b..dfe9a04618 100644 --- a/src/v0/destinations/fb_custom_audience/transform.js +++ b/src/v0/destinations/fb_custom_audience/transform.js @@ -20,6 +20,7 @@ const { prepareDataField, getSchemaForEventMappedToDest, batchingWithPayloadSize, + generateAppSecretProof, } = require('./util'); const { getEndPoint, @@ -88,7 +89,7 @@ const prepareResponse = ( userSchema, isHashRequired = true, ) => { - const { accessToken, disableFormat, type, subType, isRaw } = destination.Config; + const { accessToken, disableFormat, type, subType, isRaw, appSecret } = destination.Config; const mappedToDestination = get(message, MappedToDestinationKey); @@ -105,6 +106,12 @@ const prepareResponse = ( prepareParams.access_token = accessToken; + if (isDefinedAndNotNullAndNotEmpty(appSecret)) { + const dateNow = Date.now(); + prepareParams.appsecret_time = Math.floor(dateNow / 1000); // Get current Unix time in seconds + prepareParams.appsecret_proof = generateAppSecretProof(accessToken, appSecret, dateNow); + } + // creating the payload field for parameters if (isRaw) { paramsPayload.is_raw = isRaw; diff --git a/src/v0/destinations/fb_custom_audience/util.js b/src/v0/destinations/fb_custom_audience/util.js index 1f096215f3..9385dfbd36 100644 --- a/src/v0/destinations/fb_custom_audience/util.js +++ b/src/v0/destinations/fb_custom_audience/util.js @@ -1,5 +1,6 @@ const lodash = require('lodash'); const sha256 = require('sha256'); +const crypto = require('crypto'); const get = require('get-value'); const jsonSize = require('json-size'); const { InstrumentationError, ConfigurationError } = require('@rudderstack/integrations-lib'); @@ -206,4 +207,23 @@ const prepareDataField = ( return data; }; -module.exports = { prepareDataField, getSchemaForEventMappedToDest, batchingWithPayloadSize }; +// ref: https://developers.facebook.com/docs/facebook-login/security/#generate-the-proof + +const generateAppSecretProof = (accessToken, appSecret, dateNow) => { + const currentTime = Math.floor(dateNow / 1000); // Get current Unix time in seconds + const data = `${accessToken}|${currentTime}`; + + // Creating a HMAC SHA-256 hash with the app_secret as the key + const hmac = crypto.createHmac('sha256', appSecret); + hmac.update(data); + const appsecretProof = hmac.digest('hex'); + + return appsecretProof; +}; + +module.exports = { + prepareDataField, + getSchemaForEventMappedToDest, + batchingWithPayloadSize, + generateAppSecretProof, +}; diff --git a/src/v0/destinations/google_adwords_enhanced_conversions/data/trackConfig.json b/src/v0/destinations/google_adwords_enhanced_conversions/data/trackConfig.json index bf5485270b..c38b24598d 100644 --- a/src/v0/destinations/google_adwords_enhanced_conversions/data/trackConfig.json +++ b/src/v0/destinations/google_adwords_enhanced_conversions/data/trackConfig.json @@ -55,7 +55,7 @@ "sourceFromGenericMap": true, "required": false, "metadata": { - "type": "hashToSha256" + "type": ["trim", "hashToSha256"] } }, { @@ -64,7 +64,7 @@ "sourceFromGenericMap": true, "required": false, "metadata": { - "type": "hashToSha256" + "type": ["trim", "hashToSha256"] } }, { @@ -73,7 +73,7 @@ "sourceFromGenericMap": true, "required": false, "metadata": { - "type": "hashToSha256" + "type": ["trim", "hashToSha256"] } }, { @@ -82,7 +82,7 @@ "sourceFromGenericMap": true, "required": false, "metadata": { - "type": "hashToSha256" + "type": ["trim", "hashToSha256"] } }, { @@ -127,7 +127,7 @@ "sourceKeys": ["context.traits.streetAddress", "context.traits.address"], "required": false, "metadata": { - "type": "hashToSha256" + "type": ["trim", "hashToSha256"] } } ] diff --git a/src/v0/destinations/google_adwords_enhanced_conversions/transform.js b/src/v0/destinations/google_adwords_enhanced_conversions/transform.js index 0be7c3f0ee..55d0c16c8c 100644 --- a/src/v0/destinations/google_adwords_enhanced_conversions/transform.js +++ b/src/v0/destinations/google_adwords_enhanced_conversions/transform.js @@ -24,7 +24,7 @@ const { JSON_MIME_TYPE } = require('../../util/constant'); const updateMappingJson = (mapping) => { const newMapping = []; mapping.forEach((element) => { - if (get(element, 'metadata.type') && element.metadata.type === 'hashToSha256') { + if (get(element, 'metadata.type') && element.metadata.type.includes('hashToSha256')) { element.metadata.type = 'toString'; } newMapping.push(element); diff --git a/src/v0/destinations/google_adwords_offline_conversions/utils.js b/src/v0/destinations/google_adwords_offline_conversions/utils.js index 70b42e2157..dfa892a769 100644 --- a/src/v0/destinations/google_adwords_offline_conversions/utils.js +++ b/src/v0/destinations/google_adwords_offline_conversions/utils.js @@ -140,17 +140,17 @@ const buildAndGetAddress = (message, hashUserIdentifier) => { const address = constructPayload(message, trackAddStoreAddressConversionsMapping); if (address.hashed_last_name) { address.hashed_last_name = hashUserIdentifier - ? sha256(address.hashed_last_name).toString() + ? sha256(address.hashed_last_name.trim()).toString() : address.hashed_last_name; } if (address.hashed_first_name) { address.hashed_first_name = hashUserIdentifier - ? sha256(address.hashed_first_name).toString() + ? sha256(address.hashed_first_name.trim()).toString() : address.hashed_first_name; } if (address.hashed_street_address) { address.hashed_street_address = hashUserIdentifier - ? sha256(address.hashed_street_address).toString() + ? sha256(address.hashed_street_address.trim()).toString() : address.hashed_street_address; } return Object.keys(address).length > 0 ? address : null; @@ -269,8 +269,10 @@ const getAddConversionPayload = (message, Config) => { const phone = getFieldValueFromMessage(message, 'phone'); const userIdentifierInfo = { - email: hashUserIdentifier && isDefinedAndNotNull(email) ? sha256(email).toString() : email, - phone: hashUserIdentifier && isDefinedAndNotNull(phone) ? sha256(phone).toString() : phone, + email: + hashUserIdentifier && isDefinedAndNotNull(email) ? sha256(email.trim()).toString() : email, + phone: + hashUserIdentifier && isDefinedAndNotNull(phone) ? sha256(phone.trim()).toString() : phone, address: buildAndGetAddress(message, hashUserIdentifier), }; @@ -363,8 +365,10 @@ const getClickConversionPayloadAndEndpoint = ( // Ref - https://developers.google.com/google-ads/api/rest/reference/rest/v11/customers/uploadClickConversions#ClickConversion const userIdentifierInfo = { - email: hashUserIdentifier && isDefinedAndNotNull(email) ? sha256(email).toString() : email, - phone: hashUserIdentifier && isDefinedAndNotNull(phone) ? sha256(phone).toString() : phone, + email: + hashUserIdentifier && isDefinedAndNotNull(email) ? sha256(email.trim()).toString() : email, + phone: + hashUserIdentifier && isDefinedAndNotNull(phone) ? sha256(phone.trim()).toString() : phone, }; const keyName = getExisitingUserIdentifier(userIdentifierInfo, defaultUserIdentifier); diff --git a/src/v0/destinations/impact/transform.js b/src/v0/destinations/impact/transform.js index 2eefdf7992..729f988938 100644 --- a/src/v0/destinations/impact/transform.js +++ b/src/v0/destinations/impact/transform.js @@ -59,7 +59,7 @@ const buildPageLoadPayload = (message, campaignId, impactAppId, enableEmailHashi let payload = constructPayload(message, MAPPING_CONFIG[CONFIG_CATEGORIES.PAGELOAD.name]); if (isDefinedAndNotNull(payload.CustomerEmail)) { payload.CustomerEmail = enableEmailHashing - ? sha1(payload?.CustomerEmail) + ? sha1(payload?.CustomerEmail.trim()) : payload?.CustomerEmail; } payload.CampaignId = campaignId; @@ -155,7 +155,7 @@ const processTrackEvent = (message, Config) => { payload.ImpactAppId = impactAppId; if (isDefinedAndNotNull(payload.CustomerEmail)) { payload.CustomerEmail = enableEmailHashing - ? sha1(payload?.CustomerEmail) + ? sha1(payload?.CustomerEmail.trim()) : payload?.CustomerEmail; } diff --git a/src/v0/destinations/mp/config.js b/src/v0/destinations/mp/config.js index 35b40294f5..3abdf2eebb 100644 --- a/src/v0/destinations/mp/config.js +++ b/src/v0/destinations/mp/config.js @@ -49,7 +49,6 @@ const MP_IDENTIFY_EXCLUSION_LIST = [ ]; const GEO_SOURCE_ALLOWED_VALUES = [null, 'reverse_geocoding']; -const TRACK_MAX_BATCH_SIZE = 50; const IMPORT_MAX_BATCH_SIZE = 2000; const ENGAGE_MAX_BATCH_SIZE = 2000; const GROUPS_MAX_BATCH_SIZE = 200; @@ -68,7 +67,6 @@ module.exports = { MP_IDENTIFY_EXCLUSION_LIST, getCreateDeletionTaskEndpoint, DISTINCT_ID_MAX_BATCH_SIZE, - TRACK_MAX_BATCH_SIZE, IMPORT_MAX_BATCH_SIZE, ENGAGE_MAX_BATCH_SIZE, GROUPS_MAX_BATCH_SIZE, diff --git a/src/v0/destinations/mp/transform.js b/src/v0/destinations/mp/transform.js index 09a7862f9a..2065764b98 100644 --- a/src/v0/destinations/mp/transform.js +++ b/src/v0/destinations/mp/transform.js @@ -24,7 +24,6 @@ const { mappingConfig, BASE_ENDPOINT, BASE_ENDPOINT_EU, - TRACK_MAX_BATCH_SIZE, IMPORT_MAX_BATCH_SIZE, ENGAGE_MAX_BATCH_SIZE, GROUPS_MAX_BATCH_SIZE, @@ -47,21 +46,19 @@ const mPEventPropertiesConfigJson = mappingConfig[ConfigCategory.EVENT_PROPERTIE const setImportCredentials = (destConfig) => { const endpoint = destConfig.dataResidency === 'eu' ? `${BASE_ENDPOINT_EU}/import/` : `${BASE_ENDPOINT}/import/`; - const headers = { 'Content-Type': 'application/json' }; const params = { strict: destConfig.strictMode ? 1 : 0 }; - const { apiSecret, serviceAccountUserName, serviceAccountSecret, projectId } = destConfig; - if (apiSecret) { - headers.Authorization = `Basic ${base64Convertor(`${apiSecret}:`)}`; + const { serviceAccountUserName, serviceAccountSecret, projectId, token } = destConfig; + let credentials; + if (token) { + credentials = `${token}:`; } else if (serviceAccountUserName && serviceAccountSecret && projectId) { - headers.Authorization = `Basic ${base64Convertor( - `${serviceAccountUserName}:${serviceAccountSecret}`, - )}`; + credentials = `${serviceAccountUserName}:${serviceAccountSecret}`; params.projectId = projectId; - } else { - throw new InstrumentationError( - 'Event timestamp is older than 5 days and no API secret or service account credentials (i.e. username, secret and projectId) are provided in destination configuration', - ); } + const headers = { + 'Content-Type': 'application/json', + Authorization: `Basic ${base64Convertor(credentials)}`, + }; return { endpoint, headers, params }; }; @@ -70,46 +67,34 @@ const responseBuilderSimple = (payload, message, eventType, destConfig) => { response.method = defaultPostRequestConfig.requestMethod; response.userId = message.userId || message.anonymousId; response.body.JSON_ARRAY = { batch: JSON.stringify([removeUndefinedValues(payload)]) }; - const { apiSecret, serviceAccountUserName, serviceAccountSecret, projectId, dataResidency } = - destConfig; + const { dataResidency } = destConfig; const duration = getTimeDifference(message.timestamp); + + const setCredentials = () => { + const credentials = setImportCredentials(destConfig); + response.endpoint = credentials.endpoint; + response.headers = credentials.headers; + response.params = { + project_id: credentials.params?.projectId, + strict: credentials.params.strict, + }; + }; + switch (eventType) { case EventType.ALIAS: case EventType.TRACK: case EventType.SCREEN: - case EventType.PAGE: - if ( - !apiSecret && - !(serviceAccountUserName && serviceAccountSecret && projectId) && - duration.days <= 5 - ) { - response.endpoint = - dataResidency === 'eu' ? `${BASE_ENDPOINT_EU}/track/` : `${BASE_ENDPOINT}/track/`; - response.headers = {}; - } else if (duration.years > 5) { + case EventType.PAGE: { + if (duration.years > 5) { throw new InstrumentationError('Event timestamp should be within last 5 years'); - } else { - const credentials = setImportCredentials(destConfig); - response.endpoint = credentials.endpoint; - response.headers = credentials.headers; - response.params = { - project_id: credentials.params?.projectId, - strict: credentials.params.strict, - }; - break; } + setCredentials(); break; - case 'merge': - // eslint-disable-next-line no-case-declarations - const credentials = setImportCredentials(destConfig); - response.endpoint = credentials.endpoint; - response.headers = credentials.headers; - response.params = { - project_id: credentials.params?.projectId, - strict: credentials.params.strict, - }; + } + case 'merge': { + setCredentials(); break; - + } default: response.endpoint = dataResidency === 'eu' ? `${BASE_ENDPOINT_EU}/engage/` : `${BASE_ENDPOINT}/engage/`; @@ -484,7 +469,6 @@ const processRouterDest = async (inputs, reqMetadata) => { const batchSize = { engage: 0, groups: 0, - track: 0, import: 0, }; @@ -516,23 +500,16 @@ const processRouterDest = async (inputs, reqMetadata) => { ); transformedPayloads = lodash.flatMap(transformedPayloads); - const { engageEvents, groupsEvents, trackEvents, importEvents, batchErrorRespList } = + const { engageEvents, groupsEvents, importEvents, batchErrorRespList } = groupEventsByEndpoint(transformedPayloads); const engageRespList = batchEvents(engageEvents, ENGAGE_MAX_BATCH_SIZE, reqMetadata); const groupsRespList = batchEvents(groupsEvents, GROUPS_MAX_BATCH_SIZE, reqMetadata); - const trackRespList = batchEvents(trackEvents, TRACK_MAX_BATCH_SIZE, reqMetadata); const importRespList = batchEvents(importEvents, IMPORT_MAX_BATCH_SIZE, reqMetadata); - const batchSuccessRespList = [ - ...engageRespList, - ...groupsRespList, - ...trackRespList, - ...importRespList, - ]; + const batchSuccessRespList = [...engageRespList, ...groupsRespList, ...importRespList]; batchSize.engage += engageRespList.length; batchSize.groups += groupsRespList.length; - batchSize.track += trackRespList.length; batchSize.import += importRespList.length; return [...batchSuccessRespList, ...batchErrorRespList]; diff --git a/src/v0/destinations/mp/util.js b/src/v0/destinations/mp/util.js index d564e805ad..b2807d6e11 100644 --- a/src/v0/destinations/mp/util.js +++ b/src/v0/destinations/mp/util.js @@ -136,7 +136,7 @@ const createIdentifyResponse = (message, type, destination, responseBuilderSimpl * @returns */ const isImportAuthCredentialsAvailable = (destination) => - destination.Config.apiSecret || + destination.Config.token || (destination.Config.serviceAccountSecret && destination.Config.serviceAccountUserName && destination.Config.projectId); @@ -179,7 +179,6 @@ const groupEventsByEndpoint = (events) => { const eventMap = { engage: [], groups: [], - track: [], import: [], }; const batchErrorRespList = []; @@ -204,7 +203,6 @@ const groupEventsByEndpoint = (events) => { return { engageEvents: eventMap.engage, groupsEvents: eventMap.groups, - trackEvents: eventMap.track, importEvents: eventMap.import, batchErrorRespList, }; @@ -349,7 +347,6 @@ const generatePageOrScreenCustomEventName = (message, userDefinedEventTemplate) * @param {Object} batchSize - The object containing the batch size for different endpoints. * @param {number} batchSize.engage - The batch size for engage endpoint. * @param {number} batchSize.groups - The batch size for group endpoint. - * @param {number} batchSize.track - The batch size for track endpoint. * @param {number} batchSize.import - The batch size for import endpoint. * @param {string} destinationId - The ID of the destination. * @returns {void} @@ -361,9 +358,6 @@ const recordBatchSizeMetrics = (batchSize, destinationId) => { stats.gauge('mixpanel_batch_group_pack_size', batchSize.groups, { destination_id: destinationId, }); - stats.gauge('mixpanel_batch_track_pack_size', batchSize.track, { - destination_id: destinationId, - }); stats.gauge('mixpanel_batch_import_pack_size', batchSize.import, { destination_id: destinationId, }); diff --git a/src/v0/destinations/mp/util.test.js b/src/v0/destinations/mp/util.test.js index 40cdb34649..3666081f59 100644 --- a/src/v0/destinations/mp/util.test.js +++ b/src/v0/destinations/mp/util.test.js @@ -18,7 +18,6 @@ describe('Unit test cases for groupEventsByEndpoint', () => { expect(result).toEqual({ engageEvents: [], groupsEvents: [], - trackEvents: [], importEvents: [], batchErrorRespList: [], }); @@ -122,19 +121,6 @@ describe('Unit test cases for groupEventsByEndpoint', () => { }, }, ], - trackEvents: [ - { - message: { - endpoint: '/track', - body: { - JSON_ARRAY: { - batch: '[{prop:4}]', - }, - }, - userId: 'user1', - }, - }, - ], importEvents: [ { message: { diff --git a/src/v0/destinations/pinterest_tag/utils.js b/src/v0/destinations/pinterest_tag/utils.js index 340fba498e..57d595571f 100644 --- a/src/v0/destinations/pinterest_tag/utils.js +++ b/src/v0/destinations/pinterest_tag/utils.js @@ -41,8 +41,8 @@ const getHashedValue = (key, value) => { case 'fn': case 'ge': value = Array.isArray(value) - ? value.map((val) => val.toString().toLowerCase()) - : value.toString().toLowerCase(); + ? value.map((val) => val.toString().trim().toLowerCase()) + : value.toString().trim().toLowerCase(); break; case 'ph': // phone numbers should only contain digits & should not contain leading zeros @@ -53,7 +53,7 @@ const getHashedValue = (key, value) => { case 'zp': // zip fields should only contain digits value = Array.isArray(value) - ? value.map((val) => val.toString().replace(/\D/g, '')) + ? value.map((val) => val.toString().trim().replace(/\D/g, '')) : value.toString().replace(/\D/g, ''); break; case 'hashed_maids': diff --git a/src/v0/destinations/redis/transform.js b/src/v0/destinations/redis/transform.js index 23c73f0ba4..ec0e858d3e 100644 --- a/src/v0/destinations/redis/transform.js +++ b/src/v0/destinations/redis/transform.js @@ -2,7 +2,7 @@ const lodash = require('lodash'); const flatten = require('flat'); const { InstrumentationError } = require('@rudderstack/integrations-lib'); -const { isEmpty, isObject } = require('../../util'); +const { isEmpty, isObject, getFieldValueFromMessage } = require('../../util'); const { EventType } = require('../../../constants'); // processValues: @@ -46,6 +46,19 @@ const transformSubEventTypeProfiles = (message, workspaceId, destinationId) => { }; }; +const getJSONValue = (message) => { + const eventType = message.type.toLowerCase(); + if (eventType === EventType.IDENTIFY) { + return getFieldValueFromMessage(message, 'traits'); + } + return {}; +}; + +const getTransformedPayloadForJSON = ({ key, path, value, userId }) => ({ + message: { key, path, value }, + userId, +}); + const process = (event) => { const { message, destination, metadata } = event; const messageType = message && message.type && message.type.toLowerCase(); @@ -58,15 +71,35 @@ const process = (event) => { throw new InstrumentationError('Blank userId passed in identify event'); } - const { prefix } = destination.Config; + const { prefix, useJSONModule } = destination.Config; const destinationId = destination.ID; const keyPrefix = isEmpty(prefix) ? '' : `${prefix.trim()}:`; + const jsonValue = getJSONValue(message); + if (isSubEventTypeProfiles(message)) { const { workspaceId } = metadata; + if (useJSONModule) { + // If redis should store information as JSON type + return getTransformedPayloadForJSON({ + key: `${workspaceId}:${destinationId}:${message.context.sources.profiles_entity}:${message.context.sources.profiles_id_type}:${message.userId}`, + path: message.context.sources.profiles_model, + value: jsonValue, + userId: message.userId, + }); + } return transformSubEventTypeProfiles(message, workspaceId, destinationId); } + if (useJSONModule) { + // If redis should store information as JSON type + return getTransformedPayloadForJSON({ + key: `${keyPrefix}user:${lodash.toString(message.userId)}`, + value: jsonValue, + userId: message.userId, + }); + } + const hmap = { key: `${keyPrefix}user:${lodash.toString(message.userId)}`, fields: {}, diff --git a/src/v0/destinations/tiktok_ads/data/TikTokTrackV2.json b/src/v0/destinations/tiktok_ads/data/TikTokTrackV2.json index 530d6e392a..2910f1b44c 100644 --- a/src/v0/destinations/tiktok_ads/data/TikTokTrackV2.json +++ b/src/v0/destinations/tiktok_ads/data/TikTokTrackV2.json @@ -25,10 +25,7 @@ }, { "destKey": "properties.content_type", - "sourceKeys": ["properties.contentType", "properties.content_type"], - "metadata": { - "defaultValue": "product" - } + "sourceKeys": ["properties.contentType", "properties.content_type"] }, { "destKey": "properties.shop_id", diff --git a/src/v0/destinations/tiktok_ads/transformV2.js b/src/v0/destinations/tiktok_ads/transformV2.js index 3bd8699e3a..4624ec9181 100644 --- a/src/v0/destinations/tiktok_ads/transformV2.js +++ b/src/v0/destinations/tiktok_ads/transformV2.js @@ -31,7 +31,7 @@ const { JSON_MIME_TYPE } = require('../../util/constant'); * @param {*} event * @returns track payload */ -const getTrackResponsePayload = (message, destConfig, event) => { +const getTrackResponsePayload = (message, destConfig, event, setDefaultForContentType = true) => { const payload = constructPayload(message, trackMappingV2); // if contents is not an array converting it into array @@ -53,6 +53,10 @@ const getTrackResponsePayload = (message, destConfig, event) => { if (destConfig.hashUserProperties && isDefinedAndNotNullAndNotEmpty(payload.user)) { payload.user = hashUserField(payload.user); } + // setting content-type default value in case of all standard event except `page-view` + if (!payload.properties?.content_type && setDefaultForContentType) { + payload.properties.content_type = 'product'; + } payload.event = event; // add partner name and return payload return removeUndefinedAndNullValues(payload); @@ -90,13 +94,17 @@ const trackResponseBuilder = async (message, { Config }) => { }); } }); - } else { + } else if (!eventNameMapping[event]) { /* + Custom Event Case -> if there exists no event mapping we will build payload with custom event recieved For custom event we do not want to lower case the event or trim it we just want to send those as it is Doc https://ads.tiktok.com/help/article/standard-events-parameters?lang=en */ - event = eventNameMapping[event] || message.event; - // if there exists no event mapping we will build payload with custom event recieved + event = message.event; + responseList.push(getTrackResponsePayload(message, Config, event, false)); + } else { + // incoming event name is already a standard event name + event = eventNameMapping[event]; responseList.push(getTrackResponsePayload(message, Config, event)); } // set event source and event_source_id diff --git a/src/v0/destinations/yahoo_dsp/util.js b/src/v0/destinations/yahoo_dsp/util.js index 255f84d1c9..54002a3bce 100644 --- a/src/v0/destinations/yahoo_dsp/util.js +++ b/src/v0/destinations/yahoo_dsp/util.js @@ -51,7 +51,7 @@ const populateIdentifiers = (audienceList, Config) => { } // here, hashing the data if is not hashed and pushing in the seedList array. if (hashRequired) { - seedList.push(sha256(userTraits[audienceAttribute])); + seedList.push(sha256(userTraits[audienceAttribute].trim())); } else { seedList.push(userTraits[audienceAttribute]); } diff --git a/src/v0/sources/adjust/config.ts b/src/v0/sources/adjust/config.ts new file mode 100644 index 0000000000..d1c6ab8242 --- /dev/null +++ b/src/v0/sources/adjust/config.ts @@ -0,0 +1,16 @@ +export const excludedFieldList = [ + 'activity_kind', + 'event', + 'event_name', + 'gps_adid', + 'idfa', + 'idfv', + 'adid', + 'tracker', + 'tracker_name', + 'app_name', + 'ip_address', + 'tracking_enabled', + 'tracker_token', + 'created_at', +]; diff --git a/src/v0/sources/adjust/mapping.json b/src/v0/sources/adjust/mapping.json new file mode 100644 index 0000000000..60ea66281e --- /dev/null +++ b/src/v0/sources/adjust/mapping.json @@ -0,0 +1,52 @@ +[ + { + "sourceKeys": "activity_kind", + "destKeys": "properties.activity_kind" + }, + { + "sourceKeys": "event", + "destKeys": "properties.event_token" + }, + { + "sourceKeys": "event_name", + "destKeys": "event" + }, + { + "sourceKeys": "gps_adid", + "destKeys": "properties.gps_adid" + }, + { + "sourceKeys": "idfa", + "destKeys": "context.device.advertisingId" + }, + { + "sourceKeys": "idfv", + "destKeys": "context.device.id" + }, + { + "sourceKeys": "adid", + "destKeys": "context.device.id " + }, + { + "sourceKeys": "tracker", + "destKeys": "properties.tracker" + }, + { + "sourceKeys": "tracker_name", + "destKeys": "properties.tracker_name" + }, + { "sourceKeys": "tracker_token", "destKeys": "properties.tracker_token" }, + + { + "sourceKeys": "app_name", + "destKeys": "context.app.name" + }, + { + "sourceKeys": "ip_address", + "destKeys": ["context.ip", "request_ip"] + }, + { + "sourceKeys": "tracking_enabled", + "destKeys": "properties.tracking_enabled" + } +] diff --git a/src/v0/sources/adjust/transform.js b/src/v0/sources/adjust/transform.js new file mode 100644 index 0000000000..8568622aeb --- /dev/null +++ b/src/v0/sources/adjust/transform.js @@ -0,0 +1,61 @@ +const lodash = require('lodash'); +const path = require('path'); +const fs = require('fs'); +const { TransformationError, structuredLogger: logger } = require('@rudderstack/integrations-lib'); +const Message = require('../message'); +const { CommonUtils } = require('../../../util/common'); +const { excludedFieldList } = require('./config'); +const { extractCustomFields, generateUUID } = require('../../util'); + +// ref : https://help.adjust.com/en/article/global-callbacks#general-recommended-placeholders +// import mapping json using JSON.parse to preserve object key order +const mapping = JSON.parse(fs.readFileSync(path.resolve(__dirname, './mapping.json'), 'utf-8')); + +const formatProperties = (input) => { + const { query_parameters: qParams } = input; + logger.debug(`[Adjust] Input event: query_params: ${JSON.stringify(qParams)}`); + if (!qParams) { + throw new TransformationError('Query_parameters is missing'); + } + const formattedOutput = {}; + Object.entries(qParams).forEach(([key, [value]]) => { + formattedOutput[key] = value; + }); + return formattedOutput; +}; + +const processEvent = (inputEvent) => { + const message = new Message(`Adjust`); + const event = lodash.cloneDeep(inputEvent); + const formattedPayload = formatProperties(event); + // event type is always track + const eventType = 'track'; + message.setEventType(eventType); + message.setPropertiesV2(formattedPayload, mapping); + let customProperties = {}; + customProperties = extractCustomFields( + formattedPayload, + customProperties, + 'root', + excludedFieldList, + ); + message.properties = { ...message.properties, ...customProperties }; + + if (formattedPayload.created_at) { + const ts = new Date(formattedPayload.created_at * 1000).toISOString(); + message.setProperty('originalTimestamp', ts); + message.setProperty('timestamp', ts); + } + + // adjust does not has the concept of user but we need to set some random anonymousId in order to make the server accept the message + message.anonymousId = generateUUID(); + return message; +}; + +// This fucntion just converts the incoming payload to array of already not and sends it to processEvent +const process = (events) => { + const eventsArray = CommonUtils.toArray(events); + return eventsArray.map(processEvent); +}; + +module.exports = { process }; diff --git a/src/v0/sources/auth0/mapping.json b/src/v0/sources/auth0/mapping.json index 45dcf939ad..bc5869a19b 100644 --- a/src/v0/sources/auth0/mapping.json +++ b/src/v0/sources/auth0/mapping.json @@ -62,5 +62,9 @@ { "sourceKeys": "date", "destKeys": ["originalTimestamp", "sentAt"] + }, + { + "sourceKeys": "type", + "destKeys": "source_type" } ] diff --git a/src/v0/util/facebookUtils/index.js b/src/v0/util/facebookUtils/index.js index c7753d255f..7462320cca 100644 --- a/src/v0/util/facebookUtils/index.js +++ b/src/v0/util/facebookUtils/index.js @@ -292,7 +292,13 @@ const formingFinalResponse = ( throw new TransformationError('Payload could not be constructed'); }; +const isHtmlFormat = (string) => { + const htmlTags = /<(?!(!doctype\s*html|html))\b[^>]*>[\S\s]*?<\/[^>]*>/i; + return htmlTags.test(string); +}; + module.exports = { + isHtmlFormat, getContentType, getContentCategory, transformedPayloadData, diff --git a/src/v0/util/facebookUtils/index.test.js b/src/v0/util/facebookUtils/index.test.js index 20c4ee59f2..1a2de4ed12 100644 --- a/src/v0/util/facebookUtils/index.test.js +++ b/src/v0/util/facebookUtils/index.test.js @@ -3,6 +3,7 @@ const { fetchUserData, deduceFbcParam, getContentType, + isHtmlFormat, } = require('./index'); const sha256 = require('sha256'); const { MAPPING_CONFIG, CONFIG_CATEGORIES } = require('../../destinations/facebook_pixel/config'); @@ -639,3 +640,53 @@ describe('getContentType', () => { expect(result).toBe(defaultValue); }); }); + +describe('isHtmlFormat', () => { + it('should return false for Json', () => { + expect(isHtmlFormat('{"a": 1, "b":2}')).toBe(false); + }); + + it('should return false for empty Json', () => { + expect(isHtmlFormat('{}')).toBe(false); + }); + + it('should return false for undefined', () => { + expect(isHtmlFormat(undefined)).toBe(false); + }); + + it('should return false for null', () => { + expect(isHtmlFormat(null)).toBe(false); + }); + + it('should return false for empty array', () => { + expect(isHtmlFormat([])).toBe(false); + }); + + it('should return true for html doctype', () => { + expect( + isHtmlFormat( + '
', + ), + ).toBe(true); + }); + + it('should return true for html', () => { + expect( + isHtmlFormat( + '