From 71c3d46236fff9209625cfb0737c21db2d275345 Mon Sep 17 00:00:00 2001 From: Yashasvi Bajpai <33063622+yashasvibajpai@users.noreply.github.com> Date: Fri, 17 May 2024 13:40:39 +0530 Subject: [PATCH 1/6] fix: move af_currency outside properties in eventValue (#3316) * fix: move af_currency outside properties in eventValue * chore: add af_currency to eventValue root via config --------- Co-authored-by: Sankeerth --- src/v0/destinations/af/transform.js | 17 ++++++++--------- .../destinations/af/processor/data.ts | 3 ++- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/src/v0/destinations/af/transform.js b/src/v0/destinations/af/transform.js index d6c41937a1..72ba47a227 100644 --- a/src/v0/destinations/af/transform.js +++ b/src/v0/destinations/af/transform.js @@ -113,13 +113,9 @@ function getEventValueForUnIdentifiedTrackEvent(message) { return { eventValue }; } -function getEventValueMapFromMappingJson( - message, - mappingJson, - isMultiSupport, - addPropertiesAtRoot, -) { +function getEventValueMapFromMappingJson(message, mappingJson, isMultiSupport, config) { let eventValue = {}; + const { addPropertiesAtRoot, afCurrencyAtRoot } = config; if (addPropertiesAtRoot) { eventValue = message.properties; @@ -152,6 +148,9 @@ function getEventValueMapFromMappingJson( af_price: prices, }; } + if (afCurrencyAtRoot) { + eventValue.af_currency = message.properties.currency; + } eventValue = removeUndefinedValues(eventValue); if (Object.keys(eventValue).length > 0) { eventValue = JSON.stringify(eventValue); @@ -171,7 +170,7 @@ function processNonTrackEvents(message, eventName) { return payload; } -function processEventTypeTrack(message, addPropertiesAtRoot) { +function processEventTypeTrack(message, config) { let isMultiSupport = true; const evType = message.event && message.event.toLowerCase(); let category = ConfigCategory.DEFAULT; @@ -195,7 +194,7 @@ function processEventTypeTrack(message, addPropertiesAtRoot) { message, mappingConfig[category.name], isMultiSupport, - addPropertiesAtRoot, + config, ); payload.eventName = message.event; payload.eventCurrency = message?.properties?.currency; @@ -208,7 +207,7 @@ function processSingleMessage(message, destination) { let payload; switch (messageType) { case EventType.TRACK: { - payload = processEventTypeTrack(message, destination.Config.addPropertiesAtRoot); + payload = processEventTypeTrack(message, destination.Config); break; } case EventType.SCREEN: { diff --git a/test/integrations/destinations/af/processor/data.ts b/test/integrations/destinations/af/processor/data.ts index d0fd29b089..210f04331d 100644 --- a/test/integrations/destinations/af/processor/data.ts +++ b/test/integrations/destinations/af/processor/data.ts @@ -1057,6 +1057,7 @@ export const data = [ groupValueTrait: 'age', trackProductsOnce: false, trackRevenuePerProduct: false, + afCurrencyAtRoot: true, }, }, }, @@ -1079,7 +1080,7 @@ export const data = [ body: { JSON: { eventValue: - '{"properties":{"tax":2,"total":27.5,"coupon":"hasbros","revenue":48,"price":25,"quantity":2,"currency":"ZAR","discount":2.5,"order_id":"50314b8e9bcf000000000000","products":[{"sku":"45790-32","url":"https://www.example.com/product/path","name":"Monopoly: 3rd Edition","price":19,"category":"Games","quantity":1,"image_url":"https:///www.example.com/product/path.jpg","product_id":"507f1f77bcf86cd799439011"},{"sku":"46493-32","name":"Uno Card Game","price":3,"category":"Games","quantity":2,"product_id":"505bd76785ebb509fc183733"}],"shipping":3,"subtotal":22.5,"affiliation":"Google Store","checkout_id":"fksdjfsdjfisjf9sdfjsd9f"},"af_revenue":48,"af_quantity":2,"af_price":25}', + '{"properties":{"tax":2,"total":27.5,"coupon":"hasbros","revenue":48,"price":25,"quantity":2,"currency":"ZAR","discount":2.5,"order_id":"50314b8e9bcf000000000000","products":[{"sku":"45790-32","url":"https://www.example.com/product/path","name":"Monopoly: 3rd Edition","price":19,"category":"Games","quantity":1,"image_url":"https:///www.example.com/product/path.jpg","product_id":"507f1f77bcf86cd799439011"},{"sku":"46493-32","name":"Uno Card Game","price":3,"category":"Games","quantity":2,"product_id":"505bd76785ebb509fc183733"}],"shipping":3,"subtotal":22.5,"affiliation":"Google Store","checkout_id":"fksdjfsdjfisjf9sdfjsd9f"},"af_revenue":48,"af_quantity":2,"af_price":25,"af_currency":"ZAR"}', eventName: 'normal track event', eventTime: '2020-08-14T05:30:30.118Z', eventCurrency: 'ZAR', From e7678cbdae4c06449ea9352ce3db390d2a29da14 Mon Sep 17 00:00:00 2001 From: Sankeerth Date: Fri, 17 May 2024 15:51:31 +0530 Subject: [PATCH 2/6] fix: gaoc store sales batching transform contract (#3384) --- .../transform.js | 3 +- .../router/data.ts | 283 ++++++++++++++++++ 2 files changed, 284 insertions(+), 2 deletions(-) diff --git a/src/v0/destinations/google_adwords_offline_conversions/transform.js b/src/v0/destinations/google_adwords_offline_conversions/transform.js index c3be0f7cab..76b12587cd 100644 --- a/src/v0/destinations/google_adwords_offline_conversions/transform.js +++ b/src/v0/destinations/google_adwords_offline_conversions/transform.js @@ -170,9 +170,8 @@ const batchEvents = (storeSalesEvents) => { storeSalesEvent.message?.body?.JSON?.addConversionPayload?.operations, ); batchEventResponse.metadatas.push(storeSalesEvent.metadata); - batchEventResponse.destination = storeSalesEvent.destination; }); - + batchEventResponse.destination = storeSalesEvents[0].destination; return [ getSuccessRespEvents( batchEventResponse.batchedRequest, diff --git a/test/integrations/destinations/google_adwords_offline_conversions/router/data.ts b/test/integrations/destinations/google_adwords_offline_conversions/router/data.ts index 596e7550e5..9d1ba220c8 100644 --- a/test/integrations/destinations/google_adwords_offline_conversions/router/data.ts +++ b/test/integrations/destinations/google_adwords_offline_conversions/router/data.ts @@ -999,4 +999,287 @@ export const data = [ }, mockFns: timestampMock, }, + { + name: 'google_adwords_offline_conversions', + description: 'Test 1 should include destination when single store sales event is sent', + feature: 'router', + module: 'destination', + version: 'v0', + input: { + request: { + body: { + input: [ + { + message: { + type: 'track', + event: 'Order Completed', + sentAt: '2024-05-09T00:02:48.319Z', + userId: '7fe0de4847f6dafb0cba694ef725404a', + channel: 'sources', + context: { + banner: { + key: 'CS', + domain: 'www.champssports.com', + }, + traits: { + email: 'johnwick@gmail.com', + address: { + city: 'homestead', + state: 'fl', + country: 'usa', + postalCode: '33032', + }, + lastName: 'wick', + firstName: 'john', + }, + privacy: { + ccpa: true, + }, + sources: { + job_id: '123344545565466', + version: 'v1.48.7', + job_run_id: 'cou1407gdjb6rkrrtv5g', + task_run_id: 'cou1407gdjb6rkrrtv6g', + }, + snowflake: { + ID: '44acd2006efb6b7d1a0eaf0da2b05b69', + TAX: 8.05, + NAME: 'johnwick', + PHONE: '', + TOTAL: 115, + email: 'JONBOBBYwick@GMAIL.COM', + BANNER: 'CS', + COUPON: '[null]', + REVENUE: 123.05, + CATEGORY: 'Retail', + CURRENCY: 'USD', + DISCOUNT: 0, + LASTNAME: 'wick', + ORDER_ID: '12343-4886-294995', + PRODUCTS: + '[{"sku":"C2302100","product_id":816827,"category":"1","size":"10.5","name":"NIKE AF1 \'07 AN21-WH/BK","brand":"NIKE INC","variant":"WHITE/BLACK","class":"MENS","price":"115.0","division":"CASUAL-ATHLETIC","quantity":"1","discountFlag":"false"}]', + RUDDERID: 'UNAVAILABLE', + SHIPPING: 'n/a', + STORE_ID: '14540', + SUBTOTAL: 115, + FIRSTNAME: 'john', + MESSAGEID: 'UNAVAILABLE', + TIMESTAMP: '2024-05-07T17:27:28.262Z', + TOTAL_VAT: 123.05, + EVENT_DATE: '2024-05-07T00:00:00Z', + STORE_NAME: 'CHAMPS ', + DISCOUNT_VAT: 0, + IS_E_RECEIPT: '1', + SUBTOTAL_VAT: 123.05, + USER_ADDRESS: '', + FLX_CARDNUMBER: '99000002697409', + PAYMENT_METHOD: null, + ACCOUNT_ADDRESS: null, + CM_PHONE_NUMBER: '7868007626', + SHIPPING_METHOD: 'n/a', + STORE_ADDR_CITY: 'CUTLER BAY ', + CM_BILL_ADDRESS1: '13020 SW 256TH ST', + STORE_ADDR_STATE: 'FL', + STORE_ADDR_STREET: + 'SOUTHLAND MALL 20505 SOUTH DIXIE HWY SPACE 1401 ', + STORE_ADDR_COUNTRY: 'UNITED STATES', + STORE_ADDR_ZIPCODE: '33189 ', + ACCOUNT_ADDRESS_CITY: 'HOMESTEAD', + BILLING_ADDRESS_CITY: 'HOMESTEAD', + SHIP_TO_ADDRESS_CITY: 'UNAVAILABLE', + ACCOUNT_ADDRESS_STATE: 'FL', + BILLING_ADDRESS_STATE: 'FL', + SHIP_TO_ADDRESS_STATE: 'UNAVAILABLE', + SHIP_TO_ADDRESS_STREET: 'UNAVAILABLE', + ACCOUNT_ADDRESS_COUNTRY: 'US', + BILLING_ADDRESS_COUNTRY: 'USA', + SHIP_TO_ADDRESS_COUNTRY: 'UNAVAILABLE', + SHIP_TO_ADDRESS_POSTALCODE: 'UNAVAILABLE', + ACCOUNT_ADDRESS_POSTAL_CODE: '33032', + BILLING_ADDRESS_POSTAL_CODE: '33032', + }, + account_id: 'xxxxxxxxxx', + account_mcc: '1234556775', + }, + recordId: '1230', + rudderId: '35d5060a-2756-45d1-9808-cae9aec19166', + messageId: '23d5060b-2756-45c1-9108-c229aec19126', + timestamp: '2024-05-07 17:27:28-00:00', + properties: { + value: 123.05, + currency: 'USD', + order_id: '12343-4886-294995', + products: [ + { + sku: 'C2302100', + price: 115, + quantity: '1', + }, + ], + conversionDateTime: '2024-05-07 17:27:28-00:00', + }, + receivedAt: '2024-05-09T00:02:43.864Z', + request_ip: '10.7.208.179', + originalTimestamp: '2024-05-09T00:02:48.319Z', + }, + metadata: { + secret: { + access_token: 'abcd1234', + refresh_token: 'efgh5678', + developer_token: 'ijkl91011', + }, + jobId: 1, + userId: 'u1', + }, + destination: { + Config: { + // customerId: '962-581-2972', + customerId: '{{ event.context.account_mcc || "1234556775" }}', + subAccount: false, + loginCustomerId: '{{ event.context.account_mcc || "1234556775" }}', + eventsToOfflineConversionsTypeMapping: [ + { + from: 'Order Completed', + to: 'store', + }, + ], + eventsToConversionsNamesMapping: [ + { + from: 'Order Completed', + to: 'Store sales', + }, + ], + UserIdentifierSource: 'FIRST_PARTY', + conversionEnvironment: 'none', + defaultUserIdentifier: 'email', + hashUserIdentifier: true, + validateOnly: false, + oneTrustCookieCategories: [], + eventDelivery: false, + eventDeliveryTS: 1715104236592, + rudderAccountId: '25u5whFH7gVTnCiAjn4ykoCLGoC', + }, + }, + }, + ], + destType: 'google_adwords_offline_conversions', + }, + method: 'POST', + }, + }, + output: { + response: { + status: 200, + body: { + output: [ + { + batchedRequest: { + version: '1', + type: 'REST', + method: 'POST', + endpoint: + 'https://googleads.googleapis.com/v16/customers/1234556775/offlineUserDataJobs', + headers: { + Authorization: 'Bearer abcd1234', + 'Content-Type': 'application/json', + 'developer-token': 'ijkl91011', + }, + params: { event: 'Store sales', customerId: '1234556775' }, + body: { + JSON: { + event: '1234556775', + isStoreConversion: true, + createJobPayload: { + job: { + type: 'STORE_SALES_UPLOAD_FIRST_PARTY', + storeSalesMetadata: { + loyaltyFraction: '1', + transaction_upload_fraction: '1', + }, + }, + }, + addConversionPayload: { + operations: [ + { + create: { + transaction_attribute: { + transaction_amount_micros: '123050000', + order_id: '12343-4886-294995', + currency_code: 'USD', + transaction_date_time: '2019-10-14 16:45:18+05:30', + }, + userIdentifiers: [ + { + hashedEmail: + 'cd54e8f2e90e2a092a153f7e27e7b47a8ad29adb7943a05d749f0f9836935a2f', + }, + ], + consent: { + adPersonalization: 'UNSPECIFIED', + adUserData: 'UNSPECIFIED', + }, + }, + }, + ], + enable_partial_failure: false, + enable_warnings: false, + validate_only: false, + }, + executeJobPayload: { validate_only: false }, + }, + JSON_ARRAY: {}, + XML: {}, + FORM: {}, + }, + files: {}, + }, + metadata: [ + { + secret: { + access_token: 'abcd1234', + refresh_token: 'efgh5678', + developer_token: 'ijkl91011', + }, + jobId: 1, + userId: 'u1', + }, + ], + destination: { + Config: { + // customerId: '962-581-2972', + customerId: '1234556775', + subAccount: false, + loginCustomerId: '1234556775', + eventsToOfflineConversionsTypeMapping: [ + { + from: 'Order Completed', + to: 'store', + }, + ], + eventsToConversionsNamesMapping: [ + { + from: 'Order Completed', + to: 'Store sales', + }, + ], + UserIdentifierSource: 'FIRST_PARTY', + conversionEnvironment: 'none', + defaultUserIdentifier: 'email', + hashUserIdentifier: true, + validateOnly: false, + oneTrustCookieCategories: [], + eventDelivery: false, + eventDeliveryTS: 1715104236592, + rudderAccountId: '25u5whFH7gVTnCiAjn4ykoCLGoC', + }, + }, + batched: true, + statusCode: 200, + }, + ], + }, + }, + }, + mockFns: timestampMock, + }, ]; From fbb0811aa0e417b0cffcea4ecc103979afccfe74 Mon Sep 17 00:00:00 2001 From: Sudip Paul <67197965+ItsSudip@users.noreply.github.com> Date: Mon, 20 May 2024 12:57:31 +0530 Subject: [PATCH 3/6] fix: remove default traits from ortto (#3389) --- src/cdk/v2/destinations/ortto/procWorkflow.yaml | 6 +++--- .../destinations/ortto/processor/data.ts | 12 ++---------- test/integrations/destinations/ortto/router/data.ts | 7 +------ 3 files changed, 6 insertions(+), 19 deletions(-) diff --git a/src/cdk/v2/destinations/ortto/procWorkflow.yaml b/src/cdk/v2/destinations/ortto/procWorkflow.yaml index a8fb9b2984..27c3749cc8 100644 --- a/src/cdk/v2/destinations/ortto/procWorkflow.yaml +++ b/src/cdk/v2/destinations/ortto/procWorkflow.yaml @@ -48,9 +48,9 @@ steps: "str::ei": {{{{$.getGenericPaths("userId")}}}}, "str::language": .context.traits.language || .context.locale, "phn::phone": phone ? {"n": phone}, - "bol::gdpr": .context.traits.gdpr ?? true, - "bol::p": .context.traits.emailConsent || false, - "bol::sp": .context.traits.smsConsent || false, + "bol::gdpr": .context.traits.gdpr, + "bol::p": .context.traits.emailConsent, + "bol::sp": .context.traits.smsConsent, }, "location": {"source_ip": .context.ip} }); diff --git a/test/integrations/destinations/ortto/processor/data.ts b/test/integrations/destinations/ortto/processor/data.ts index 0afdb05318..ff84f5dbbd 100644 --- a/test/integrations/destinations/ortto/processor/data.ts +++ b/test/integrations/destinations/ortto/processor/data.ts @@ -158,6 +158,8 @@ export const data = [ people: [ { fields: { + 'bol::p': true, + 'bol::sp': true, 'str::first': 'John', 'str::last': 'Sparrow', 'str::email': 'identify@test.com', @@ -172,9 +174,6 @@ export const data = [ 'phn::phone': { n: '9112340345', }, - 'bol::gdpr': true, - 'bol::p': true, - 'bol::sp': true, 'str:cm:ortto-attirbute0': 'abc', 'str:cm:ortto-attirbute1': 'def', }, @@ -1036,8 +1035,6 @@ export const data = [ n: '9112340345', }, 'bol::gdpr': false, - 'bol::p': false, - 'bol::sp': false, 'str:cm:ortto-attirbute0': 'abc', 'str:cm:ortto-attirbute1': 'def', }, @@ -1250,8 +1247,6 @@ export const data = [ 'str::language': 'en-US', 'str::ei': 'sample_user_id', 'bol::gdpr': false, - 'bol::p': false, - 'bol::sp': false, 'str:cm:ortto-attirbute0': 'abc', 'str:cm:ortto-attirbute1': 'def', }, @@ -1623,9 +1618,6 @@ export const data = [ 'geo::region': {}, 'str::ei': 'XxXxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxx', 'str::language': 'en-MU', - 'bol::gdpr': true, - 'bol::p': false, - 'bol::sp': false, }, activity_id: 'act:cm:liked-a-set', attributes: { diff --git a/test/integrations/destinations/ortto/router/data.ts b/test/integrations/destinations/ortto/router/data.ts index cf5731be80..8999637a66 100644 --- a/test/integrations/destinations/ortto/router/data.ts +++ b/test/integrations/destinations/ortto/router/data.ts @@ -298,9 +298,6 @@ export const data = [ 'phn::phone': { n: '9112340345', }, - 'bol::gdpr': true, - 'bol::p': false, - 'bol::sp': false, 'str:cm:ortto-attirbute0': 'abc', 'str:cm:ortto-attirbute1': 'def', }, @@ -385,6 +382,7 @@ export const data = [ activities: [ { fields: { + 'bol::gdpr': false, 'str::first': 'John', 'str::last': 'Sparrow', 'str::email': 'identify@test.com', @@ -404,9 +402,6 @@ export const data = [ 'phn::phone': { n: '9112340345', }, - 'bol::gdpr': false, - 'bol::p': false, - 'bol::sp': false, 'str:cm:ortto-attirbute0': 'abc', 'str:cm:ortto-attirbute1': 'def', }, From 755073c4341a454785050d835021d9f17e0b9d3f Mon Sep 17 00:00:00 2001 From: Sudip Paul <67197965+ItsSudip@users.noreply.github.com> Date: Mon, 20 May 2024 13:30:31 +0530 Subject: [PATCH 4/6] fix: add validation for null/undefined traits in slack (#3382) --- src/v0/destinations/slack/util.js | 2 +- .../destinations/slack/processor/data.ts | 155 ++++++++++++++++++ 2 files changed, 156 insertions(+), 1 deletion(-) diff --git a/src/v0/destinations/slack/util.js b/src/v0/destinations/slack/util.js index f5d407018b..2267aa0bcd 100644 --- a/src/v0/destinations/slack/util.js +++ b/src/v0/destinations/slack/util.js @@ -83,7 +83,7 @@ const buildDefaultTraitTemplate = (traitsList, traits, template) => { generatedStringFromTemplate += `${trait}: {{"${trait}"}} `; }); // else with all traits - if (traitsList.length === 0) { + if (traitsList.length === 0 && !!traits) { Object.keys(traits).forEach((traitKey) => { generatedStringFromTemplate += `${traitKey}: {{"${traitKey}"}} `; }); diff --git a/test/integrations/destinations/slack/processor/data.ts b/test/integrations/destinations/slack/processor/data.ts index 7deb555fa9..1fcbb2ca03 100644 --- a/test/integrations/destinations/slack/processor/data.ts +++ b/test/integrations/destinations/slack/processor/data.ts @@ -2017,4 +2017,159 @@ export const data = [ }, }, }, + { + name: 'slack', + description: + 'Test 12-> Identify -> Default template with some whiteListed traits and traits as null', + feature: 'processor', + module: 'destination', + version: 'v0', + input: { + request: { + body: [ + { + destination: { + ID: '1ZQVSU9SXNg6KYgZALaqjAO3PIL', + Name: 'test-slack', + DestinationDefinition: { + ID: '1ZQUiJVMlmF7lfsdfXg7KXQnlLV', + Name: 'SLACK', + DisplayName: 'Slack', + Config: { + excludeKeys: [], + includeKeys: [], + }, + }, + Config: { + eventChannelSettings: [ + { + eventChannelWebhook: 'https://hooks.slack.com/services/example/test/demo', + eventName: 'is', + eventRegex: true, + }, + ], + eventTemplateSettings: [ + { + eventName: 'is', + eventRegex: true, + eventTemplate: + '{{name}} performed {{event}} with {{properties.key1}} {{properties.key2}}', + }, + { + eventName: '', + eventRegex: false, + eventTemplate: '', + }, + ], + webhookUrl: 'https://hooks.slack.com/services/THZM86VSS/BV9HZ2UN6/demo', + whitelistedTraitsSettings: [], + }, + Enabled: true, + Transformations: [], + IsProcessorEnabled: true, + }, + message: { + anonymousId: '4de817fb-7f8e-4e23-b9be-f6736dbda20f', + channel: 'web', + context: { + app: { + build: '1.0.0', + name: 'RudderLabs JavaScript SDK', + namespace: 'com.rudderlabs.javascript', + version: '1.1.1-rc.1', + }, + library: { + name: 'RudderLabs JavaScript SDK', + version: '1.1.1-rc.1', + }, + locale: 'en-US', + os: { + name: '', + version: '', + }, + page: { + path: '/tests/html/script-test.html', + referrer: 'http://localhost:1111/tests/html/', + search: '', + title: '', + url: 'http://localhost:1111/tests/html/script-test.html', + }, + screen: { + density: 1.7999999523162842, + }, + userAgent: + 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.149 Safari/537.36', + }, + integrations: { + All: true, + }, + messageId: '9ecc0183-89ed-48bd-87eb-b2d8e1ca6780', + originalTimestamp: '2020-03-23T03:46:30.916Z', + properties: { + path: '/tests/html/script-test.html', + referrer: 'http://localhost:1111/tests/html/', + search: '', + title: '', + url: 'http://localhost:1111/tests/html/script-test.html', + }, + receivedAt: '2020-03-23T09:16:31.041+05:30', + request_ip: '[::1]:52056', + sentAt: '2020-03-23T03:46:30.916Z', + timestamp: '2020-03-23T09:16:31.041+05:30', + type: 'identify', + userId: '12345', + }, + metadata: { + anonymousId: '4de817fb-7f8e-4e23-b9be-f6736dbda20f', + destinationId: '1ZQVSU9SXNg6KYgZALaqjAO3PIL', + destinationType: 'SLACK', + jobId: 126, + messageId: '9ecc0183-89ed-48bd-87eb-b2d8e1ca6780', + sourceId: '1YhwKyDcKstudlGxkeN5p2wgsrp', + }, + }, + ], + }, + }, + output: { + response: { + status: 200, + body: [ + { + output: { + version: '1', + type: 'REST', + method: 'POST', + endpoint: 'https://hooks.slack.com/services/THZM86VSS/BV9HZ2UN6/demo', + headers: { + 'Content-Type': 'application/x-www-form-urlencoded', + }, + params: {}, + body: { + JSON: {}, + JSON_ARRAY: {}, + XML: {}, + FORM: { + payload: + '{"text":"Identified User 12345","username":"RudderStack","icon_url":"https://cdn.rudderlabs.com/rudderstack.png"}', + }, + }, + files: {}, + userId: '12345', + statusCode: 200, + }, + metadata: { + anonymousId: '4de817fb-7f8e-4e23-b9be-f6736dbda20f', + destinationId: '1ZQVSU9SXNg6KYgZALaqjAO3PIL', + destinationType: 'SLACK', + jobId: 126, + messageId: '9ecc0183-89ed-48bd-87eb-b2d8e1ca6780', + sourceId: '1YhwKyDcKstudlGxkeN5p2wgsrp', + }, + statusCode: 200, + }, + ], + }, + }, + }, ]; From 3ad78506446915ada8bdc5f5594dc2710e6b0646 Mon Sep 17 00:00:00 2001 From: Sudip Paul <67197965+ItsSudip@users.noreply.github.com> Date: Mon, 20 May 2024 13:59:11 +0530 Subject: [PATCH 5/6] fix: update validation of event duration (#3376) * fix: update validation of event duration * chore: address comment * chore: add test case --- .../facebook_conversions/transform.js | 16 +--- .../destinations/facebook_pixel/transform.js | 21 +---- src/v0/destinations/facebook_pixel/utils.js | 26 ++++++ .../destinations/facebook_pixel/utils.test.js | 79 +++++++++++++++- .../facebook_conversions/processor/data.ts | 2 +- .../processor/validationTestData.ts | 90 ++++++++++++++++++- 6 files changed, 198 insertions(+), 36 deletions(-) diff --git a/src/v0/destinations/facebook_conversions/transform.js b/src/v0/destinations/facebook_conversions/transform.js index 1bb97b2672..e4aee9c503 100644 --- a/src/v0/destinations/facebook_conversions/transform.js +++ b/src/v0/destinations/facebook_conversions/transform.js @@ -1,6 +1,5 @@ /* eslint-disable no-param-reassign */ const get = require('get-value'); -const moment = require('moment'); const { InstrumentationError, ConfigurationError } = require('@rudderstack/integrations-lib'); const { CONFIG_CATEGORIES, @@ -33,6 +32,7 @@ const { fetchUserData, formingFinalResponse, } = require('../../util/facebookUtils'); +const { verifyEventDuration } = require('../facebook_pixel/utils'); const responseBuilderSimple = (message, category, destination) => { const { Config, ID } = destination; @@ -121,19 +121,7 @@ const processEvent = (message, destination) => { } const timeStamp = getFieldValueFromMessage(message, 'timestamp'); - if (timeStamp) { - const start = moment.unix(moment(timeStamp).format('X')); - const current = moment.unix(moment().format('X')); - // calculates past event in days - const deltaDay = Math.ceil(moment.duration(current.diff(start)).asDays()); - // calculates future event in minutes - const deltaMin = Math.ceil(moment.duration(start.diff(current)).asMinutes()); - if (deltaDay > 7 || deltaMin > 1) { - throw new InstrumentationError( - 'Events must be sent within seven days of their occurrence or up to one minute in the future.', - ); - } - } + verifyEventDuration(message, destination, timeStamp); const { datasetId, accessToken } = destination.Config; if (!datasetId) { diff --git a/src/v0/destinations/facebook_pixel/transform.js b/src/v0/destinations/facebook_pixel/transform.js index 8a63998b45..d44a38aa69 100644 --- a/src/v0/destinations/facebook_pixel/transform.js +++ b/src/v0/destinations/facebook_pixel/transform.js @@ -1,8 +1,6 @@ /* eslint-disable no-param-reassign */ const get = require('get-value'); -const moment = require('moment'); const { InstrumentationError, ConfigurationError } = require('@rudderstack/integrations-lib'); -const stats = require('../../../util/stats'); const { VERSION, CONFIG_CATEGORIES, @@ -31,6 +29,7 @@ const { handleOrder, populateCustomDataBasedOnCategory, getCategoryFromEvent, + verifyEventDuration, } = require('./utils'); const { @@ -170,23 +169,7 @@ const processEvent = (message, destination) => { } const timeStamp = message.timestamp || message.originalTimestamp; - if (timeStamp) { - const start = moment.unix(moment(timeStamp).format('X')); - const current = moment.unix(moment().format('X')); - // calculates past event in days - const deltaDay = Math.ceil(moment.duration(current.diff(start)).asDays()); - // calculates future event in minutes - const deltaMin = Math.ceil(moment.duration(start.diff(current)).asMinutes()); - if (deltaDay > 7 || deltaMin > 1) { - // TODO: Remove after testing in mirror transformer - stats.increment('fb_pixel_timestamp_error', { - destinationId: destination.ID, - }); - throw new InstrumentationError( - 'Events must be sent within seven days of their occurrence or up to one minute in the future.', - ); - } - } + verifyEventDuration(message, destination, timeStamp); let eventsToEvents; if (Array.isArray(destination.Config.eventsToEvents)) { diff --git a/src/v0/destinations/facebook_pixel/utils.js b/src/v0/destinations/facebook_pixel/utils.js index cfa625ee3d..74d5240f2a 100644 --- a/src/v0/destinations/facebook_pixel/utils.js +++ b/src/v0/destinations/facebook_pixel/utils.js @@ -1,4 +1,6 @@ const { InstrumentationError } = require('@rudderstack/integrations-lib'); +const get = require('get-value'); +const moment = require('moment'); const { isObject } = require('../../util'); const { ACTION_SOURCES_VALUES, @@ -339,6 +341,29 @@ const getCategoryFromEvent = (eventName) => { return category; }; +const verifyEventDuration = (message, destination, timeStamp) => { + const actionSource = + get(message, 'traits.action_source') || + get(message, 'context.traits.action_source') || + get(message, 'properties.action_source'); + + const start = moment.unix(moment(timeStamp).format('X')); + const current = moment.unix(moment().format('X')); + // calculates past event in days + const deltaDay = Math.ceil(moment.duration(current.diff(start)).asDays()); + // calculates future event in minutes + const deltaMin = Math.ceil(moment.duration(start.diff(current)).asMinutes()); + let defaultSupportedDelta = 7; + if (actionSource === 'physical_store') { + defaultSupportedDelta = 62; + } + if (deltaDay > defaultSupportedDelta || deltaMin > 1) { + throw new InstrumentationError( + `Events must be sent within ${defaultSupportedDelta} days of their occurrence or up to one minute in the future.`, + ); + } +}; + module.exports = { formatRevenue, getActionSource, @@ -348,4 +373,5 @@ module.exports = { handleOrder, populateCustomDataBasedOnCategory, getCategoryFromEvent, + verifyEventDuration, }; diff --git a/src/v0/destinations/facebook_pixel/utils.test.js b/src/v0/destinations/facebook_pixel/utils.test.js index f32d7d7024..fa17aebd76 100644 --- a/src/v0/destinations/facebook_pixel/utils.test.js +++ b/src/v0/destinations/facebook_pixel/utils.test.js @@ -1,7 +1,13 @@ const { InstrumentationError } = require('@rudderstack/integrations-lib'); -const { getActionSource, formatRevenue, getCategoryFromEvent } = require('./utils'); +const { + getActionSource, + formatRevenue, + getCategoryFromEvent, + verifyEventDuration, +} = require('./utils'); const { CONFIG_CATEGORIES, OTHER_STANDARD_EVENTS } = require('./config'); +Date.now = jest.fn(() => new Date('2022-01-20T00:00:00Z')); describe('Test Facebook Pixel Utils', () => { describe('getActionSource', () => { // Returns 'other' if payload.action_source is not defined and channel is neither 'web' nor 'mobile' @@ -151,4 +157,75 @@ describe('Test Facebook Pixel Utils', () => { expect(result).toEqual(CONFIG_CATEGORIES.SIMPLE_TRACK); }); }); + + describe('verifyEventDuration', () => { + it('should not throw an InstrumentationError when event duration is less than 8 days after the event occurred', () => { + const message = { + traits: { + action_source: 'some_action_source', + }, + context: { + traits: { + action_source: 'some_action_source', + }, + }, + properties: { + action_source: 'some_action_source', + }, + }; + const destination = { + ID: 'some_destination_id', + }; + const timeStamp = '2022-01-20T00:00:00Z'; + expect(() => { + verifyEventDuration(message, destination, timeStamp); + }).not.toThrow(InstrumentationError); + }); + it('should throw an InstrumentationError when event duration is exactly 8 days after the event occurred', () => { + const message = { + traits: { + action_source: 'some_action_source', + }, + context: { + traits: { + action_source: 'some_action_source', + }, + }, + properties: { + action_source: 'some_action_source', + }, + }; + const destination = { + ID: 'some_destination_id', + }; + const timeStamp = '2022-01-12T00:00:00Z'; + + expect(() => { + verifyEventDuration(message, destination, timeStamp); + }).toThrow(InstrumentationError); + }); + it('should not throw an InstrumentationError when event duration is greater than 8 days after the event occurred and action_source is physical_store', () => { + const message = { + traits: { + action_source: 'physical_store', + }, + context: { + traits: { + action_source: 'some_action_source', + }, + }, + properties: { + action_source: 'some_action_source', + }, + }; + const destination = { + ID: 'some_destination_id', + }; + const timeStamp = '2022-01-12T00:00:00Z'; + + expect(() => { + verifyEventDuration(message, destination, timeStamp); + }).not.toThrow(InstrumentationError); + }); + }); }); diff --git a/test/integrations/destinations/facebook_conversions/processor/data.ts b/test/integrations/destinations/facebook_conversions/processor/data.ts index bfa35bc22b..3224a15771 100644 --- a/test/integrations/destinations/facebook_conversions/processor/data.ts +++ b/test/integrations/destinations/facebook_conversions/processor/data.ts @@ -94,7 +94,7 @@ export const data = [ body: [ { error: - 'Events must be sent within seven days of their occurrence or up to one minute in the future.', + 'Events must be sent within 7 days of their occurrence or up to one minute in the future.', statusCode: 400, statTags: { destType: 'FACEBOOK_CONVERSIONS', diff --git a/test/integrations/destinations/facebook_pixel/processor/validationTestData.ts b/test/integrations/destinations/facebook_pixel/processor/validationTestData.ts index 8e24801464..a0f85e45e3 100644 --- a/test/integrations/destinations/facebook_pixel/processor/validationTestData.ts +++ b/test/integrations/destinations/facebook_pixel/processor/validationTestData.ts @@ -267,7 +267,7 @@ export const validationTestData = [ { statusCode: 400, error: - 'Events must be sent within seven days of their occurrence or up to one minute in the future.', + 'Events must be sent within 7 days of their occurrence or up to one minute in the future.', statTags: commonStatTags, }, ], @@ -370,4 +370,92 @@ export const validationTestData = [ }, }, }, + { + id: 'fbPixel-validation-test-5', + name: 'facebook_pixel', + description: '[Error]: validate event date and time', + scenario: 'Framework + business', + successCriteria: + 'Response should contain error message and status code should be 400, as we are sending an event which is older than 7 days and the error message should be Events must be sent within seven days of their occurrence or up to one minute in the future.', + feature: 'processor', + module: 'destination', + version: 'v0', + input: { + request: { + body: [ + { + message: generateSimplifiedTrackPayload({ + // Sun Oct 15 2023 + type: 'track', + event: 'TestEven001', + sentAt: '2023-08-25T16:12:02.048Z', + userId: 'sajal12', + context: { + traits: { + action_source: 'physical_store', + email: 'test@rudderstack.com', + phone: '9112340375', + event_id: 'x9lk3gfte768o1oy08cyaylx5t2j9q2wwfl2', + plan_details: { + plan_type: 'gold', + duration: '3 months', + }, + }, + }, + properties: { + revenue: 400, + additional_bet_index: 0, + }, + anonymousId: '9c6bd77ea9da3e68', + originalTimestamp: '2023-08-25T15:32:56.409Z', + }), + destination: overrideDestination(commonDestination, { + accessToken: '09876', + pixelId: 'dummyPixelId', + }), + }, + ], + }, + }, + output: { + response: { + status: 200, + body: [ + { + statusCode: 200, + output: { + body: { + FORM: { + data: [ + JSON.stringify({ + user_data: { + em: '1c5e54849f5c711ce38fa60716fbbe44bff478f9ca250897b39cdfc2438cd1bd', + ph: '820c46baccd33a1664f583b4505a7e39e033197e06e0bd7c87109e33c57c5497', + }, + event_name: 'TestEven001', + event_time: 1692977576, + event_id: 'x9lk3gfte768o1oy08cyaylx5t2j9q2wwfl2', + action_source: 'physical_store', + custom_data: { additional_bet_index: 0, value: 400 }, + }), + ], + }, + JSON: {}, + JSON_ARRAY: {}, + XML: {}, + }, + endpoint: 'https://graph.facebook.com/v18.0/dummyPixelId/events?access_token=09876', + files: {}, + headers: {}, + method: 'POST', + params: {}, + type: 'REST', + userId: '', + version: '1', + }, + }, + ], + }, + }, + }, ]; From d54f6ad794fc3a0f2356e0bec561ce51acaf8964 Mon Sep 17 00:00:00 2001 From: GitHub Actions Date: Mon, 20 May 2024 08:33:02 +0000 Subject: [PATCH 6/6] chore(release): 1.66.1 --- CHANGELOG.md | 11 +++++++++++ package-lock.json | 4 ++-- package.json | 2 +- 3 files changed, 14 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5db5ec047a..4258fdfcc5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,17 @@ All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines. +### [1.66.1](https://github.com/rudderlabs/rudder-transformer/compare/v1.66.0...v1.66.1) (2024-05-20) + + +### Bug Fixes + +* add validation for null/undefined traits in slack ([#3382](https://github.com/rudderlabs/rudder-transformer/issues/3382)) ([755073c](https://github.com/rudderlabs/rudder-transformer/commit/755073c4341a454785050d835021d9f17e0b9d3f)) +* gaoc store sales batching transform contract ([#3384](https://github.com/rudderlabs/rudder-transformer/issues/3384)) ([e7678cb](https://github.com/rudderlabs/rudder-transformer/commit/e7678cbdae4c06449ea9352ce3db390d2a29da14)) +* move af_currency outside properties in eventValue ([#3316](https://github.com/rudderlabs/rudder-transformer/issues/3316)) ([71c3d46](https://github.com/rudderlabs/rudder-transformer/commit/71c3d46236fff9209625cfb0737c21db2d275345)) +* remove default traits from ortto ([#3389](https://github.com/rudderlabs/rudder-transformer/issues/3389)) ([fbb0811](https://github.com/rudderlabs/rudder-transformer/commit/fbb0811aa0e417b0cffcea4ecc103979afccfe74)) +* update validation of event duration ([#3376](https://github.com/rudderlabs/rudder-transformer/issues/3376)) ([3ad7850](https://github.com/rudderlabs/rudder-transformer/commit/3ad78506446915ada8bdc5f5594dc2710e6b0646)) + ## [1.66.0](https://github.com/rudderlabs/rudder-transformer/compare/v1.65.1...v1.66.0) (2024-05-13) diff --git a/package-lock.json b/package-lock.json index 14ab853b82..f51f3ccd8e 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "rudder-transformer", - "version": "1.66.0", + "version": "1.66.1", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "rudder-transformer", - "version": "1.66.0", + "version": "1.66.1", "license": "ISC", "dependencies": { "@amplitude/ua-parser-js": "0.7.24", diff --git a/package.json b/package.json index 168f49da5e..50a276ce42 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "rudder-transformer", - "version": "1.66.0", + "version": "1.66.1", "description": "", "homepage": "https://github.com/rudderlabs/rudder-transformer#readme", "bugs": {