From b55ef387825db6da864410b2c4d69c56f885fe60 Mon Sep 17 00:00:00 2001 From: Yashasvi Bajpai <33063622+yashasvibajpai@users.noreply.github.com> Date: Mon, 1 Apr 2024 11:51:54 +0530 Subject: [PATCH 1/8] feat: onboard yandex metrica offline events destination, initial changes --- .../yandex_metrica_offline_events/config.js | 5 +++ .../procWorkflow.yaml | 41 +++++++++++++++++++ .../yandex_metrica_offline_events/utils.js | 0 3 files changed, 46 insertions(+) create mode 100644 src/cdk/v2/destinations/yandex_metrica_offline_events/config.js create mode 100644 src/cdk/v2/destinations/yandex_metrica_offline_events/procWorkflow.yaml create mode 100644 src/cdk/v2/destinations/yandex_metrica_offline_events/utils.js diff --git a/src/cdk/v2/destinations/yandex_metrica_offline_events/config.js b/src/cdk/v2/destinations/yandex_metrica_offline_events/config.js new file mode 100644 index 0000000000..83513c3856 --- /dev/null +++ b/src/cdk/v2/destinations/yandex_metrica_offline_events/config.js @@ -0,0 +1,5 @@ +const YANDEX_METRICA_OFFLINE_EVENTS = 'yandex_metrica_offline_events'; + +module.exports = { + YANDEX_METRICA_OFFLINE_EVENTS, +}; diff --git a/src/cdk/v2/destinations/yandex_metrica_offline_events/procWorkflow.yaml b/src/cdk/v2/destinations/yandex_metrica_offline_events/procWorkflow.yaml new file mode 100644 index 0000000000..d693bbb29f --- /dev/null +++ b/src/cdk/v2/destinations/yandex_metrica_offline_events/procWorkflow.yaml @@ -0,0 +1,41 @@ +bindings: + - name: EventType + path: ../../../../constants + - path: ../../bindings/jsontemplate + exportAll: true + - path: ./config + - name: removeUndefinedAndNullValues + path: ../../../../v0/util + - name: defaultRequestConfig + path: ../../../../v0/util + - path: ./utils + +steps: + - name: validateInput + template: | + let messageType = .message.type; + $.assert(messageType, "message Type is not present. Aborting message."); + $.assert(.message.type.toLowerCase() ==='identify' || .message.type.toLowerCase() ==='track', "Event type " + .message.type.toLowerCase() + " is not supported. Aborting message."); + $.assert(.message.traits || .message.properties, "Message traits/properties not present. Aborting message."); + + - name: prepareData + template: | + let data = .message.traits + let identifierType = .message.context.externalId[0].identifierType; + let identifierValue = .message.context.externalId[0].id; + let ClientId = identifierValue; + data.ClientId = ClientId; + data + + - name: buildResponseForProcessTransformation + description: build response + template: | + const response = $.defaultRequestConfig(); + response.body.JSON = $.outputs.prepareData + console.log(JSON.stringify(response)); + response + + + # header should change to clientID, YCLID etc + # batching support -> rtWorkflow + \ No newline at end of file diff --git a/src/cdk/v2/destinations/yandex_metrica_offline_events/utils.js b/src/cdk/v2/destinations/yandex_metrica_offline_events/utils.js new file mode 100644 index 0000000000..e69de29bb2 From a3e56f4e626e47bad543ca394172a7843ea1d805 Mon Sep 17 00:00:00 2001 From: Yashasvi Bajpai <33063622+yashasvibajpai@users.noreply.github.com> Date: Tue, 2 Apr 2024 04:01:55 +0530 Subject: [PATCH 2/8] chore: update id logic, add tests --- .../procWorkflow.yaml | 8 +- .../yandex_metrica_offline_events/utils.js | 15 + .../processor/data.ts | 296 ++++++++++++++++++ 3 files changed, 312 insertions(+), 7 deletions(-) create mode 100644 test/integrations/destinations/yandex_metrica_offline_events/processor/data.ts diff --git a/src/cdk/v2/destinations/yandex_metrica_offline_events/procWorkflow.yaml b/src/cdk/v2/destinations/yandex_metrica_offline_events/procWorkflow.yaml index d693bbb29f..18cd15d42d 100644 --- a/src/cdk/v2/destinations/yandex_metrica_offline_events/procWorkflow.yaml +++ b/src/cdk/v2/destinations/yandex_metrica_offline_events/procWorkflow.yaml @@ -23,8 +23,7 @@ steps: let data = .message.traits let identifierType = .message.context.externalId[0].identifierType; let identifierValue = .message.context.externalId[0].id; - let ClientId = identifierValue; - data.ClientId = ClientId; + data = $.setIdentifier(data, identifierType, identifierValue) data - name: buildResponseForProcessTransformation @@ -34,8 +33,3 @@ steps: response.body.JSON = $.outputs.prepareData console.log(JSON.stringify(response)); response - - - # header should change to clientID, YCLID etc - # batching support -> rtWorkflow - \ No newline at end of file diff --git a/src/cdk/v2/destinations/yandex_metrica_offline_events/utils.js b/src/cdk/v2/destinations/yandex_metrica_offline_events/utils.js index e69de29bb2..e3e3b2107e 100644 --- a/src/cdk/v2/destinations/yandex_metrica_offline_events/utils.js +++ b/src/cdk/v2/destinations/yandex_metrica_offline_events/utils.js @@ -0,0 +1,15 @@ +const setIdentifier = (data, identifierType, identifierValue) => { + const updatedData = data; + if (identifierType === 'ClientId') { + updatedData.ClientId = identifierValue; + } else if (identifierType === 'YCLID') { + updatedData.Yclid = identifierValue; + } else if (identifierType === 'UserId') { + updatedData.UserId = identifierValue; + } + return updatedData; +}; + +module.exports = { + setIdentifier, +}; diff --git a/test/integrations/destinations/yandex_metrica_offline_events/processor/data.ts b/test/integrations/destinations/yandex_metrica_offline_events/processor/data.ts new file mode 100644 index 0000000000..dc0b5cb045 --- /dev/null +++ b/test/integrations/destinations/yandex_metrica_offline_events/processor/data.ts @@ -0,0 +1,296 @@ +export const data = [ + { + name: 'yandex_metrica_offline_events', + description: 'Successful identify event with YCLID identifier type', + feature: 'processor', + module: 'destination', + version: 'v0', + input: { + request: { + body: [ + { + message: { + type: 'identify', + traits: { + Price: '100', + Target: 'GOAL1', + Currency: 'RUB', + DateTime: '1481718166', + }, + userId: '', + channel: 'sources', + context: { + sources: { + job_id: '2du7rQyxlbIJl4LKgZAaaErEjcE', + version: '1849/merge', + job_run_id: 'cnsn3tt5fleigsfclr6g', + task_run_id: 'cnsn3tt5fleigsfclr70', + }, + externalId: [ + { + id: '133591247640966458', + type: 'YANDEX_METRICA_OFFLINE_EVENTS-conversions', + identifierType: 'YCLID', + }, + ], + mappedToDestination: 'true', + }, + recordId: '1', + rudderId: '14a58046-23a5-46bd-afbf-87c8acaa9d2e', + messageId: '91ef85d9-b170-440c-bae2-6284d4070338', + }, + destination: { + DestinationDefinition: { + Config: { + cdkV2Enabled: true, + }, + }, + Config: { + counterId: '574342423', + goalId: '23432565', + rudderAccountId: '2du7fLeK82nk9L2Xd1X507uiD1B', + authStatus: 'active', + }, + }, + metadata: { + destinationId: 'destId', + workspaceId: 'wspId', + }, + }, + ], + }, + }, + output: { + response: { + status: 200, + body: [ + { + output: { + version: '1', + type: 'REST', + method: 'POST', + endpoint: '', + headers: {}, + params: {}, + body: { + JSON: { + Currency: 'RUB', + DateTime: '1481718166', + Price: '100', + Target: 'GOAL1', + Yclid: '133591247640966458', + }, + JSON_ARRAY: {}, + XML: {}, + FORM: {}, + }, + files: {}, + userId: '', + }, + statusCode: 200, + metadata: { + destinationId: 'destId', + workspaceId: 'wspId', + }, + }, + ], + }, + }, + }, + { + name: 'yandex_metrica_offline_events', + description: 'Successful identify event with ClientId identifier type', + feature: 'processor', + module: 'destination', + version: 'v0', + input: { + request: { + body: [ + { + message: { + type: 'identify', + traits: { + Price: '100', + Target: 'GOAL1', + Currency: 'RUB', + DateTime: '1481718166', + }, + userId: '', + channel: 'sources', + context: { + sources: { + job_id: '2du7rQyxlbIJl4LKgZAaaErEjcE', + version: '1849/merge', + job_run_id: 'cnsn3tt5fleigsfclr6g', + task_run_id: 'cnsn3tt5fleigsfclr70', + }, + externalId: [ + { + id: '133591247640966458', + type: 'YANDEX_METRICA_OFFLINE_EVENTS-conversions', + identifierType: 'ClientId', + }, + ], + mappedToDestination: 'true', + }, + recordId: '1', + rudderId: '14a58046-23a5-46bd-afbf-87c8acaa9d2e', + messageId: '91ef85d9-b170-440c-bae2-6284d4070338', + }, + destination: { + DestinationDefinition: { + Config: { + cdkV2Enabled: true, + }, + }, + Config: { + counterId: '574342423', + goalId: '23432565', + rudderAccountId: '2du7fLeK82nk9L2Xd1X507uiD1B', + authStatus: 'active', + }, + }, + metadata: { + destinationId: 'destId', + workspaceId: 'wspId', + }, + }, + ], + }, + }, + output: { + response: { + status: 200, + body: [ + { + output: { + version: '1', + type: 'REST', + method: 'POST', + endpoint: '', + headers: {}, + params: {}, + body: { + JSON: { + Currency: 'RUB', + DateTime: '1481718166', + Price: '100', + Target: 'GOAL1', + ClientId: '133591247640966458', + }, + JSON_ARRAY: {}, + XML: {}, + FORM: {}, + }, + files: {}, + userId: '', + }, + statusCode: 200, + metadata: { + destinationId: 'destId', + workspaceId: 'wspId', + }, + }, + ], + }, + }, + }, + { + name: 'yandex_metrica_offline_events', + description: 'Successful identify event with UserId identifier type', + feature: 'processor', + module: 'destination', + version: 'v0', + input: { + request: { + body: [ + { + message: { + type: 'identify', + traits: { + Price: '100', + Target: 'GOAL1', + Currency: 'RUB', + DateTime: '1481718166', + }, + userId: '', + channel: 'sources', + context: { + sources: { + job_id: '2du7rQyxlbIJl4LKgZAaaErEjcE', + version: '1849/merge', + job_run_id: 'cnsn3tt5fleigsfclr6g', + task_run_id: 'cnsn3tt5fleigsfclr70', + }, + externalId: [ + { + id: '133591247640966458', + type: 'YANDEX_METRICA_OFFLINE_EVENTS-conversions', + identifierType: 'UserId', + }, + ], + mappedToDestination: 'true', + }, + recordId: '1', + rudderId: '14a58046-23a5-46bd-afbf-87c8acaa9d2e', + messageId: '91ef85d9-b170-440c-bae2-6284d4070338', + }, + destination: { + DestinationDefinition: { + Config: { + cdkV2Enabled: true, + }, + }, + Config: { + counterId: '574342423', + goalId: '23432565', + rudderAccountId: '2du7fLeK82nk9L2Xd1X507uiD1B', + authStatus: 'active', + }, + }, + metadata: { + destinationId: 'destId', + workspaceId: 'wspId', + }, + }, + ], + }, + }, + output: { + response: { + status: 200, + body: [ + { + output: { + version: '1', + type: 'REST', + method: 'POST', + endpoint: '', + headers: {}, + params: {}, + body: { + JSON: { + Currency: 'RUB', + DateTime: '1481718166', + Price: '100', + Target: 'GOAL1', + UserId: '133591247640966458', + }, + JSON_ARRAY: {}, + XML: {}, + FORM: {}, + }, + files: {}, + userId: '', + }, + statusCode: 200, + metadata: { + destinationId: 'destId', + workspaceId: 'wspId', + }, + }, + ], + }, + }, + }, +]; From 940884324d39574d6e10e7debc0e1815223fd4f6 Mon Sep 17 00:00:00 2001 From: Yashasvi Bajpai <33063622+yashasvibajpai@users.noreply.github.com> Date: Tue, 16 Apr 2024 11:28:52 +0530 Subject: [PATCH 3/8] chore: address commentsx1 --- .../yandex_metrica_offline_events/procWorkflow.yaml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/cdk/v2/destinations/yandex_metrica_offline_events/procWorkflow.yaml b/src/cdk/v2/destinations/yandex_metrica_offline_events/procWorkflow.yaml index 18cd15d42d..453b4dd2dc 100644 --- a/src/cdk/v2/destinations/yandex_metrica_offline_events/procWorkflow.yaml +++ b/src/cdk/v2/destinations/yandex_metrica_offline_events/procWorkflow.yaml @@ -15,7 +15,7 @@ steps: template: | let messageType = .message.type; $.assert(messageType, "message Type is not present. Aborting message."); - $.assert(.message.type.toLowerCase() ==='identify' || .message.type.toLowerCase() ==='track', "Event type " + .message.type.toLowerCase() + " is not supported. Aborting message."); + $.assert(.message.type.toLowerCase() ==='identify', "Event type " + .message.type.toLowerCase() + " is not supported. Aborting message."); $.assert(.message.traits || .message.properties, "Message traits/properties not present. Aborting message."); - name: prepareData @@ -31,5 +31,4 @@ steps: template: | const response = $.defaultRequestConfig(); response.body.JSON = $.outputs.prepareData - console.log(JSON.stringify(response)); response From b61cbc164e0dad7af441fadbe81dc17116ccccbf Mon Sep 17 00:00:00 2001 From: Yashasvi Bajpai <33063622+yashasvibajpai@users.noreply.github.com> Date: Thu, 18 Apr 2024 12:07:38 +0530 Subject: [PATCH 4/8] chore: add validations and tests --- .../procWorkflow.yaml | 1 + .../yandex_metrica_offline_events/utils.js | 18 ++++ .../processor/data.ts | 100 ++++++++++++++++-- 3 files changed, 113 insertions(+), 6 deletions(-) diff --git a/src/cdk/v2/destinations/yandex_metrica_offline_events/procWorkflow.yaml b/src/cdk/v2/destinations/yandex_metrica_offline_events/procWorkflow.yaml index 453b4dd2dc..9affe40df5 100644 --- a/src/cdk/v2/destinations/yandex_metrica_offline_events/procWorkflow.yaml +++ b/src/cdk/v2/destinations/yandex_metrica_offline_events/procWorkflow.yaml @@ -21,6 +21,7 @@ steps: - name: prepareData template: | let data = .message.traits + data = $.validateData(data) let identifierType = .message.context.externalId[0].identifierType; let identifierValue = .message.context.externalId[0].id; data = $.setIdentifier(data, identifierType, identifierValue) diff --git a/src/cdk/v2/destinations/yandex_metrica_offline_events/utils.js b/src/cdk/v2/destinations/yandex_metrica_offline_events/utils.js index e3e3b2107e..6cd63e9410 100644 --- a/src/cdk/v2/destinations/yandex_metrica_offline_events/utils.js +++ b/src/cdk/v2/destinations/yandex_metrica_offline_events/utils.js @@ -1,3 +1,5 @@ +const { InstrumentationError } = require('@rudderstack/integrations-lib'); + const setIdentifier = (data, identifierType, identifierValue) => { const updatedData = data; if (identifierType === 'ClientId') { @@ -6,10 +8,26 @@ const setIdentifier = (data, identifierType, identifierValue) => { updatedData.Yclid = identifierValue; } else if (identifierType === 'UserId') { updatedData.UserId = identifierValue; + } else { + throw new InstrumentationError( + 'Invalid identifier type passed in external Id. Valid types are ClientId, YCLID, UserId. Aborting!', + ); } return updatedData; }; +const validateData = (data) => { + const { Price } = data; + if (!data) { + throw new InstrumentationError('No traits found in the payload. Aborting!'); + } + if (Price && typeof Price !== 'number') { + throw new InstrumentationError('Price can only be a numerical value. Aborting!'); + } + return data; +}; + module.exports = { setIdentifier, + validateData, }; diff --git a/test/integrations/destinations/yandex_metrica_offline_events/processor/data.ts b/test/integrations/destinations/yandex_metrica_offline_events/processor/data.ts index dc0b5cb045..dce8d7fa7d 100644 --- a/test/integrations/destinations/yandex_metrica_offline_events/processor/data.ts +++ b/test/integrations/destinations/yandex_metrica_offline_events/processor/data.ts @@ -12,7 +12,7 @@ export const data = [ message: { type: 'identify', traits: { - Price: '100', + Price: 100, Target: 'GOAL1', Currency: 'RUB', DateTime: '1481718166', @@ -76,7 +76,7 @@ export const data = [ JSON: { Currency: 'RUB', DateTime: '1481718166', - Price: '100', + Price: 100, Target: 'GOAL1', Yclid: '133591247640966458', }, @@ -110,7 +110,7 @@ export const data = [ message: { type: 'identify', traits: { - Price: '100', + Price: 100, Target: 'GOAL1', Currency: 'RUB', DateTime: '1481718166', @@ -174,7 +174,7 @@ export const data = [ JSON: { Currency: 'RUB', DateTime: '1481718166', - Price: '100', + Price: 100, Target: 'GOAL1', ClientId: '133591247640966458', }, @@ -208,7 +208,7 @@ export const data = [ message: { type: 'identify', traits: { - Price: '100', + Price: 100, Target: 'GOAL1', Currency: 'RUB', DateTime: '1481718166', @@ -272,7 +272,7 @@ export const data = [ JSON: { Currency: 'RUB', DateTime: '1481718166', - Price: '100', + Price: 100, Target: 'GOAL1', UserId: '133591247640966458', }, @@ -293,4 +293,92 @@ export const data = [ }, }, }, + { + name: 'yandex_metrica_offline_events', + description: 'Successful identify event with UserId identifier type', + feature: 'processor', + module: 'destination', + version: 'v0', + input: { + request: { + body: [ + { + message: { + type: 'identify', + traits: { + Price: '100', + Target: 'GOAL1', + Currency: 'RUB', + DateTime: '1481718166', + }, + userId: '', + channel: 'sources', + context: { + sources: { + job_id: '2du7rQyxlbIJl4LKgZAaaErEjcE', + version: '1849/merge', + job_run_id: 'cnsn3tt5fleigsfclr6g', + task_run_id: 'cnsn3tt5fleigsfclr70', + }, + externalId: [ + { + id: '133591247640966458', + type: 'YANDEX_METRICA_OFFLINE_EVENTS-conversions', + identifierType: 'UserId', + }, + ], + mappedToDestination: 'true', + }, + recordId: '1', + rudderId: '14a58046-23a5-46bd-afbf-87c8acaa9d2e', + messageId: '91ef85d9-b170-440c-bae2-6284d4070338', + }, + destination: { + DestinationDefinition: { + Config: { + cdkV2Enabled: true, + }, + }, + Config: { + counterId: '574342423', + goalId: '23432565', + rudderAccountId: '2du7fLeK82nk9L2Xd1X507uiD1B', + authStatus: 'active', + }, + }, + metadata: { + destinationId: 'destId', + workspaceId: 'wspId', + }, + }, + ], + }, + }, + output: { + response: { + status: 200, + body: [ + { + statusCode: 400, + error: + 'Price can only be a numerical value. Aborting!: Workflow: procWorkflow, Step: prepareData, ChildStep: undefined, OriginalError: Price can only be a numerical value. Aborting!', + statTags: { + destType: 'YANDEX_METRICA_OFFLINE_EVENTS', + destinationId: 'destId', + errorCategory: 'dataValidation', + errorType: 'instrumentation', + feature: 'processor', + implementation: 'cdkV2', + module: 'destination', + workspaceId: 'wspId', + }, + metadata: { + destinationId: 'destId', + workspaceId: 'wspId', + }, + }, + ], + }, + }, + }, ]; From 8541223a6f0794b939028e05e695544769cd12d5 Mon Sep 17 00:00:00 2001 From: Yashasvi Bajpai <33063622+yashasvibajpai@users.noreply.github.com> Date: Thu, 18 Apr 2024 18:56:31 +0530 Subject: [PATCH 5/8] chore: address commentsx2 --- .../yandex_metrica_offline_events/utils.js | 4 +- .../processor/data.ts | 173 +++++++++++++++++- 2 files changed, 174 insertions(+), 3 deletions(-) diff --git a/src/cdk/v2/destinations/yandex_metrica_offline_events/utils.js b/src/cdk/v2/destinations/yandex_metrica_offline_events/utils.js index 6cd63e9410..b79f55da3a 100644 --- a/src/cdk/v2/destinations/yandex_metrica_offline_events/utils.js +++ b/src/cdk/v2/destinations/yandex_metrica_offline_events/utils.js @@ -1,4 +1,4 @@ -const { InstrumentationError } = require('@rudderstack/integrations-lib'); +const { InstrumentationError, isDefinedNotNullNotEmpty } = require('@rudderstack/integrations-lib'); const setIdentifier = (data, identifierType, identifierValue) => { const updatedData = data; @@ -18,7 +18,7 @@ const setIdentifier = (data, identifierType, identifierValue) => { const validateData = (data) => { const { Price } = data; - if (!data) { + if (!isDefinedNotNullNotEmpty(data)) { throw new InstrumentationError('No traits found in the payload. Aborting!'); } if (Price && typeof Price !== 'number') { diff --git a/test/integrations/destinations/yandex_metrica_offline_events/processor/data.ts b/test/integrations/destinations/yandex_metrica_offline_events/processor/data.ts index dce8d7fa7d..b6dbf2b731 100644 --- a/test/integrations/destinations/yandex_metrica_offline_events/processor/data.ts +++ b/test/integrations/destinations/yandex_metrica_offline_events/processor/data.ts @@ -295,7 +295,7 @@ export const data = [ }, { name: 'yandex_metrica_offline_events', - description: 'Successful identify event with UserId identifier type', + description: 'Failed identify event with Price passed as string', feature: 'processor', module: 'destination', version: 'v0', @@ -381,4 +381,175 @@ export const data = [ }, }, }, + { + name: 'yandex_metrica_offline_events', + description: 'Failed identify event with traits missing', + feature: 'processor', + module: 'destination', + version: 'v0', + input: { + request: { + body: [ + { + message: { + type: 'identify', + traits: {}, + userId: '', + channel: 'sources', + context: { + sources: { + job_id: '2du7rQyxlbIJl4LKgZAaaErEjcE', + version: '1849/merge', + job_run_id: 'cnsn3tt5fleigsfclr6g', + task_run_id: 'cnsn3tt5fleigsfclr70', + }, + externalId: [ + { + id: '133591247640966458', + type: 'YANDEX_METRICA_OFFLINE_EVENTS-conversions', + identifierType: 'UserId', + }, + ], + mappedToDestination: 'true', + }, + recordId: '1', + rudderId: '14a58046-23a5-46bd-afbf-87c8acaa9d2e', + messageId: '91ef85d9-b170-440c-bae2-6284d4070338', + }, + destination: { + DestinationDefinition: { + Config: { + cdkV2Enabled: true, + }, + }, + Config: { + counterId: '574342423', + goalId: '23432565', + rudderAccountId: '2du7fLeK82nk9L2Xd1X507uiD1B', + authStatus: 'active', + }, + }, + metadata: { + destinationId: 'destId', + workspaceId: 'wspId', + }, + }, + ], + }, + }, + output: { + response: { + status: 200, + body: [ + { + statusCode: 400, + error: + 'No traits found in the payload. Aborting!: Workflow: procWorkflow, Step: prepareData, ChildStep: undefined, OriginalError: No traits found in the payload. Aborting!', + statTags: { + destType: 'YANDEX_METRICA_OFFLINE_EVENTS', + destinationId: 'destId', + errorCategory: 'dataValidation', + errorType: 'instrumentation', + feature: 'processor', + implementation: 'cdkV2', + module: 'destination', + workspaceId: 'wspId', + }, + metadata: { + destinationId: 'destId', + workspaceId: 'wspId', + }, + }, + ], + }, + }, + }, + { + name: 'yandex_metrica_offline_events', + description: 'Failed identify event with invalid identifier type', + feature: 'processor', + module: 'destination', + version: 'v0', + input: { + request: { + body: [ + { + message: { + type: 'identify', + traits: { + Price: 100, + Target: 'GOAL1', + Currency: 'RUB', + DateTime: '1481718166', + }, + userId: '', + channel: 'sources', + context: { + sources: { + job_id: '2du7rQyxlbIJl4LKgZAaaErEjcE', + version: '1849/merge', + job_run_id: 'cnsn3tt5fleigsfclr6g', + task_run_id: 'cnsn3tt5fleigsfclr70', + }, + externalId: [ + { + id: '133591247640966458', + type: 'YANDEX_METRICA_OFFLINE_EVENTS-conversions', + identifierType: 'InvalidIdentifierType', + }, + ], + mappedToDestination: 'true', + }, + recordId: '1', + rudderId: '14a58046-23a5-46bd-afbf-87c8acaa9d2e', + messageId: '91ef85d9-b170-440c-bae2-6284d4070338', + }, + destination: { + DestinationDefinition: { + Config: { + cdkV2Enabled: true, + }, + }, + Config: { + counterId: '574342423', + goalId: '23432565', + rudderAccountId: '2du7fLeK82nk9L2Xd1X507uiD1B', + authStatus: 'active', + }, + }, + metadata: { + destinationId: 'destId', + workspaceId: 'wspId', + }, + }, + ], + }, + }, + output: { + response: { + status: 200, + body: [ + { + statusCode: 400, + error: + 'Invalid identifier type passed in external Id. Valid types are ClientId, YCLID, UserId. Aborting!: Workflow: procWorkflow, Step: prepareData, ChildStep: undefined, OriginalError: Invalid identifier type passed in external Id. Valid types are ClientId, YCLID, UserId. Aborting!', + statTags: { + destType: 'YANDEX_METRICA_OFFLINE_EVENTS', + destinationId: 'destId', + errorCategory: 'dataValidation', + errorType: 'instrumentation', + feature: 'processor', + implementation: 'cdkV2', + module: 'destination', + workspaceId: 'wspId', + }, + metadata: { + destinationId: 'destId', + workspaceId: 'wspId', + }, + }, + ], + }, + }, + }, ]; From e57a2fb0ab034206e1f6756fc5a8ca1635629a87 Mon Sep 17 00:00:00 2001 From: Yashasvi Bajpai <33063622+yashasvibajpai@users.noreply.github.com> Date: Tue, 23 Apr 2024 14:21:11 +0530 Subject: [PATCH 6/8] chore: add validations for tiemstamp and id --- .../procWorkflow.yaml | 3 +- .../yandex_metrica_offline_events/utils.js | 19 ++- .../processor/data.ts | 119 ++++++++++++++++-- 3 files changed, 131 insertions(+), 10 deletions(-) diff --git a/src/cdk/v2/destinations/yandex_metrica_offline_events/procWorkflow.yaml b/src/cdk/v2/destinations/yandex_metrica_offline_events/procWorkflow.yaml index 9affe40df5..690bc399ee 100644 --- a/src/cdk/v2/destinations/yandex_metrica_offline_events/procWorkflow.yaml +++ b/src/cdk/v2/destinations/yandex_metrica_offline_events/procWorkflow.yaml @@ -21,10 +21,11 @@ steps: - name: prepareData template: | let data = .message.traits - data = $.validateData(data) let identifierType = .message.context.externalId[0].identifierType; let identifierValue = .message.context.externalId[0].id; + identifierValue = String(identifierValue); data = $.setIdentifier(data, identifierType, identifierValue) + data = $.validateData(data) data - name: buildResponseForProcessTransformation diff --git a/src/cdk/v2/destinations/yandex_metrica_offline_events/utils.js b/src/cdk/v2/destinations/yandex_metrica_offline_events/utils.js index b79f55da3a..fd6642a9d5 100644 --- a/src/cdk/v2/destinations/yandex_metrica_offline_events/utils.js +++ b/src/cdk/v2/destinations/yandex_metrica_offline_events/utils.js @@ -1,4 +1,6 @@ +/* eslint-disable no-param-reassign */ const { InstrumentationError, isDefinedNotNullNotEmpty } = require('@rudderstack/integrations-lib'); +const moment = require('moment'); const setIdentifier = (data, identifierType, identifierValue) => { const updatedData = data; @@ -16,18 +18,33 @@ const setIdentifier = (data, identifierType, identifierValue) => { return updatedData; }; +function isUnixTimestamp(datetime) { + if (moment.unix(datetime).isValid()) { + return datetime; + } + const unixTimestamp = moment(datetime).unix(); + if (moment.unix(unixTimestamp).isValid()) { + return unixTimestamp; + } + throw new InstrumentationError('Invalid timestamp. Aborting!'); +} + const validateData = (data) => { - const { Price } = data; + const { Price, DateTime } = data; if (!isDefinedNotNullNotEmpty(data)) { throw new InstrumentationError('No traits found in the payload. Aborting!'); } if (Price && typeof Price !== 'number') { throw new InstrumentationError('Price can only be a numerical value. Aborting!'); } + if (DateTime) { + data.DateTime = String(isUnixTimestamp(DateTime)); + } return data; }; module.exports = { setIdentifier, validateData, + isUnixTimestamp, }; diff --git a/test/integrations/destinations/yandex_metrica_offline_events/processor/data.ts b/test/integrations/destinations/yandex_metrica_offline_events/processor/data.ts index b6dbf2b731..fc027e71bc 100644 --- a/test/integrations/destinations/yandex_metrica_offline_events/processor/data.ts +++ b/test/integrations/destinations/yandex_metrica_offline_events/processor/data.ts @@ -383,7 +383,7 @@ export const data = [ }, { name: 'yandex_metrica_offline_events', - description: 'Failed identify event with traits missing', + description: 'Failed identify event with invalid identifier type', feature: 'processor', module: 'destination', version: 'v0', @@ -393,7 +393,12 @@ export const data = [ { message: { type: 'identify', - traits: {}, + traits: { + Price: 100, + Target: 'GOAL1', + Currency: 'RUB', + DateTime: '1481718166', + }, userId: '', channel: 'sources', context: { @@ -407,7 +412,7 @@ export const data = [ { id: '133591247640966458', type: 'YANDEX_METRICA_OFFLINE_EVENTS-conversions', - identifierType: 'UserId', + identifierType: 'InvalidIdentifierType', }, ], mappedToDestination: 'true', @@ -444,7 +449,7 @@ export const data = [ { statusCode: 400, error: - 'No traits found in the payload. Aborting!: Workflow: procWorkflow, Step: prepareData, ChildStep: undefined, OriginalError: No traits found in the payload. Aborting!', + 'Invalid identifier type passed in external Id. Valid types are ClientId, YCLID, UserId. Aborting!: Workflow: procWorkflow, Step: prepareData, ChildStep: undefined, OriginalError: Invalid identifier type passed in external Id. Valid types are ClientId, YCLID, UserId. Aborting!', statTags: { destType: 'YANDEX_METRICA_OFFLINE_EVENTS', destinationId: 'destId', @@ -466,7 +471,7 @@ export const data = [ }, { name: 'yandex_metrica_offline_events', - description: 'Failed identify event with invalid identifier type', + description: 'Failed identify event with invalid timestamp', feature: 'processor', module: 'destination', version: 'v0', @@ -480,7 +485,7 @@ export const data = [ Price: 100, Target: 'GOAL1', Currency: 'RUB', - DateTime: '1481718166', + DateTime: 'invalidTimestamp', }, userId: '', channel: 'sources', @@ -495,7 +500,7 @@ export const data = [ { id: '133591247640966458', type: 'YANDEX_METRICA_OFFLINE_EVENTS-conversions', - identifierType: 'InvalidIdentifierType', + identifierType: 'ClientId', }, ], mappedToDestination: 'true', @@ -532,7 +537,7 @@ export const data = [ { statusCode: 400, error: - 'Invalid identifier type passed in external Id. Valid types are ClientId, YCLID, UserId. Aborting!: Workflow: procWorkflow, Step: prepareData, ChildStep: undefined, OriginalError: Invalid identifier type passed in external Id. Valid types are ClientId, YCLID, UserId. Aborting!', + 'Invalid timestamp. Aborting!: Workflow: procWorkflow, Step: prepareData, ChildStep: undefined, OriginalError: Invalid timestamp. Aborting!', statTags: { destType: 'YANDEX_METRICA_OFFLINE_EVENTS', destinationId: 'destId', @@ -552,4 +557,102 @@ export const data = [ }, }, }, + { + name: 'yandex_metrica_offline_events', + description: 'Successful identify event with non unix timestamp', + feature: 'processor', + module: 'destination', + version: 'v0', + input: { + request: { + body: [ + { + message: { + type: 'identify', + traits: { + Price: 100, + Target: 'GOAL1', + Currency: 'RUB', + DateTime: '2023-08-14T05:30:30.118Z', + }, + userId: '', + channel: 'sources', + context: { + sources: { + job_id: '2du7rQyxlbIJl4LKgZAaaErEjcE', + version: '1849/merge', + job_run_id: 'cnsn3tt5fleigsfclr6g', + task_run_id: 'cnsn3tt5fleigsfclr70', + }, + externalId: [ + { + id: '133591247640966458', + type: 'YANDEX_METRICA_OFFLINE_EVENTS-conversions', + identifierType: 'YCLID', + }, + ], + mappedToDestination: 'true', + }, + recordId: '1', + rudderId: '14a58046-23a5-46bd-afbf-87c8acaa9d2e', + messageId: '91ef85d9-b170-440c-bae2-6284d4070338', + }, + destination: { + DestinationDefinition: { + Config: { + cdkV2Enabled: true, + }, + }, + Config: { + counterId: '574342423', + goalId: '23432565', + rudderAccountId: '2du7fLeK82nk9L2Xd1X507uiD1B', + authStatus: 'active', + }, + }, + metadata: { + destinationId: 'destId', + workspaceId: 'wspId', + }, + }, + ], + }, + }, + output: { + response: { + status: 200, + body: [ + { + output: { + version: '1', + type: 'REST', + method: 'POST', + endpoint: '', + headers: {}, + params: {}, + body: { + JSON: { + Currency: 'RUB', + DateTime: '1691991030', + Price: 100, + Target: 'GOAL1', + Yclid: '133591247640966458', + }, + JSON_ARRAY: {}, + XML: {}, + FORM: {}, + }, + files: {}, + userId: '', + }, + statusCode: 200, + metadata: { + destinationId: 'destId', + workspaceId: 'wspId', + }, + }, + ], + }, + }, + }, ]; From 7b97eb10283da0774440d9032fbdcc7455f1c936 Mon Sep 17 00:00:00 2001 From: Yashasvi Bajpai <33063622+yashasvibajpai@users.noreply.github.com> Date: Tue, 23 Apr 2024 15:22:35 +0530 Subject: [PATCH 7/8] chore: add null check for DateTime --- src/cdk/v2/destinations/yandex_metrica_offline_events/utils.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/cdk/v2/destinations/yandex_metrica_offline_events/utils.js b/src/cdk/v2/destinations/yandex_metrica_offline_events/utils.js index fd6642a9d5..426e073ae6 100644 --- a/src/cdk/v2/destinations/yandex_metrica_offline_events/utils.js +++ b/src/cdk/v2/destinations/yandex_metrica_offline_events/utils.js @@ -37,7 +37,7 @@ const validateData = (data) => { if (Price && typeof Price !== 'number') { throw new InstrumentationError('Price can only be a numerical value. Aborting!'); } - if (DateTime) { + if (isDefinedNotNullNotEmpty(DateTime)) { data.DateTime = String(isUnixTimestamp(DateTime)); } return data; From 5bd68114b29ed004bee0a07e28f3c47c87372218 Mon Sep 17 00:00:00 2001 From: Yashasvi Bajpai <33063622+yashasvibajpai@users.noreply.github.com> Date: Tue, 23 Apr 2024 15:33:49 +0530 Subject: [PATCH 8/8] chore: add null error throw for DateTime --- .../yandex_metrica_offline_events/utils.js | 5 +- .../processor/data.ts | 88 +++++++++++++++++++ 2 files changed, 91 insertions(+), 2 deletions(-) diff --git a/src/cdk/v2/destinations/yandex_metrica_offline_events/utils.js b/src/cdk/v2/destinations/yandex_metrica_offline_events/utils.js index 426e073ae6..032b0b636d 100644 --- a/src/cdk/v2/destinations/yandex_metrica_offline_events/utils.js +++ b/src/cdk/v2/destinations/yandex_metrica_offline_events/utils.js @@ -37,9 +37,10 @@ const validateData = (data) => { if (Price && typeof Price !== 'number') { throw new InstrumentationError('Price can only be a numerical value. Aborting!'); } - if (isDefinedNotNullNotEmpty(DateTime)) { - data.DateTime = String(isUnixTimestamp(DateTime)); + if (!isDefinedNotNullNotEmpty(DateTime)) { + throw new InstrumentationError('DateTime cannot be empty. Aborting!'); } + data.DateTime = String(isUnixTimestamp(DateTime)); return data; }; diff --git a/test/integrations/destinations/yandex_metrica_offline_events/processor/data.ts b/test/integrations/destinations/yandex_metrica_offline_events/processor/data.ts index fc027e71bc..4bb27bdf57 100644 --- a/test/integrations/destinations/yandex_metrica_offline_events/processor/data.ts +++ b/test/integrations/destinations/yandex_metrica_offline_events/processor/data.ts @@ -655,4 +655,92 @@ export const data = [ }, }, }, + { + name: 'yandex_metrica_offline_events', + description: 'Failed identify event with null or empty timestamp', + feature: 'processor', + module: 'destination', + version: 'v0', + input: { + request: { + body: [ + { + message: { + type: 'identify', + traits: { + Price: 100, + Target: 'GOAL1', + Currency: 'RUB', + DateTime: '', + }, + userId: '', + channel: 'sources', + context: { + sources: { + job_id: '2du7rQyxlbIJl4LKgZAaaErEjcE', + version: '1849/merge', + job_run_id: 'cnsn3tt5fleigsfclr6g', + task_run_id: 'cnsn3tt5fleigsfclr70', + }, + externalId: [ + { + id: '133591247640966458', + type: 'YANDEX_METRICA_OFFLINE_EVENTS-conversions', + identifierType: 'ClientId', + }, + ], + mappedToDestination: 'true', + }, + recordId: '1', + rudderId: '14a58046-23a5-46bd-afbf-87c8acaa9d2e', + messageId: '91ef85d9-b170-440c-bae2-6284d4070338', + }, + destination: { + DestinationDefinition: { + Config: { + cdkV2Enabled: true, + }, + }, + Config: { + counterId: '574342423', + goalId: '23432565', + rudderAccountId: '2du7fLeK82nk9L2Xd1X507uiD1B', + authStatus: 'active', + }, + }, + metadata: { + destinationId: 'destId', + workspaceId: 'wspId', + }, + }, + ], + }, + }, + output: { + response: { + status: 200, + body: [ + { + statusCode: 400, + error: + 'DateTime cannot be empty. Aborting!: Workflow: procWorkflow, Step: prepareData, ChildStep: undefined, OriginalError: DateTime cannot be empty. Aborting!', + statTags: { + destType: 'YANDEX_METRICA_OFFLINE_EVENTS', + destinationId: 'destId', + errorCategory: 'dataValidation', + errorType: 'instrumentation', + feature: 'processor', + implementation: 'cdkV2', + module: 'destination', + workspaceId: 'wspId', + }, + metadata: { + destinationId: 'destId', + workspaceId: 'wspId', + }, + }, + ], + }, + }, + }, ];