diff --git a/.github/workflows/create-hotfix-branch.yml b/.github/workflows/create-hotfix-branch.yml index d1397cb608..97611f1eee 100644 --- a/.github/workflows/create-hotfix-branch.yml +++ b/.github/workflows/create-hotfix-branch.yml @@ -13,7 +13,7 @@ jobs: runs-on: ubuntu-latest # Only allow these users to create new hotfix branch from 'main' - if: github.ref == 'refs/heads/main' && (github.actor == 'ItsSudip' || github.actor == 'krishna2020' || github.actor == 'saikumarrs' || github.actor == 'sandeepdsvs' || github.actor == 'shrouti1507' || github.actor == 'anantjain45823' || github.actor == 'chandumlg' || github.actor == 'mihir-4116') && (github.triggering_actor == 'ItsSudip' || github.triggering_actor == 'krishna2020' || github.triggering_actor == 'saikumarrs' || github.triggering_actor == 'sandeepdsvs' || github.triggering_actor == 'shrouti1507' || github.triggering_actor == 'anantjain45823' || github.triggering_actor == 'chandumlg' || github.triggering_actor == 'mihir-4116') + if: github.ref == 'refs/heads/main' && (github.actor == 'ItsSudip' || github.actor == 'krishna2020' || github.actor == 'saikumarrs' || github.actor == 'sandeepdsvs' || github.actor == 'shrouti1507' || github.actor == 'anantjain45823' || github.actor == 'chandumlg' || github.actor == 'mihir-4116' || github.actor == 'ujjwal-ab') && (github.triggering_actor == 'ItsSudip' || github.triggering_actor == 'krishna2020' || github.triggering_actor == 'saikumarrs' || github.triggering_actor == 'sandeepdsvs' || github.triggering_actor == 'shrouti1507' || github.triggering_actor == 'anantjain45823' || github.triggering_actor == 'chandumlg' || github.triggering_actor == 'mihir-4116' || github.triggering_actor == 'ujjwal-ab) steps: - name: Create Branch uses: peterjgrainger/action-create-branch@v2.4.0 diff --git a/.github/workflows/draft-new-release.yml b/.github/workflows/draft-new-release.yml index a0a558440a..23e243918f 100644 --- a/.github/workflows/draft-new-release.yml +++ b/.github/workflows/draft-new-release.yml @@ -8,7 +8,7 @@ jobs: runs-on: ubuntu-latest # Only allow release stakeholders to initiate releases - if: (github.ref == 'refs/heads/develop' || startsWith(github.ref, 'refs/heads/hotfix/')) && (github.actor == 'ItsSudip' || github.actor == 'krishna2020' || github.actor == 'saikumarrs' || github.actor == 'sandeepdsvs' || github.actor == 'shrouti1507' || github.actor == 'anantjain45823' || github.actor == 'chandumlg' || github.actor == 'mihir-4116' || github.actor == 'yashasvibajpai' || github.actor == 'sanpj2292') && (github.triggering_actor == 'ItsSudip' || github.triggering_actor == 'krishna2020' || github.triggering_actor == 'saikumarrs' || github.triggering_actor == 'sandeepdsvs' || github.triggering_actor == 'shrouti1507' || github.triggering_actor == 'anantjain45823' || github.triggering_actor == 'chandumlg' || github.triggering_actor == 'mihir-4116' || github.triggering_actor == 'yashasvibajpai' || github.triggering_actor == 'sanpj2292') + if: (github.ref == 'refs/heads/develop' || startsWith(github.ref, 'refs/heads/hotfix/')) && (github.actor == 'ItsSudip' || github.actor == 'krishna2020' || github.actor == 'saikumarrs' || github.actor == 'sandeepdsvs' || github.actor == 'shrouti1507' || github.actor == 'anantjain45823' || github.actor == 'chandumlg' || github.actor == 'mihir-4116' || github.actor == 'yashasvibajpai' || github.actor == 'sanpj2292' || github.actor == 'ujjwal-ab') && (github.triggering_actor == 'ItsSudip' || github.triggering_actor == 'krishna2020' || github.triggering_actor == 'saikumarrs' || github.triggering_actor == 'sandeepdsvs' || github.triggering_actor == 'shrouti1507' || github.triggering_actor == 'anantjain45823' || github.triggering_actor == 'chandumlg' || github.triggering_actor == 'mihir-4116' || github.triggering_actor == 'yashasvibajpai' || github.triggering_actor == 'sanpj2292' || github.triggering_actor == 'ujjwal-ab') steps: - name: Checkout uses: actions/checkout@v3.5.3 diff --git a/CHANGELOG.md b/CHANGELOG.md index e7d295645f..01631435d2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,13 @@ All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines. +## [1.48.0](https://github.com/rudderlabs/rudder-transformer/compare/v1.47.0...v1.48.0) (2023-11-02) + + +### Features + +* add support to add custom network policies for specific workspaces in faas pods ([bc1a760](https://github.com/rudderlabs/rudder-transformer/commit/bc1a76066c0aeb43776ded0b266ec48f5e69aa16)) + ## [1.47.0](https://github.com/rudderlabs/rudder-transformer/compare/v1.46.5...v1.47.0) (2023-10-30) diff --git a/package-lock.json b/package-lock.json index 4643d1325d..0822a9b42b 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "rudder-transformer", - "version": "1.47.0", + "version": "1.48.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "rudder-transformer", - "version": "1.47.0", + "version": "1.48.0", "license": "ISC", "dependencies": { "@amplitude/ua-parser-js": "^0.7.24", diff --git a/package.json b/package.json index adc5f0e8f5..46f728664d 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "rudder-transformer", - "version": "1.47.0", + "version": "1.48.0", "description": "", "homepage": "https://github.com/rudderlabs/rudder-transformer#readme", "bugs": { diff --git a/src/cdk/v2/bindings/default.js b/src/cdk/v2/bindings/default.js index 0bba7210f0..b86b6d2b63 100644 --- a/src/cdk/v2/bindings/default.js +++ b/src/cdk/v2/bindings/default.js @@ -1,3 +1,4 @@ +const crypto = require('crypto'); const { InstrumentationError, ConfigurationError, @@ -47,7 +48,12 @@ function assertHttpResp(processedResponse, message) { } } +function MD5(data) { + return crypto.createHash('md5').update(data).digest('hex'); +} + module.exports = { + MD5, isValidEventType, assert, assertConfig, diff --git a/src/cdk/v2/destinations/tiktok_audience/config.js b/src/cdk/v2/destinations/tiktok_audience/config.js new file mode 100644 index 0000000000..853f372505 --- /dev/null +++ b/src/cdk/v2/destinations/tiktok_audience/config.js @@ -0,0 +1,9 @@ +const ACTION_MAP = { + add: 'add', + remove: 'delete', +}; +const SHA256_TRAITS = ['IDFA_SHA256', 'AAID_SHA256', 'EMAIL_SHA256', 'PHONE_SHA256']; +module.exports = { + ACTION_MAP, + SHA256_TRAITS, +}; diff --git a/src/cdk/v2/destinations/tiktok_audience/procWorkflow.yaml b/src/cdk/v2/destinations/tiktok_audience/procWorkflow.yaml new file mode 100644 index 0000000000..cd84ecbc87 --- /dev/null +++ b/src/cdk/v2/destinations/tiktok_audience/procWorkflow.yaml @@ -0,0 +1,67 @@ + +bindings: + - name: EventType + path: ../../../../constants + - path: ../../bindings/jsontemplate + exportAll: true + - path: ./config + - name: removeUndefinedAndNullValues + path: ../../../../v0/util + - name: defaultRequestConfig + path: ../../../../v0/util + +steps: + - name: validateInput + template: | + let messageType = .message.type; + $.assert(.message.type, "message Type is not present. Aborting message."); + $.assert(.message.type.toLowerCase() ==='audiencelist', "Event type " + .message.type.toLowerCase() + " is not supported. Aborting message."); + $.assert(.message.properties, "Message properties is not present. Aborting message."); + $.assert(.message.properties.listData, "listData is not present inside properties. Aborting message."); + $.assert($.containsAll(Object.keys(.message.properties.listData), ["add", "remove"]), "unsupported action type. Aborting message.") + + - name: prepareIdentifiersList + description: | + Populate list of identifiers to be updated + template: | + const destinationFields = .message.context.destinationFields.split(", ") + const audienceId = .message.context.externalId[0].type.split("-")[1]; + const isHashRequired = .destination.Config.isHashRequired; + const advertiserIds = .metadata.secret.advertiserIds; + const hashTraits = function(traits) { + traits@trait.(destinationFields@destinationField.( + trait[destinationField] ? { + id: isHashRequired ? + destinationField in $.SHA256_TRAITS ? + $.SHA256(trait[destinationField]) : $.MD5(trait[destinationField]) + : trait[destinationField], + audience_ids:[audienceId] + } : {} + )[]) + }; + const listData = .message.properties.listData; + const actions = Object.keys(listData) + actions@action.({ + "batch_data": hashTraits(listData[action]), + "id_schema": destinationFields, + "advertiser_ids": advertiserIds, + "action": $.ACTION_MAP[action], + })[] + + + - name: buildResponseForProcessTransformation + description: build response + template: | + const accessToken = .metadata.secret.accessToken + const anonymousId = .message.anonymousId; + $.outputs.prepareIdentifiersList@body.( + let response = $.defaultRequestConfig(); + response.body.JSON = body; + response.userId = anonymousId; + response.endpoint = "https://business-api.tiktok.com/open_api/v1.3/segment/mapping/"; + response.headers = { + "Access-Token": accessToken, + "Content-Type": "application/json" + }; + response + ) diff --git a/src/cdk/v2/destinations/tiktok_audience/rtWorkflow.yaml b/src/cdk/v2/destinations/tiktok_audience/rtWorkflow.yaml new file mode 100644 index 0000000000..3db4c405ad --- /dev/null +++ b/src/cdk/v2/destinations/tiktok_audience/rtWorkflow.yaml @@ -0,0 +1,32 @@ +bindings: + - name: handleRtTfSingleEventError + path: ../../../../v0/util/index + +steps: + - name: validateInput + template: | + $.assert(Array.isArray(^) && ^.length > 0, "Invalid event array") + + - name: transform + externalWorkflow: + path: ./procWorkflow.yaml + loopOverInput: true + + - name: successfulEvents + debug: true + template: | + $.outputs.transform#idx{"output" in .}.({ + "batchedRequest": .output, + "batched": true, + "destination": ^[idx].destination, + "metadata": ^[idx].metadata[], + "statusCode": 200 + })[] + - name: failedEvents + template: | + $.outputs.transform#idx.error.( + $.handleRtTfSingleEventError(^[idx], .originalError ?? ., {}) + )[] + - name: finalPayload + template: | + [...$.outputs.failedEvents, ...$.outputs.successfulEvents] diff --git a/src/features.json b/src/features.json index 7de214ab39..9793f667e3 100644 --- a/src/features.json +++ b/src/features.json @@ -58,6 +58,8 @@ "OPTIMIZELY_FULLSTACK": true, "TWITTER_ADS": true, "CLEVERTAP": true, - "ORTTO": true + "ORTTO": true, + "ONE_SIGNAL": true, + "TIKTOK_AUDIENCE": true } } diff --git a/src/util/customTransformer-faas.js b/src/util/customTransformer-faas.js index d1fa48d2d1..54d2410313 100644 --- a/src/util/customTransformer-faas.js +++ b/src/util/customTransformer-faas.js @@ -1,7 +1,7 @@ const { v4: uuidv4 } = require('uuid'); const crypto = require('crypto'); const NodeCache = require('node-cache'); -const { getMetadata } = require('../v0/util'); +const { getMetadata, getTransformationMetadata } = require('../v0/util'); const stats = require('./stats'); const { setupFaasFunction, @@ -82,10 +82,10 @@ async function setOpenFaasUserTransform( libraryVersionIds, pregeneratedFnName, testMode = false, + trMetadata = {}, ) { const tags = { transformerVersionId: userTransformation.versionId, - language: userTransformation.language, identifier: 'openfaas', testMode, }; @@ -106,6 +106,7 @@ async function setOpenFaasUserTransform( testMode, ), testMode, + trMetadata, ); stats.timing('creation_time', setupTime, tags); @@ -129,16 +130,22 @@ async function runOpenFaasUserTransform( const metaTags = events[0].metadata ? getMetadata(events[0].metadata) : {}; const tags = { transformerVersionId: userTransformation.versionId, - language: userTransformation.language, identifier: 'openfaas', testMode, ...metaTags, }; + const trMetadata = events[0].metadata ? getTransformationMetadata(events[0].metadata) : {}; // check and deploy faas function if not exists const functionName = generateFunctionName(userTransformation, libraryVersionIds, testMode); if (testMode) { - await setOpenFaasUserTransform(userTransformation, libraryVersionIds, functionName, testMode); + await setOpenFaasUserTransform( + userTransformation, + libraryVersionIds, + functionName, + testMode, + trMetadata, + ); } const invokeTime = new Date(); @@ -156,6 +163,7 @@ async function runOpenFaasUserTransform( testMode, ), testMode, + trMetadata, ); stats.timing('run_time', invokeTime, tags); return result; diff --git a/src/util/openfaas/index.js b/src/util/openfaas/index.js index 60ad316e1b..f80aa01c23 100644 --- a/src/util/openfaas/index.js +++ b/src/util/openfaas/index.js @@ -23,6 +23,8 @@ const CONFIG_BACKEND_URL = process.env.CONFIG_BACKEND_URL || 'https://api.rudder const GEOLOCATION_URL = process.env.GEOLOCATION_URL || ''; const FAAS_AST_VID = 'ast'; const FAAS_AST_FN_NAME = 'fn-ast'; +const CUSTOM_NETWORK_POLICY_WORKSPACE_IDS = process.env.CUSTOM_NETWORK_POLICY_WORKSPACE_IDS || ''; +const customNetworkPolicyWorkspaceIds = CUSTOM_NETWORK_POLICY_WORKSPACE_IDS.split(','); // Initialise node cache const functionListCache = new NodeCache(); @@ -111,7 +113,14 @@ const invalidateFnCache = () => { functionListCache.set(FUNC_LIST_KEY, []); }; -const deployFaasFunction = async (functionName, code, versionId, libraryVersionIDs, testMode) => { +const deployFaasFunction = async ( + functionName, + code, + versionId, + libraryVersionIDs, + testMode, + trMetadata = {}, +) => { try { logger.debug('[Faas] Deploying a faas function'); let envProcess = 'python index.py'; @@ -132,6 +141,22 @@ const deployFaasFunction = async (functionName, code, versionId, libraryVersionI if (GEOLOCATION_URL) { envVars.geolocation_url = GEOLOCATION_URL; } + // labels + const labels = { + 'openfaas-fn': 'true', + 'parent-component': 'openfaas', + 'com.openfaas.scale.max': FAAS_MAX_PODS_IN_TEXT, + 'com.openfaas.scale.min': FAAS_MIN_PODS_IN_TEXT, + transformationId: trMetadata.transformationId, + workspaceId: trMetadata.workspaceId, + }; + if ( + trMetadata.workspaceId && + customNetworkPolicyWorkspaceIds.includes(trMetadata.workspaceId) + ) { + labels['custom-network-policy'] = 'true'; + } + // TODO: investigate and add more required labels and annotations const payload = { service: functionName, @@ -139,12 +164,7 @@ const deployFaasFunction = async (functionName, code, versionId, libraryVersionI image: FAAS_BASE_IMG, envProcess, envVars, - labels: { - 'openfaas-fn': 'true', - 'parent-component': 'openfaas', - 'com.openfaas.scale.max': FAAS_MAX_PODS_IN_TEXT, - 'com.openfaas.scale.min': FAAS_MIN_PODS_IN_TEXT, - }, + labels, annotations: { 'prometheus.io.scrape': 'true', }, @@ -175,14 +195,28 @@ const deployFaasFunction = async (functionName, code, versionId, libraryVersionI } }; -async function setupFaasFunction(functionName, code, versionId, libraryVersionIDs, testMode) { +async function setupFaasFunction( + functionName, + code, + versionId, + libraryVersionIDs, + testMode, + trMetadata = {}, +) { try { if (!testMode && isFunctionDeployed(functionName)) { logger.debug(`[Faas] Function ${functionName} already deployed`); return; } // deploy faas function - await deployFaasFunction(functionName, code, versionId, libraryVersionIDs, testMode); + await deployFaasFunction( + functionName, + code, + versionId, + libraryVersionIDs, + testMode, + trMetadata, + ); // This api call is only used to check if function is spinned correctly await awaitFunctionReadiness(functionName); @@ -201,6 +235,7 @@ const executeFaasFunction = async ( versionId, libraryVersionIDs, testMode, + trMetadata = {}, ) => { try { logger.debug('[Faas] Invoking faas function'); @@ -217,7 +252,14 @@ const executeFaasFunction = async ( error.message.includes(`error finding function ${functionName}`) ) { removeFunctionFromCache(functionName); - await setupFaasFunction(functionName, null, versionId, libraryVersionIDs, testMode); + await setupFaasFunction( + functionName, + null, + versionId, + libraryVersionIDs, + testMode, + trMetadata, + ); throw new RetryRequestError(`${functionName} not found`); } diff --git a/src/v0/destinations/adobe_analytics/transform.js b/src/v0/destinations/adobe_analytics/transform.js index 54806bf578..67bb66310a 100644 --- a/src/v0/destinations/adobe_analytics/transform.js +++ b/src/v0/destinations/adobe_analytics/transform.js @@ -11,6 +11,7 @@ const { isDefinedAndNotNull, isDefinedAndNotNullAndNotEmpty, getIntegrationsObj, + removeUndefinedAndNullValues, simpleProcessRouterDest, } = require('../../util'); const { @@ -394,7 +395,7 @@ const handleTrack = (message, destinationConfig) => { break; } - return payload; + return removeUndefinedAndNullValues(payload); }; const process = async (event) => { diff --git a/src/v0/destinations/customerio/transform.js b/src/v0/destinations/customerio/transform.js index 5f953ee2f0..984fb7e67f 100644 --- a/src/v0/destinations/customerio/transform.js +++ b/src/v0/destinations/customerio/transform.js @@ -12,6 +12,7 @@ const { adduserIdFromExternalId, getFieldValueFromMessage, handleRtTfSingleEventError, + validateEventName, } = require('../../util'); const logger = require('../../../logger'); @@ -101,6 +102,7 @@ function processSingleMessage(message, destination) { break; case EventType.TRACK: evType = 'event'; + validateEventName(message.event); evName = message.event; break; case EventType.ALIAS: @@ -113,6 +115,7 @@ function processSingleMessage(message, destination) { logger.error(`could not determine type ${messageType}`); throw new InstrumentationError(`could not determine type ${messageType}`); } + evName = evName ? String(evName) : evName; const response = responseBuilder(message, evType, evName, destination, messageType); // replace default domain with EU data center domainc for EU based account diff --git a/src/v0/destinations/customerio/util.js b/src/v0/destinations/customerio/util.js index 2e7f000fba..6b4dbc0e11 100644 --- a/src/v0/destinations/customerio/util.js +++ b/src/v0/destinations/customerio/util.js @@ -10,7 +10,6 @@ const { defaultDeleteRequestConfig, isAppleFamily, validateEmail, - validateEventType, } = require('../../util'); const { EventType, SpecedTraits, TraitsMapping } = require('../../../constants'); @@ -288,7 +287,6 @@ const defaultResponseBuilder = (message, evName, userId, evType, destination, me // 100 - len(`Viewed Screen`) = 86 trimmedEvName = `Viewed ${truncate(message.event || message.properties.name, 86)} Screen`; } else { - validateEventType(evName); trimmedEvName = truncate(evName, 100); } // anonymous_id needs to be sent for anon track calls to provide information on which anon user is being tracked diff --git a/src/v0/destinations/freshmarketer/config.js b/src/v0/destinations/freshmarketer/config.js index f1018d439c..a0d6449c3a 100644 --- a/src/v0/destinations/freshmarketer/config.js +++ b/src/v0/destinations/freshmarketer/config.js @@ -4,23 +4,23 @@ const CONFIG_CATEGORIES = { IDENTIFY: { name: 'FRESHMARKETERIdentifyConfig', type: 'identify', - baseUrl: '.myfreshworks.com/crm/sales/api/contacts/upsert', + baseUrl: '/crm/sales/api/contacts/upsert', }, GROUP: { name: 'FRESHMARKETERGroupConfig', type: 'group', - baseUrlAccount: '.myfreshworks.com/crm/sales/api/sales_accounts/upsert', - baseUrlList: '.myfreshworks.com/crm/sales/api/lists', + baseUrlAccount: '/crm/sales/api/sales_accounts/upsert', + baseUrlList: '/crm/sales/api/lists', }, SALES_ACTIVITY: { name: 'SalesActivityConfig', - baseUrlCreate: '.myfreshworks.com/crm/sales/api/sales_activities', - baseUrlListAll: '.myfreshworks.com/crm/sales/api/selector/sales_activity_types', + baseUrlCreate: '/crm/sales/api/sales_activities', + baseUrlListAll: '/crm/sales/api/selector/sales_activity_types', }, }; -const DELETE_ENDPOINT = '.myfreshworks.com/crm/sales/api/contacts/'; -const LIFECYCLE_STAGE_ENDPOINT = '.myfreshworks.com/crm/sales/api/selector/lifecycle_stages'; +const DELETE_ENDPOINT = '/crm/sales/api/contacts/'; +const LIFECYCLE_STAGE_ENDPOINT = '/crm/sales/api/selector/lifecycle_stages'; const MAPPING_CONFIG = getMappingConfig(CONFIG_CATEGORIES, __dirname); module.exports = { diff --git a/src/v0/destinations/freshmarketer/utils.js b/src/v0/destinations/freshmarketer/utils.js index 5b47bb9170..f7dcc46b06 100644 --- a/src/v0/destinations/freshmarketer/utils.js +++ b/src/v0/destinations/freshmarketer/utils.js @@ -203,7 +203,7 @@ const updateAccountWOContact = (payload, Config) => { */ const updateContactWithList = (userId, listId, Config) => { const response = defaultRequestConfig(); - response.endpoint = `https://${Config.domain}.myfreshworks.com/crm/sales/api/lists/${listId}/add_contacts`; + response.endpoint = `https://${Config.domain}/crm/sales/api/lists/${listId}/add_contacts`; response.headers = getHeaders(Config.apiKey); response.body.JSON = { ids: [userId], diff --git a/src/v0/destinations/freshsales/config.js b/src/v0/destinations/freshsales/config.js index c6ae4b8165..62f54c1297 100644 --- a/src/v0/destinations/freshsales/config.js +++ b/src/v0/destinations/freshsales/config.js @@ -4,23 +4,23 @@ const CONFIG_CATEGORIES = { IDENTIFY: { name: 'identifyConfig', type: 'identify', - baseUrl: '.myfreshworks.com/crm/sales/api/contacts/upsert', + baseUrl: '/crm/sales/api/contacts/upsert', method: 'POST', }, GROUP: { name: 'groupConfig', type: 'group', - baseUrlAccount: '.myfreshworks.com/crm/sales/api/sales_accounts/upsert', + baseUrlAccount: '/crm/sales/api/sales_accounts/upsert', method: 'POST', }, SALES_ACTIVITY: { name: 'SalesActivityConfig', - baseUrlCreate: '.myfreshworks.com/crm/sales/api/sales_activities', - baseUrlListAll: '.myfreshworks.com/crm/sales/api/selector/sales_activity_types', + baseUrlCreate: '/crm/sales/api/sales_activities', + baseUrlListAll: '/crm/sales/api/selector/sales_activity_types', }, }; -const LIFECYCLE_STAGE_ENDPOINT = '.myfreshworks.com/crm/sales/api/selector/lifecycle_stages'; +const LIFECYCLE_STAGE_ENDPOINT = '/crm/sales/api/selector/lifecycle_stages'; const MAPPING_CONFIG = getMappingConfig(CONFIG_CATEGORIES, __dirname); module.exports = { diff --git a/src/v0/destinations/freshsales/transform.js b/src/v0/destinations/freshsales/transform.js index cd7518a101..c1e18482ed 100644 --- a/src/v0/destinations/freshsales/transform.js +++ b/src/v0/destinations/freshsales/transform.js @@ -8,7 +8,7 @@ const { defaultPostRequestConfig, getValidDynamicFormConfig, simpleProcessRouterDest, - validateEventType, + validateEventName, } = require('../../util'); const { InstrumentationError, TransformationError } = require('../../util/errorTypes'); const { CONFIG_CATEGORIES, MAPPING_CONFIG } = require('./config'); @@ -67,7 +67,6 @@ const identifyResponseBuilder = (message, { Config }) => { * @returns */ const trackResponseBuilder = async (message, { Config }, event) => { - validateEventType(event); let payload; const response = defaultRequestConfig(); @@ -125,9 +124,6 @@ const groupResponseBuilder = async (message, { Config }) => { // Checks if there are any mapping events for the track event and returns them function eventMappingHandler(message, destination) { const event = get(message, 'event'); - if (!event) { - throw new InstrumentationError('Event name is required'); - } let { rudderEventsToFreshsalesEvents } = destination.Config; const mappedEvents = new Set(); @@ -161,6 +157,7 @@ const processEvent = async (message, destination) => { response = identifyResponseBuilder(message, destination); break; case EventType.TRACK: { + validateEventName(message.event); const mappedEvents = eventMappingHandler(message, destination); if (mappedEvents.length > 0) { const respList = await Promise.all( diff --git a/src/v0/destinations/klaviyo/util.js b/src/v0/destinations/klaviyo/util.js index 4304edd78f..b31dafd78b 100644 --- a/src/v0/destinations/klaviyo/util.js +++ b/src/v0/destinations/klaviyo/util.js @@ -17,7 +17,6 @@ const { handleHttpRequest } = require('../../../adapters/network'); const { JSON_MIME_TYPE, HTTP_STATUS_CODES } = require('../../util/constant'); const { NetworkError, InstrumentationError } = require('../../util/errorTypes'); const { getDynamicErrorType } = require('../../../adapters/utils/networkUtils'); -const { client: errNotificationClient } = require('../../../util/errorNotifier'); const { BASE_ENDPOINT, MAPPING_CONFIG, CONFIG_CATEGORIES, MAX_BATCH_SIZE } = require('./config'); const REVISION_CONSTANT = '2023-02-22'; @@ -69,11 +68,6 @@ const getIdFromNewOrExistingProfile = async (endpoint, payload, requestOptions) let statusCode = resp.status; if (resp.status === 201 || resp.status === 409) { // retryable error if the profile id is not found in the response - errNotificationClient.notify( - new Error('Klaviyo: ProfileId not found'), - 'Profile Id not Found in the response', - JSON.stringify(resp.response), - ); statusCode = 500; } diff --git a/src/v0/destinations/monday/transform.js b/src/v0/destinations/monday/transform.js index 34ada34780..37ee835e50 100644 --- a/src/v0/destinations/monday/transform.js +++ b/src/v0/destinations/monday/transform.js @@ -8,7 +8,7 @@ const { removeUndefinedAndNullValues, simpleProcessRouterDest, getDestinationExternalID, - validateEventType, + validateEventName, } = require('../../util'); const { ConfigurationError, @@ -42,7 +42,7 @@ const trackResponseBuilder = async (message, { Config }) => { const { apiToken } = Config; let boardId = getDestinationExternalID(message, 'boardId'); const event = get(message, 'event'); - validateEventType(event); + validateEventName(event); if (!boardId) { boardId = Config.boardId; } diff --git a/src/v0/sources/revenuecat/mapping.json b/src/v0/sources/revenuecat/mapping.json new file mode 100644 index 0000000000..541568b71b --- /dev/null +++ b/src/v0/sources/revenuecat/mapping.json @@ -0,0 +1,10 @@ +[ + { + "sourceKeys": "event.type", + "destKeys": "event" + }, + { + "sourceKeys": "event.id", + "destKeys": "messageId" + } +] diff --git a/src/v0/sources/revenuecat/transform.js b/src/v0/sources/revenuecat/transform.js new file mode 100644 index 0000000000..36944e10fa --- /dev/null +++ b/src/v0/sources/revenuecat/transform.js @@ -0,0 +1,47 @@ +const { camelCase } = require('lodash'); +const moment = require('moment'); +const { removeUndefinedAndNullValues, isDefinedAndNotNull } = require('../../util'); +const Message = require('../message'); + +function process(event) { + const message = new Message(`RevenueCat`); + + // we are setting event type as track always + message.setEventType('track'); + + const properties = {}; + // dump all event properties to message.properties after converting them to camelCase + if (event.event) { + Object.keys(event.event).forEach((key) => { + properties[camelCase(key)] = event.event[key]; + }); + message.setProperty('properties', properties); + } + + // setting up app_user_id to externalId : revenuecatAppUserId + if (event?.event?.app_user_id) { + message.context.externalId = [ + { + type: 'revenuecatAppUserId', + id: event?.event?.app_user_id, + }, + ]; + } + + if ( + isDefinedAndNotNull(event?.event?.event_timestamp_ms) && + moment(event?.event?.event_timestamp_ms).isValid() + ) { + const validTimestamp = new Date(event.event.event_timestamp_ms).toISOString(); + message.setProperty('originalTimestamp', validTimestamp); + message.setProperty('sentAt', validTimestamp); + } + message.event = event?.event?.type; + message.messageId = event?.event?.id; + + // removing undefined and null values from message + removeUndefinedAndNullValues(message); + return message; +} + +module.exports = { process }; diff --git a/src/v0/util/index.js b/src/v0/util/index.js index ea08d08c8a..d6f6621220 100644 --- a/src/v0/util/index.js +++ b/src/v0/util/index.js @@ -1403,6 +1403,12 @@ const getMetadata = (metadata) => ({ destinationType: metadata.destinationType, k8_namespace: metadata.namespace, }); + +const getTransformationMetadata = (metadata) => ({ + transformationId: metadata.transformationId, + workspaceId: metadata.workspaceId, +}); + // checks if array 2 is a subset of array 1 function checkSubsetOfArray(array1, array2) { const result = array2.every((val) => array1.includes(val)); @@ -2063,7 +2069,7 @@ const isValidInteger = (value) => { // Use a regular expression to check if the string is a valid integer or a valid floating-point number return typeof value === 'string' ? /^-?\d+$/.test(value) : false; }; -const validateEventType = (event) => { +const validateEventName = (event) => { if (!event || typeof event !== 'string') { throw new InstrumentationError('Event is a required field and should be a string'); } @@ -2123,6 +2129,7 @@ module.exports = { getIntegrationsObj, getMappingConfig, getMetadata, + getTransformationMetadata, getParsedIP, getStringValueOfJSON, getSuccessRespEvents, @@ -2170,7 +2177,7 @@ module.exports = { getDestAuthCacheInstance, refinePayload, validateEmail, - validateEventType, + validateEventName, validatePhoneWithCountryCode, getEventReqMetadata, isHybridModeEnabled, diff --git a/test/__tests__/data/adobe_analytics.json b/test/__tests__/data/adobe_analytics.json index cfffccb8da..6361f92640 100644 --- a/test/__tests__/data/adobe_analytics.json +++ b/test/__tests__/data/adobe_analytics.json @@ -1010,7 +1010,6 @@ }, "messageId": "1578564113557-af022c68-429e-4af4-b99b-2b9174056383", "properties": { - "order_id": "1234", "affiliation": "Apple Store", "value": 20, "revenue": 15.0, @@ -1019,6 +1018,7 @@ "discount": 1.5, "coupon": "ImagePro", "currency": "USD", + "purchaseId": "p101", "products": [ { "product_id": "123", @@ -1205,7 +1205,7 @@ "JSON": {}, "JSON_ARRAY": {}, "XML": { - "payload": "17941080sales campaignwebUSD127.0.0.1en-US12341234Dalvik/2.1.0 (Linux; U; Android 9; Android SDK built for x86 Build/PSR1.180720.075)https://www.google.com/search?q=estore+bestsellerroottval001RudderLabs JavaScript SDKocheckout startedhttps://www.estore.com/best-seller/12020-01-09T10:01:53.558Zmktcloudid001scCheckoutGames;Monopoly;1;14.00,Games;UNO;2;6.90footlockerrudderstackpoc" + "payload": "17941080sales campaignwebUSD127.0.0.1en-USDalvik/2.1.0 (Linux; U; Android 9; Android SDK built for x86 Build/PSR1.180720.075)https://www.google.com/search?q=estore+bestsellerroottval001RudderLabs JavaScript SDKocheckout startedhttps://www.estore.com/best-seller/12020-01-09T10:01:53.558Zmktcloudid001p101scCheckoutGames;Monopoly;1;14.00,Games;UNO;2;6.90footlockerrudderstackpoc" }, "FORM": {} }, diff --git a/test/__tests__/data/freshmarketer.json b/test/__tests__/data/freshmarketer.json index 3d30841b30..390c0fb44e 100644 --- a/test/__tests__/data/freshmarketer.json +++ b/test/__tests__/data/freshmarketer.json @@ -5,7 +5,7 @@ "destination": { "Config": { "apiKey": "dummyApiKey", - "domain": "rudderstack-476952domain3105" + "domain": "rudderstack-476952domain3105.myfreshworks.com" } }, "message": { @@ -94,7 +94,7 @@ "destination": { "Config": { "apiKey": "dummyApiKey", - "domain": "rudderstack-476952domain3105" + "domain": "rudderstack-476952domain3105.myfreshworks.com" } }, "message": { @@ -183,7 +183,7 @@ "destination": { "Config": { "apiKey": "dummyApiKey", - "domain": "rudderstack-476952domain3105" + "domain": "rudderstack-476952domain3105.myfreshworks.com" } }, "message": { @@ -248,7 +248,7 @@ "destination": { "Config": { "apiKey": "dummyApiKey", - "domain": "rudderstack-476952domain3105" + "domain": "rudderstack-476952domain3105.myfreshworks.com" } }, "message": { @@ -312,7 +312,7 @@ "destination": { "Config": { "apiKey": "dummyApiKey", - "domain": "domain-rudder" + "domain": "domain-rudder.myfreshworks.com" } }, "message": { @@ -422,7 +422,7 @@ "destination": { "Config": { "apiKey": "dummyApiKey", - "domain": "domain-rudder" + "domain": "domain-rudder.myfreshworks.com" } }, "message": { @@ -491,7 +491,7 @@ "destination": { "Config": { "apiKey": "dummyApiKey", - "domain": "domain-rudder" + "domain": "domain-rudder.myfreshworks.com" } }, "message": { @@ -558,7 +558,7 @@ "destination": { "Config": { "apiKey": "dummyApiKey", - "domain": "domain-rudder" + "domain": "domain-rudder.myfreshworks.com" } }, "message": { @@ -627,7 +627,7 @@ "destination": { "Config": { "apiKey": "dummyApiKey", - "domain": "domain-rudder" + "domain": "domain-rudder.myfreshworks.com" } }, "message": { @@ -760,7 +760,7 @@ "destination": { "Config": { "apiKey": "dummyApiKey", - "domain": "domain-rudder" + "domain": "domain-rudder.myfreshworks.com" } } }, @@ -843,7 +843,7 @@ "destination": { "Config": { "apiKey": "dummyApiKey", - "domain": "domain-rudder" + "domain": "domain-rudder.myfreshworks.com" } } }, @@ -927,7 +927,7 @@ "destination": { "Config": { "apiKey": "dummyApiKey", - "domain": "domain-rudder" + "domain": "domain-rudder.myfreshworks.com" } } }, @@ -1019,7 +1019,7 @@ "destination": { "Config": { "apiKey": "dummyApiKey", - "domain": "domain-rudder" + "domain": "domain-rudder.myfreshworks.com" } } }, @@ -1127,7 +1127,7 @@ "destination": { "Config": { "apiKey": "dummyApiKey", - "domain": "domain-rudder" + "domain": "domain-rudder.myfreshworks.com" } } }, @@ -1228,7 +1228,7 @@ "destination": { "Config": { "apiKey": "dummyApiKey", - "domain": "domain-rudder" + "domain": "domain-rudder.myfreshworks.com" } } }, @@ -1315,7 +1315,7 @@ "destination": { "Config": { "apiKey": "dummyApiKey", - "domain": "domain-rudder" + "domain": "domain-rudder.myfreshworks.com" } } }, @@ -1406,7 +1406,7 @@ "destination": { "Config": { "apiKey": "dummyApiKey", - "domain": "domain-rudder" + "domain": "domain-rudder.myfreshworks.com" } } }, @@ -1496,7 +1496,7 @@ "destination": { "Config": { "apiKey": "dummyApiKey", - "domain": "domain-rudder" + "domain": "domain-rudder.myfreshworks.com" } } }, @@ -1586,7 +1586,7 @@ "destination": { "Config": { "apiKey": "dummyApiKey", - "domain": "domain-rudder" + "domain": "domain-rudder.myfreshworks.com" } } }, @@ -1696,7 +1696,7 @@ "destination": { "Config": { "apiKey": "dummyApiKey", - "domain": "domain-rudder" + "domain": "domain-rudder.myfreshworks.com" } } }, @@ -1757,7 +1757,7 @@ "destination": { "Config": { "apiKey": "dummyApiKey", - "domain": "domain-rudder" + "domain": "domain-rudder.myfreshworks.com" } } }, @@ -1788,7 +1788,7 @@ "destination": { "Config": { "apiKey": "dummyApiKey", - "domain": "domain-rudder" + "domain": "domain-rudder.myfreshworks.com" } } }, @@ -1825,7 +1825,7 @@ "destination": { "Config": { "apiKey": "dummyApiKey", - "domain": "domain-rudder" + "domain": "domain-rudder.myfreshworks.com" } } }, @@ -1892,7 +1892,7 @@ "destination": { "Config": { "apiKey": "dummyApiKey", - "domain": "domain-rudder" + "domain": "domain-rudder.myfreshworks.com" } } }, @@ -1937,7 +1937,7 @@ "destination": { "Config": { "apiKey": "dummyApiKey", - "domain": "domain-rudder" + "domain": "domain-rudder.myfreshworks.com" } } }, @@ -1976,7 +1976,7 @@ "destination": { "Config": { "apiKey": "dummyApiKey", - "domain": "domain-rudder" + "domain": "domain-rudder.myfreshworks.com" } } }, @@ -2043,7 +2043,7 @@ "destination": { "Config": { "apiKey": "dummyApiKey", - "domain": "domain-rudder" + "domain": "domain-rudder.myfreshworks.com" } } }, @@ -2145,7 +2145,7 @@ "destination": { "Config": { "apiKey": "dummyApiKey", - "domain": "domain-rudder" + "domain": "domain-rudder.myfreshworks.com" } } }, @@ -2232,7 +2232,7 @@ "destination": { "Config": { "apiKey": "dummyApiKey", - "domain": "domain-rudder" + "domain": "domain-rudder.myfreshworks.com" } } }, @@ -2334,7 +2334,7 @@ "destination": { "Config": { "apiKey": "dummyApiKey", - "domain": "domain-rudder" + "domain": "domain-rudder.myfreshworks.com" } } }, @@ -2427,7 +2427,7 @@ "destination": { "Config": { "apiKey": "dummyApiKey", - "domain": "domain-rudder", + "domain": "domain-rudder.myfreshworks.com", "rudderEventsToFreshmarketerEvents": [ { "from": "test_activity", diff --git a/test/__tests__/data/freshmarketer_router_input.json b/test/__tests__/data/freshmarketer_router_input.json index 0e05c7f5f8..2cc5ce58de 100644 --- a/test/__tests__/data/freshmarketer_router_input.json +++ b/test/__tests__/data/freshmarketer_router_input.json @@ -3,7 +3,7 @@ "destination": { "Config": { "apiKey": "dummyApiKey", - "domain": "rudderstack-476952domain3105" + "domain": "rudderstack-476952domain3105.myfreshworks.com" } }, "metadata": { @@ -59,7 +59,7 @@ "destination": { "Config": { "apiKey": "dummyApiKey", - "domain": "rudderstack-476952domain3105" + "domain": "rudderstack-476952domain3105.myfreshworks.com" } }, "metadata": { @@ -115,7 +115,7 @@ "destination": { "Config": { "apiKey": "dummyApiKey", - "domain": "domain-rudder" + "domain": "domain-rudder.myfreshworks.com" } }, "metadata": { diff --git a/test/__tests__/data/freshmarketer_router_output.json b/test/__tests__/data/freshmarketer_router_output.json index 01740cb626..3525e4bb16 100644 --- a/test/__tests__/data/freshmarketer_router_output.json +++ b/test/__tests__/data/freshmarketer_router_output.json @@ -43,7 +43,7 @@ "destination": { "Config": { "apiKey": "dummyApiKey", - "domain": "rudderstack-476952domain3105" + "domain": "rudderstack-476952domain3105.myfreshworks.com" } } }, @@ -91,7 +91,7 @@ "destination": { "Config": { "apiKey": "dummyApiKey", - "domain": "rudderstack-476952domain3105" + "domain": "rudderstack-476952domain3105.myfreshworks.com" } } }, @@ -156,7 +156,7 @@ "destination": { "Config": { "apiKey": "dummyApiKey", - "domain": "domain-rudder" + "domain": "domain-rudder.myfreshworks.com" } } } diff --git a/test/__tests__/data/freshsales.json b/test/__tests__/data/freshsales.json index 2527e37b90..55193532f4 100644 --- a/test/__tests__/data/freshsales.json +++ b/test/__tests__/data/freshsales.json @@ -69,7 +69,7 @@ "destination": { "Config": { "apiKey": "dummyApiKey", - "domain": "domain-rudder" + "domain": "domain-rudder.myfreshworks.com" } } }, @@ -90,7 +90,7 @@ "destination": { "Config": { "apiKey": "dummyApiKey", - "domain": "rudderstack-476952domain3105" + "domain": "rudderstack-476952domain3105.myfreshworks.com" } }, "message": { @@ -179,7 +179,7 @@ "destination": { "Config": { "apiKey": "dummyApiKey", - "domain": "rudderstack-476952domain3105" + "domain": "rudderstack-476952domain3105.myfreshworks.com" } }, "message": { @@ -268,7 +268,7 @@ "destination": { "Config": { "apiKey": "dummyApiKey", - "domain": "rudderstack-476952domain3105" + "domain": "rudderstack-476952domain3105.myfreshworks.com" } }, "message": { @@ -356,7 +356,7 @@ "destination": { "Config": { "apiKey": "dummyApiKey", - "domain": "rudderstack-476952domain3105" + "domain": "rudderstack-476952domain3105.myfreshworks.com" } }, "message": { @@ -421,7 +421,7 @@ "destination": { "Config": { "apiKey": "dummyApiKey", - "domain": "domain-rudder" + "domain": "domain-rudder.myfreshworks.com" } }, "message": { @@ -531,7 +531,7 @@ "destination": { "Config": { "apiKey": "dummyApiKey", - "domain": "domain-rudder" + "domain": "domain-rudder.myfreshworks.com" } }, "message": { @@ -600,7 +600,7 @@ "destination": { "Config": { "apiKey": "dummyApiKey", - "domain": "domain-rudder" + "domain": "domain-rudder.myfreshworks.com" } }, "message": { @@ -667,7 +667,7 @@ "destination": { "Config": { "apiKey": "dummyApiKey", - "domain": "domain-rudder" + "domain": "domain-rudder.myfreshworks.com" } }, "message": { @@ -736,7 +736,7 @@ "destination": { "Config": { "apiKey": "dummyApiKey", - "domain": "domain-rudder" + "domain": "domain-rudder.myfreshworks.com" } }, "message": { @@ -869,7 +869,7 @@ "destination": { "Config": { "apiKey": "dummyApiKey", - "domain": "domain-rudder" + "domain": "domain-rudder.myfreshworks.com" } } }, @@ -952,7 +952,7 @@ "destination": { "Config": { "apiKey": "dummyApiKey", - "domain": "domain-rudder" + "domain": "domain-rudder.myfreshworks.com" } } }, @@ -1036,7 +1036,7 @@ "destination": { "Config": { "apiKey": "dummyApiKey", - "domain": "domain-rudder" + "domain": "domain-rudder.myfreshworks.com" } } }, @@ -1128,7 +1128,7 @@ "destination": { "Config": { "apiKey": "dummyApiKey", - "domain": "domain-rudder", + "domain": "domain-rudder.myfreshworks.com", "rudderEventsToFreshsalesEvents": [ { "from": "test", @@ -1244,7 +1244,7 @@ "destination": { "Config": { "apiKey": "dummyApiKey", - "domain": "domain-rudder" + "domain": "domain-rudder.myfreshworks.com" } } }, @@ -1345,7 +1345,7 @@ "destination": { "Config": { "apiKey": "dummyApiKey", - "domain": "domain-rudder" + "domain": "domain-rudder.myfreshworks.com" } } }, @@ -1432,7 +1432,7 @@ "destination": { "Config": { "apiKey": "dummyApiKey", - "domain": "domain-rudder" + "domain": "domain-rudder.myfreshworks.com" } } }, @@ -1523,7 +1523,7 @@ "destination": { "Config": { "apiKey": "dummyApiKey", - "domain": "domain-rudder" + "domain": "domain-rudder.myfreshworks.com" } } }, @@ -1613,7 +1613,7 @@ "destination": { "Config": { "apiKey": "dummyApiKey", - "domain": "domain-rudder" + "domain": "domain-rudder.myfreshworks.com" } } }, @@ -1703,7 +1703,7 @@ "destination": { "Config": { "apiKey": "dummyApiKey", - "domain": "domain-rudder" + "domain": "domain-rudder.myfreshworks.com" } } }, @@ -1813,7 +1813,7 @@ "destination": { "Config": { "apiKey": "dummyApiKey", - "domain": "domain-rudder" + "domain": "domain-rudder.myfreshworks.com" } } }, @@ -1874,7 +1874,7 @@ "destination": { "Config": { "apiKey": "dummyApiKey", - "domain": "domain-rudder" + "domain": "domain-rudder.myfreshworks.com" } } }, @@ -1966,7 +1966,7 @@ "destination": { "Config": { "apiKey": "dummyApiKey", - "domain": "domain-rudder" + "domain": "domain-rudder.myfreshworks.com" } } }, @@ -2053,7 +2053,7 @@ "destination": { "Config": { "apiKey": "dummyApiKey", - "domain": "domain-rudder" + "domain": "domain-rudder.myfreshworks.com" } } }, @@ -2155,7 +2155,7 @@ "destination": { "Config": { "apiKey": "dummyApiKey", - "domain": "domain-rudder" + "domain": "domain-rudder.myfreshworks.com" } } }, diff --git a/test/__tests__/data/freshsales_router_input.json b/test/__tests__/data/freshsales_router_input.json index 856a127f51..ea4b9887dd 100644 --- a/test/__tests__/data/freshsales_router_input.json +++ b/test/__tests__/data/freshsales_router_input.json @@ -66,7 +66,7 @@ }, "Config": { "apiKey": "hrkjfergeferf", - "domain": "rudderstack-479541159204968909" + "domain": "rudderstack-479541159204968909.myfreshworks.com" }, "Enabled": true, "Transformations": [], diff --git a/test/__tests__/data/freshsales_router_output.json b/test/__tests__/data/freshsales_router_output.json index 449d6eb45a..69d259ff00 100644 --- a/test/__tests__/data/freshsales_router_output.json +++ b/test/__tests__/data/freshsales_router_output.json @@ -72,7 +72,7 @@ }, "Config": { "apiKey": "hrkjfergeferf", - "domain": "rudderstack-479541159204968909" + "domain": "rudderstack-479541159204968909.myfreshworks.com" }, "Enabled": true, "Transformations": [], diff --git a/test/integrations/destinations/tiktok_audience/processor/data.ts b/test/integrations/destinations/tiktok_audience/processor/data.ts new file mode 100644 index 0000000000..a715aa2f72 --- /dev/null +++ b/test/integrations/destinations/tiktok_audience/processor/data.ts @@ -0,0 +1,854 @@ +export const data = [ + { + "name": "tiktok_audience", + "description": "Test 1: Containing SHA256 traits only", + "feature": "processor", + "module": "destination", + "version": "v0", + "input": { + "request": { + "body": [ + { + "message": { + "userId": "user 1", + "type": "audiencelist", + "properties": { + "listData": { + "add": [ + { + "EMAIL_SHA256": "alex@email.com" + }, + { + "EMAIL_SHA256": "amy@abc.com" + }, + { + "EMAIL_SHA256": "van@abc.com" + } + ], + "remove": [ + { + "EMAIL_SHA256": "alex@email.com" + }, + { + "EMAIL_SHA256": "amy@abc.com" + }, + { + "EMAIL_SHA256": "van@abc.com" + } + ] + } + }, + "context": { + "ip": "14.5.67.21", + "library": { + "name": "http" + }, + "externalId": [ + { + "type": "TIKTOK_AUDIENCE-23856594064540489", + "identifierType": "EMAIL_SHA256" + } + ], + "destinationFields": "EMAIL_SHA256" + }, + "timestamp": "2020-02-02T00:23:09.544Z" + }, + "metadata": { + "jobId": 1, + "secret": { + "accessToken": "dummyAccessToken", + "advertiserIds": [ + "dummyAdverTiserID" + ] + } + }, + "destination": { + "DestinationDefinition": { + "Config": { + "cdkV2Enabled": true + } + }, + "Config": { + "isHashRequired": true, + "registerDeviceOrBrowserApiKey": true, + "apiKey": "intercomApiKey", + "appId": "9e9cdea1-78fa-4829-a9b2-5d7f7e96d1a0", + "collectContext": false + } + } + } + ] + } + }, + "output": { + "response": { + "status": 200, + "body": [ + { + "output": { + "version": "1", + "type": "REST", + "method": "POST", + "endpoint": "https://business-api.tiktok.com/open_api/v1.3/segment/mapping/", + "headers": { + "Access-Token": "dummyAccessToken", + "Content-Type": "application/json" + }, + "params": {}, + "body": { + "JSON": { + "batch_data": [ + [ + { + "id": "ac0f1baec38a9ef3cfcb56db981df7d9bab2568c7f53ef3776d1c059ec58e72b", + "audience_ids": [ + "23856594064540489" + ] + } + ], + [ + { + "id": "49eaeca26c878f268ad33af8cfa8194ca5b8b8e448b1c775bf9153a2de734579", + "audience_ids": [ + "23856594064540489" + ] + } + ], + [ + { + "id": "2048acfa84a01121060ca2fc8a673a76d427176dc37224d4408c21973bd90e5c", + "audience_ids": [ + "23856594064540489" + ] + } + ] + ], + "id_schema": [ + "EMAIL_SHA256" + ], + "advertiser_ids": [ + "dummyAdverTiserID" + ], + "action": "add" + }, + "JSON_ARRAY": {}, + "XML": {}, + "FORM": {} + }, + "files": {}, + "userId": "" + }, + "metadata": { + "jobId": 1, + "secret": { + "accessToken": "dummyAccessToken", + "advertiserIds": [ + "dummyAdverTiserID" + ] + } + }, + "statusCode": 200 + }, + { + "output": { + "version": "1", + "type": "REST", + "method": "POST", + "endpoint": "https://business-api.tiktok.com/open_api/v1.3/segment/mapping/", + "headers": { + "Access-Token": "dummyAccessToken", + "Content-Type": "application/json" + }, + "params": {}, + "body": { + "JSON": { + "batch_data": [ + [ + { + "id": "ac0f1baec38a9ef3cfcb56db981df7d9bab2568c7f53ef3776d1c059ec58e72b", + "audience_ids": [ + "23856594064540489" + ] + } + ], + [ + { + "id": "49eaeca26c878f268ad33af8cfa8194ca5b8b8e448b1c775bf9153a2de734579", + "audience_ids": [ + "23856594064540489" + ] + } + ], + [ + { + "id": "2048acfa84a01121060ca2fc8a673a76d427176dc37224d4408c21973bd90e5c", + "audience_ids": [ + "23856594064540489" + ] + } + ] + ], + "id_schema": [ + "EMAIL_SHA256" + ], + "advertiser_ids": [ + "dummyAdverTiserID" + ], + "action": "delete" + }, + "JSON_ARRAY": {}, + "XML": {}, + "FORM": {} + }, + "files": {}, + "userId": "" + }, + "metadata": { + "jobId": 1, + "secret": { + "accessToken": "dummyAccessToken", + "advertiserIds": [ + "dummyAdverTiserID" + ] + } + }, + "statusCode": 200 + } + ] + } + } + }, + { + "name": "tiktok_audience", + "description": "Test 2: Containing SHA256 and MD5 traits", + "feature": "processor", + "module": "destination", + "version": "v0", + "input": { + "request": { + "body": [ + { + "message": { + "userId": "user 1", + "type": "audiencelist", + "properties": { + "listData": { + "add": [ + { + "EMAIL_SHA256": "alex@email.com", + "AAID_MD5": "1234567" + }, + { + "EMAIL_SHA256": "amy@abc.com", + "AAID_MD5": "1234568" + }, + { + "EMAIL_SHA256": "van@abc.com", + "AAID_MD5": "1234569" + } + ], + "remove": [ + { + "EMAIL_SHA256": "alex@email.com", + "AAID_MD5": "1234570" + }, + { + "EMAIL_SHA256": "amy@abc.com", + "AAID_MD5": "1234571" + }, + { + "EMAIL_SHA256": "van@abc.com", + "AAID_MD5": "1234572" + } + ] + } + }, + "context": { + "ip": "14.5.67.21", + "library": { + "name": "http" + }, + "externalId": [ + { + "type": "TIKTOK_AUDIENCE-23856594064540489", + "identifierType": "EMAIL_SHA256" + } + ], + "destinationFields": "EMAIL_SHA256, AAID_MD5" + }, + "timestamp": "2020-02-02T00:23:09.544Z" + }, + "metadata": { + "jobId": 1, + "secret": { + "accessToken": "dummyAccessToken", + "advertiserIds": [ + "dummyAdverTiserID" + ] + } + }, + "destination": { + "DestinationDefinition": { + "Config": { + "cdkV2Enabled": true + } + }, + "Config": { + "isHashRequired": true, + "registerDeviceOrBrowserApiKey": true, + "apiKey": "intercomApiKey", + "appId": "9e9cdea1-78fa-4829-a9b2-5d7f7e96d1a0", + "collectContext": false + } + } + } + ] + } + }, + "output": { + "response": { + "status": 200, + "body": [ + { + "output": { + "version": "1", + "type": "REST", + "method": "POST", + "endpoint": "https://business-api.tiktok.com/open_api/v1.3/segment/mapping/", + "headers": { + "Access-Token": "dummyAccessToken", + "Content-Type": "application/json" + }, + "params": {}, + "body": { + "JSON": { + "batch_data": [ + [ + { + "id": "ac0f1baec38a9ef3cfcb56db981df7d9bab2568c7f53ef3776d1c059ec58e72b", + "audience_ids": [ + "23856594064540489" + ] + }, + { + "id": "fcea920f7412b5da7be0cf42b8c93759", + "audience_ids": [ + "23856594064540489" + ] + } + ], + [ + { + "id": "49eaeca26c878f268ad33af8cfa8194ca5b8b8e448b1c775bf9153a2de734579", + "audience_ids": [ + "23856594064540489" + ] + }, + { + "id": "fe743d8d97aa7dfc6c93ccdc2e749513", + "audience_ids": [ + "23856594064540489" + ] + } + ], + [ + { + "id": "2048acfa84a01121060ca2fc8a673a76d427176dc37224d4408c21973bd90e5c", + "audience_ids": [ + "23856594064540489" + ] + }, + { + "id": "e36a2f90240e9e84483504fd4a704452", + "audience_ids": [ + "23856594064540489" + ] + } + ] + ], + "id_schema": [ + "EMAIL_SHA256", + "AAID_MD5" + ], + "advertiser_ids": [ + "dummyAdverTiserID" + ], + "action": "add" + }, + "JSON_ARRAY": {}, + "XML": {}, + "FORM": {} + }, + "files": {}, + "userId": "" + }, + "metadata": { + "jobId": 1, + "secret": { + "accessToken": "dummyAccessToken", + "advertiserIds": [ + "dummyAdverTiserID" + ] + } + }, + "statusCode": 200 + }, + { + "output": { + "version": "1", + "type": "REST", + "method": "POST", + "endpoint": "https://business-api.tiktok.com/open_api/v1.3/segment/mapping/", + "headers": { + "Access-Token": "dummyAccessToken", + "Content-Type": "application/json" + }, + "params": {}, + "body": { + "JSON": { + "batch_data": [ + [ + { + "id": "ac0f1baec38a9ef3cfcb56db981df7d9bab2568c7f53ef3776d1c059ec58e72b", + "audience_ids": [ + "23856594064540489" + ] + }, + { + "id": "c1abd65fea29d573ddef1bce925e3276", + "audience_ids": [ + "23856594064540489" + ] + } + ], + [ + { + "id": "49eaeca26c878f268ad33af8cfa8194ca5b8b8e448b1c775bf9153a2de734579", + "audience_ids": [ + "23856594064540489" + ] + }, + { + "id": "7298110702a080dfc6903f13333eb04a", + "audience_ids": [ + "23856594064540489" + ] + } + ], + [ + { + "id": "2048acfa84a01121060ca2fc8a673a76d427176dc37224d4408c21973bd90e5c", + "audience_ids": [ + "23856594064540489" + ] + }, + { + "id": "d9cb68b1fd3b9d32abc5f4cab8b42b68", + "audience_ids": [ + "23856594064540489" + ] + } + ] + ], + "id_schema": [ + "EMAIL_SHA256", + "AAID_MD5" + ], + "advertiser_ids": [ + "dummyAdverTiserID" + ], + "action": "delete" + }, + "JSON_ARRAY": {}, + "XML": {}, + "FORM": {} + }, + "files": {}, + "userId": "" + }, + "metadata": { + "jobId": 1, + "secret": { + "accessToken": "dummyAccessToken", + "advertiserIds": [ + "dummyAdverTiserID" + ] + } + }, + "statusCode": 200 + } + ] + } + } + }, + { + "name": "tiktok_audience", + "description": "Test 3: Containing all possible traits", + "feature": "processor", + "module": "destination", + "version": "v0", + "input": { + "request": { + "body": [ + { + "message": { + "userId": "user 1", + "type": "audiencelist", + "properties": { + "listData": { + "add": [ + { + "EMAIL_SHA256": "alex@email.com", + "PHONE_SHA256": "+129988776655", + "IDFA_SHA256": "1234lkasfjdalj12321", + "AAID_SHA256": "000999OOOQQQQ", + "AAID_MD5": "000999OOOQQQQ", + "IDFA_MD5": "1234lkasfjdalj12321" + }, + { + "EMAIL_SHA256": "amy@abc.com", + "PHONE_SHA256": "+129988776677", + "IDFA_SHA256": "1234lkasfjdalj114455", + "AAID_SHA256": "000999OOOPPPP", + "AAID_MD5": "000999OOOPPPP", + "IDFA_MD5": "1234lkasfjdalj114455" + } + ] + } + }, + "context": { + "ip": "14.5.67.21", + "library": { + "name": "http" + }, + "externalId": [ + { + "type": "TIKTOK_AUDIENCE-23856594064540489", + "identifierType": "EMAIL_SHA256" + } + ], + "destinationFields": "EMAIL_SHA256, PHONE_SHA256, IDFA_SHA256, AAID_SHA256, AAID_MD, IDFA_MD5" + }, + "timestamp": "2020-02-02T00:23:09.544Z" + }, + "metadata": { + "jobId": 1, + "secret": { + "accessToken": "dummyAccessToken", + "advertiserIds": [ + "dummyAdverTiserID" + ] + } + }, + "destination": { + "DestinationDefinition": { + "Config": { + "cdkV2Enabled": true + } + }, + "Config": { + "isHashRequired": true, + "registerDeviceOrBrowserApiKey": true, + "apiKey": "intercomApiKey", + "appId": "9e9cdea1-78fa-4829-a9b2-5d7f7e96d1a0", + "collectContext": false + } + } + } + ] + } + }, + "output": { + "response": { + "status": 200, + "body": [ + { + "output": { + "version": "1", + "type": "REST", + "method": "POST", + "endpoint": "https://business-api.tiktok.com/open_api/v1.3/segment/mapping/", + "headers": { + "Access-Token": "dummyAccessToken", + "Content-Type": "application/json" + }, + "params": {}, + "body": { + "JSON": { + "batch_data": [ + [ + { + "id": "ac0f1baec38a9ef3cfcb56db981df7d9bab2568c7f53ef3776d1c059ec58e72b", + "audience_ids": [ + "23856594064540489" + ] + }, + { + "id": "31e78a3bf9ce2b43316f64fe883a531d6266938091e94e2f2480272481163dee", + "audience_ids": [ + "23856594064540489" + ] + }, + { + "id": "0259f595f7172c8dd692a5c37b4d296939555f862aae8adb964391bdb65006ab", + "audience_ids": [ + "23856594064540489" + ] + }, + { + "id": "b06fbe7a29f33576a792ba3df3c9bf838cd26ea88cf574285fa60dc0234a8485", + "audience_ids": [ + "23856594064540489" + ] + }, + {}, + { + "id": "32ee3d063320815a13e0058c2498ff76", + "audience_ids": [ + "23856594064540489" + ] + } + ], + [ + { + "id": "49eaeca26c878f268ad33af8cfa8194ca5b8b8e448b1c775bf9153a2de734579", + "audience_ids": [ + "23856594064540489" + ] + }, + { + "id": "fb40adc7debbf40e7b45b0a4a91886785dff1a28809276f95f1c44f7045f9b4d", + "audience_ids": [ + "23856594064540489" + ] + }, + { + "id": "e6bbdf34c5f3472f31b2923a26811560a599233f3dea4c9971595c3bb7b1e8dc", + "audience_ids": [ + "23856594064540489" + ] + }, + { + "id": "661125f7d337811256c5b55996b22c89047804dcec494db72659e4be71e03091", + "audience_ids": [ + "23856594064540489" + ] + }, + {}, + { + "id": "94162773066d6ae88b2658dc58ca2317", + "audience_ids": [ + "23856594064540489" + ] + } + ] + ], + "id_schema": [ + "EMAIL_SHA256", + "PHONE_SHA256", + "IDFA_SHA256", + "AAID_SHA256", + "AAID_MD", + "IDFA_MD5" + ], + "advertiser_ids": [ + "dummyAdverTiserID" + ], + "action": "add" + }, + "JSON_ARRAY": {}, + "XML": {}, + "FORM": {} + }, + "files": {}, + "userId": "" + }, + "metadata": { + "jobId": 1, + "secret": { + "accessToken": "dummyAccessToken", + "advertiserIds": [ + "dummyAdverTiserID" + ] + } + }, + "statusCode": 200 + } + ] + } + } + }, + { + "name": "tiktok_audience", + "description": "Test 4: Considering some null values", + "feature": "processor", + "module": "destination", + "version": "v0", + "input": { + "request": { + "body": [ + { + "message": { + "userId": "user 1", + "type": "audiencelist", + "properties": { + "listData": { + "add": [ + { + "EMAIL_SHA256": "alex@email.com", + "PHONE_SHA256": "+129988776655", + "AAID_MD5": "000999OOOQQQQ", + "IDFA_MD5": "1234lkasfjdalj12321" + }, + { + "EMAIL_SHA256": "amy@abc.com", + "AAID_SHA256": "000999OOOPPPP", + "AAID_MD5": "000999OOOPPPP", + "IDFA_MD5": "1234lkasfjdalj114455" + } + ] + } + }, + "context": { + "ip": "14.5.67.21", + "library": { + "name": "http" + }, + "externalId": [ + { + "type": "TIKTOK_AUDIENCE-23856594064540489", + "identifierType": "EMAIL_SHA256" + } + ], + "destinationFields": "EMAIL_SHA256, PHONE_SHA256, IDFA_SHA256, AAID_SHA256, AAID_MD, IDFA_MD5" + }, + "timestamp": "2020-02-02T00:23:09.544Z" + }, + "metadata": { + "jobId": 1, + "secret": { + "accessToken": "dummyAccessToken", + "advertiserIds": [ + "dummyAdverTiserID" + ] + } + }, + "destination": { + "DestinationDefinition": { + "Config": { + "cdkV2Enabled": true + } + }, + "Config": { + "isHashRequired": true, + "registerDeviceOrBrowserApiKey": true, + "apiKey": "intercomApiKey", + "appId": "9e9cdea1-78fa-4829-a9b2-5d7f7e96d1a0", + "collectContext": false + } + } + } + ] + } + }, + "output": { + "response": { + "status": 200, + "body": [ + { + "output": { + "version": "1", + "type": "REST", + "method": "POST", + "endpoint": "https://business-api.tiktok.com/open_api/v1.3/segment/mapping/", + "headers": { + "Access-Token": "dummyAccessToken", + "Content-Type": "application/json" + }, + "params": {}, + "body": { + "JSON": { + "batch_data": [ + [ + { + "id": "ac0f1baec38a9ef3cfcb56db981df7d9bab2568c7f53ef3776d1c059ec58e72b", + "audience_ids": [ + "23856594064540489" + ] + }, + { + "id": "31e78a3bf9ce2b43316f64fe883a531d6266938091e94e2f2480272481163dee", + "audience_ids": [ + "23856594064540489" + ] + }, + {}, + {}, + {}, + { + "id": "32ee3d063320815a13e0058c2498ff76", + "audience_ids": [ + "23856594064540489" + ] + } + ], + [ + { + "id": "49eaeca26c878f268ad33af8cfa8194ca5b8b8e448b1c775bf9153a2de734579", + "audience_ids": [ + "23856594064540489" + ] + }, + {}, + {}, + { + "id": "661125f7d337811256c5b55996b22c89047804dcec494db72659e4be71e03091", + "audience_ids": [ + "23856594064540489" + ] + }, + {}, + { + "id": "94162773066d6ae88b2658dc58ca2317", + "audience_ids": [ + "23856594064540489" + ] + } + ] + ], + "id_schema": [ + "EMAIL_SHA256", + "PHONE_SHA256", + "IDFA_SHA256", + "AAID_SHA256", + "AAID_MD", + "IDFA_MD5" + ], + "advertiser_ids": [ + "dummyAdverTiserID" + ], + "action": "add" + }, + "JSON_ARRAY": {}, + "XML": {}, + "FORM": {} + }, + "files": {}, + "userId": "" + }, + "metadata": { + "jobId": 1, + "secret": { + "accessToken": "dummyAccessToken", + "advertiserIds": [ + "dummyAdverTiserID" + ] + } + }, + "statusCode": 200 + } + ] + } + } + } +] \ No newline at end of file diff --git a/test/integrations/destinations/tiktok_audience/router/data.ts b/test/integrations/destinations/tiktok_audience/router/data.ts new file mode 100644 index 0000000000..c8a8b93d30 --- /dev/null +++ b/test/integrations/destinations/tiktok_audience/router/data.ts @@ -0,0 +1,833 @@ +export const data = [ + { + name: 'tiktok_audience', + description: 'Multiple jobs with different metadata', + feature: 'router', + module: 'destination', + version: 'v0', + input: { + request: { + body: { + input: [ + { + message: { + userId: 'user 1', + type: 'audiencelist', + properties: { + listData: { + add: [ + { + EMAIL_SHA256: 'alex@email.com', + }, + { + EMAIL_SHA256: 'amy@abc.com', + }, + { + EMAIL_SHA256: 'van@abc.com', + }, + ], + remove: [ + { + EMAIL_SHA256: 'alex@email.com', + }, + { + EMAIL_SHA256: 'amy@abc.com', + }, + { + EMAIL_SHA256: 'van@abc.com', + }, + ], + }, + }, + context: { + ip: '14.5.67.21', + library: { + name: 'http', + }, + externalId: [ + { + type: 'TIKTOK_AUDIENCE-23856594064540489', + identifierType: 'EMAIL_SHA256', + }, + ], + destinationFields: 'EMAIL_SHA256', + }, + timestamp: '2020-02-02T00:23:09.544Z', + }, + metadata: { + jobId: 1, + secret: { + accessToken: 'dummyAccessToken', + advertiserIds: ['dummyAdverTiserID'], + }, + }, + destination: { + DestinationDefinition: { + Config: { + cdkV2Enabled: true, + }, + }, + Config: { + isHashRequired: true, + registerDeviceOrBrowserApiKey: true, + apiKey: 'intercomApiKey', + appId: '9e9cdea1-78fa-4829-a9b2-5d7f7e96d1a0', + collectContext: false, + }, + }, + }, + { + message: { + userId: 'user 1', + type: 'audiencelist', + properties: { + listData: { + add: [ + { + EMAIL_SHA256: 'alex@email.com', + AAID_MD5: '1234567', + }, + { + EMAIL_SHA256: 'amy@abc.com', + AAID_MD5: '1234568', + }, + { + EMAIL_SHA256: 'van@abc.com', + AAID_MD5: '1234569', + }, + ], + remove: [ + { + EMAIL_SHA256: 'alex@email.com', + AAID_MD5: '1234570', + }, + { + EMAIL_SHA256: 'amy@abc.com', + AAID_MD5: '1234571', + }, + { + EMAIL_SHA256: 'van@abc.com', + AAID_MD5: '1234572', + }, + ], + }, + }, + context: { + ip: '14.5.67.21', + library: { + name: 'http', + }, + externalId: [ + { + type: 'TIKTOK_AUDIENCE-23856594064540489', + identifierType: 'EMAIL_SHA256', + }, + ], + destinationFields: 'EMAIL_SHA256, AAID_MD5', + }, + timestamp: '2020-02-02T00:23:09.544Z', + }, + metadata: { + jobId: 2, + secret: { + accessToken: 'dummyAccessToken', + advertiserIds: ['dummyAdverTiserID'], + }, + }, + destination: { + DestinationDefinition: { + Config: { + cdkV2Enabled: true, + }, + }, + Config: { + isHashRequired: true, + registerDeviceOrBrowserApiKey: true, + apiKey: 'intercomApiKey', + appId: '9e9cdea1-78fa-4829-a9b2-5d7f7e96d1a0', + collectContext: false, + }, + }, + }, + { + message: { + userId: 'user 1', + type: 'audiencelist', + properties: { + listData: { + add: [ + { + EMAIL_SHA256: 'alex@email.com', + PHONE_SHA256: '+129988776655', + IDFA_SHA256: '1234lkasfjdalj12321', + AAID_SHA256: '000999OOOQQQQ', + AAID_MD5: '000999OOOQQQQ', + IDFA_MD5: '1234lkasfjdalj12321', + }, + { + EMAIL_SHA256: 'amy@abc.com', + PHONE_SHA256: '+129988776677', + IDFA_SHA256: '1234lkasfjdalj114455', + AAID_SHA256: '000999OOOPPPP', + AAID_MD5: '000999OOOPPPP', + IDFA_MD5: '1234lkasfjdalj114455', + }, + ], + }, + }, + context: { + ip: '14.5.67.21', + library: { + name: 'http', + }, + externalId: [ + { + type: 'TIKTOK_AUDIENCE-23856594064540489', + identifierType: 'EMAIL_SHA256', + }, + ], + destinationFields: + 'EMAIL_SHA256, PHONE_SHA256, IDFA_SHA256, AAID_SHA256, AAID_MD, IDFA_MD5', + }, + timestamp: '2020-02-02T00:23:09.544Z', + }, + metadata: { + jobId: 3, + secret: { + accessToken: 'dummyAccessToken', + advertiserIds: ['dummyAdverTiserID'], + }, + }, + destination: { + DestinationDefinition: { + Config: { + cdkV2Enabled: true, + }, + }, + Config: { + isHashRequired: true, + registerDeviceOrBrowserApiKey: true, + apiKey: 'intercomApiKey', + appId: '9e9cdea1-78fa-4829-a9b2-5d7f7e96d1a0', + collectContext: false, + }, + }, + }, + { + message: { + userId: 'user 1', + type: 'audiencelist', + properties: { + listData: { + add: [ + { + EMAIL_SHA256: 'alex@email.com', + PHONE_SHA256: '+129988776655', + AAID_MD5: '000999OOOQQQQ', + IDFA_MD5: '1234lkasfjdalj12321', + }, + { + EMAIL_SHA256: 'amy@abc.com', + AAID_SHA256: '000999OOOPPPP', + AAID_MD5: '000999OOOPPPP', + IDFA_MD5: '1234lkasfjdalj114455', + }, + ], + }, + }, + context: { + ip: '14.5.67.21', + library: { + name: 'http', + }, + externalId: [ + { + type: 'TIKTOK_AUDIENCE-23856594064540489', + identifierType: 'EMAIL_SHA256', + }, + ], + destinationFields: + 'EMAIL_SHA256, PHONE_SHA256, IDFA_SHA256, AAID_SHA256, AAID_MD, IDFA_MD5', + }, + timestamp: '2020-02-02T00:23:09.544Z', + }, + metadata: { + jobId: 4, + secret: { + accessToken: 'dummyAccessToken', + advertiserIds: ['dummyAdverTiserID'], + }, + }, + destination: { + DestinationDefinition: { + Config: { + cdkV2Enabled: true, + }, + }, + Config: { + isHashRequired: true, + registerDeviceOrBrowserApiKey: true, + apiKey: 'intercomApiKey', + appId: '9e9cdea1-78fa-4829-a9b2-5d7f7e96d1a0', + collectContext: false, + }, + }, + }, + { + message: { + userId: 'user 1', + properties: { + listData: { + add: [ + { + EMAIL_SHA256: 'alex@email.com', + PHONE_SHA256: '+129988776655', + AAID_MD5: '000999OOOQQQQ', + IDFA_MD5: '1234lkasfjdalj12321', + }, + { + EMAIL_SHA256: 'amy@abc.com', + AAID_SHA256: '000999OOOPPPP', + AAID_MD5: '000999OOOPPPP', + IDFA_MD5: '1234lkasfjdalj114455', + }, + ], + }, + }, + context: { + ip: '14.5.67.21', + library: { + name: 'http', + }, + externalId: [ + { + type: 'TIKTOK_AUDIENCE-23856594064540489', + identifierType: 'EMAIL_SHA256', + }, + ], + destinationFields: + 'EMAIL_SHA256, PHONE_SHA256, IDFA_SHA256, AAID_SHA256, AAID_MD, IDFA_MD5', + }, + timestamp: '2020-02-02T00:23:09.544Z', + }, + metadata: { + jobId: 1524545, + secret: { + accessToken: 'dummyAccessToken', + advertiserIds: ['dummyAdverTiserID'], + }, + }, + destination: { + DestinationDefinition: { + Config: { + cdkV2Enabled: true, + }, + }, + Config: { + isHashRequired: true, + registerDeviceOrBrowserApiKey: true, + apiKey: 'intercomApiKey', + appId: '9e9cdea1-78fa-4829-a9b2-5d7f7e96d1a0', + collectContext: false, + }, + }, + }, + ], + destType: 'tiktok_audience', + }, + method: 'POST', + }, + }, + output: { + response: { + status: 200, + body: { + output: [ + { + error: 'message Type is not present. Aborting message.', + batched: false, + destination: { + DestinationDefinition: { + Config: { + cdkV2Enabled: true, + }, + }, + Config: { + isHashRequired: true, + registerDeviceOrBrowserApiKey: true, + apiKey: 'intercomApiKey', + appId: '9e9cdea1-78fa-4829-a9b2-5d7f7e96d1a0', + collectContext: false, + }, + }, + metadata: [ + { + jobId: 1524545, + secret: { + accessToken: 'dummyAccessToken', + advertiserIds: ['dummyAdverTiserID'], + }, + }, + ], + statTags: { + destType: 'TIKTOK_AUDIENCE', + errorCategory: 'dataValidation', + errorType: 'instrumentation', + feature: 'router', + implementation: 'cdkV2', + module: 'destination', + }, + statusCode: 400, + }, + { + batchedRequest: [ + { + version: '1', + type: 'REST', + method: 'POST', + endpoint: 'https://business-api.tiktok.com/open_api/v1.3/segment/mapping/', + headers: { + 'Access-Token': 'dummyAccessToken', + 'Content-Type': 'application/json', + }, + params: {}, + body: { + JSON: { + batch_data: [ + [ + { + id: 'ac0f1baec38a9ef3cfcb56db981df7d9bab2568c7f53ef3776d1c059ec58e72b', + audience_ids: ['23856594064540489'], + }, + ], + [ + { + id: '49eaeca26c878f268ad33af8cfa8194ca5b8b8e448b1c775bf9153a2de734579', + audience_ids: ['23856594064540489'], + }, + ], + [ + { + id: '2048acfa84a01121060ca2fc8a673a76d427176dc37224d4408c21973bd90e5c', + audience_ids: ['23856594064540489'], + }, + ], + ], + id_schema: ['EMAIL_SHA256'], + advertiser_ids: ['dummyAdverTiserID'], + action: 'add', + }, + JSON_ARRAY: {}, + XML: {}, + FORM: {}, + }, + files: {}, + }, + { + version: '1', + type: 'REST', + method: 'POST', + endpoint: 'https://business-api.tiktok.com/open_api/v1.3/segment/mapping/', + headers: { + 'Access-Token': 'dummyAccessToken', + 'Content-Type': 'application/json', + }, + params: {}, + body: { + JSON: { + batch_data: [ + [ + { + id: 'ac0f1baec38a9ef3cfcb56db981df7d9bab2568c7f53ef3776d1c059ec58e72b', + audience_ids: ['23856594064540489'], + }, + ], + [ + { + id: '49eaeca26c878f268ad33af8cfa8194ca5b8b8e448b1c775bf9153a2de734579', + audience_ids: ['23856594064540489'], + }, + ], + [ + { + id: '2048acfa84a01121060ca2fc8a673a76d427176dc37224d4408c21973bd90e5c', + audience_ids: ['23856594064540489'], + }, + ], + ], + id_schema: ['EMAIL_SHA256'], + advertiser_ids: ['dummyAdverTiserID'], + action: 'delete', + }, + JSON_ARRAY: {}, + XML: {}, + FORM: {}, + }, + files: {}, + }, + ], + batched: true, + destination: { + DestinationDefinition: { + Config: { + cdkV2Enabled: true, + }, + }, + Config: { + isHashRequired: true, + registerDeviceOrBrowserApiKey: true, + apiKey: 'intercomApiKey', + appId: '9e9cdea1-78fa-4829-a9b2-5d7f7e96d1a0', + collectContext: false, + }, + }, + metadata: [ + { + jobId: 1, + secret: { + accessToken: 'dummyAccessToken', + advertiserIds: ['dummyAdverTiserID'], + }, + }, + ], + statusCode: 200, + }, + { + batchedRequest: [ + { + version: '1', + type: 'REST', + method: 'POST', + endpoint: 'https://business-api.tiktok.com/open_api/v1.3/segment/mapping/', + headers: { + 'Access-Token': 'dummyAccessToken', + 'Content-Type': 'application/json', + }, + params: {}, + body: { + JSON: { + batch_data: [ + [ + { + id: 'ac0f1baec38a9ef3cfcb56db981df7d9bab2568c7f53ef3776d1c059ec58e72b', + audience_ids: ['23856594064540489'], + }, + { + id: 'fcea920f7412b5da7be0cf42b8c93759', + audience_ids: ['23856594064540489'], + }, + ], + [ + { + id: '49eaeca26c878f268ad33af8cfa8194ca5b8b8e448b1c775bf9153a2de734579', + audience_ids: ['23856594064540489'], + }, + { + id: 'fe743d8d97aa7dfc6c93ccdc2e749513', + audience_ids: ['23856594064540489'], + }, + ], + [ + { + id: '2048acfa84a01121060ca2fc8a673a76d427176dc37224d4408c21973bd90e5c', + audience_ids: ['23856594064540489'], + }, + { + id: 'e36a2f90240e9e84483504fd4a704452', + audience_ids: ['23856594064540489'], + }, + ], + ], + id_schema: ['EMAIL_SHA256', 'AAID_MD5'], + advertiser_ids: ['dummyAdverTiserID'], + action: 'add', + }, + JSON_ARRAY: {}, + XML: {}, + FORM: {}, + }, + files: {}, + }, + { + version: '1', + type: 'REST', + method: 'POST', + endpoint: 'https://business-api.tiktok.com/open_api/v1.3/segment/mapping/', + headers: { + 'Access-Token': 'dummyAccessToken', + 'Content-Type': 'application/json', + }, + params: {}, + body: { + JSON: { + batch_data: [ + [ + { + id: 'ac0f1baec38a9ef3cfcb56db981df7d9bab2568c7f53ef3776d1c059ec58e72b', + audience_ids: ['23856594064540489'], + }, + { + id: 'c1abd65fea29d573ddef1bce925e3276', + audience_ids: ['23856594064540489'], + }, + ], + [ + { + id: '49eaeca26c878f268ad33af8cfa8194ca5b8b8e448b1c775bf9153a2de734579', + audience_ids: ['23856594064540489'], + }, + { + id: '7298110702a080dfc6903f13333eb04a', + audience_ids: ['23856594064540489'], + }, + ], + [ + { + id: '2048acfa84a01121060ca2fc8a673a76d427176dc37224d4408c21973bd90e5c', + audience_ids: ['23856594064540489'], + }, + { + id: 'd9cb68b1fd3b9d32abc5f4cab8b42b68', + audience_ids: ['23856594064540489'], + }, + ], + ], + id_schema: ['EMAIL_SHA256', 'AAID_MD5'], + advertiser_ids: ['dummyAdverTiserID'], + action: 'delete', + }, + JSON_ARRAY: {}, + XML: {}, + FORM: {}, + }, + files: {}, + }, + ], + batched: true, + destination: { + DestinationDefinition: { + Config: { + cdkV2Enabled: true, + }, + }, + Config: { + isHashRequired: true, + registerDeviceOrBrowserApiKey: true, + apiKey: 'intercomApiKey', + appId: '9e9cdea1-78fa-4829-a9b2-5d7f7e96d1a0', + collectContext: false, + }, + }, + metadata: [ + { + jobId: 2, + secret: { + accessToken: 'dummyAccessToken', + advertiserIds: ['dummyAdverTiserID'], + }, + }, + ], + statusCode: 200, + }, + { + batchedRequest: { + version: '1', + type: 'REST', + method: 'POST', + endpoint: 'https://business-api.tiktok.com/open_api/v1.3/segment/mapping/', + headers: { + 'Access-Token': 'dummyAccessToken', + 'Content-Type': 'application/json', + }, + params: {}, + body: { + JSON: { + batch_data: [ + [ + { + id: 'ac0f1baec38a9ef3cfcb56db981df7d9bab2568c7f53ef3776d1c059ec58e72b', + audience_ids: ['23856594064540489'], + }, + { + id: '31e78a3bf9ce2b43316f64fe883a531d6266938091e94e2f2480272481163dee', + audience_ids: ['23856594064540489'], + }, + { + id: '0259f595f7172c8dd692a5c37b4d296939555f862aae8adb964391bdb65006ab', + audience_ids: ['23856594064540489'], + }, + { + id: 'b06fbe7a29f33576a792ba3df3c9bf838cd26ea88cf574285fa60dc0234a8485', + audience_ids: ['23856594064540489'], + }, + {}, + { + id: '32ee3d063320815a13e0058c2498ff76', + audience_ids: ['23856594064540489'], + }, + ], + [ + { + id: '49eaeca26c878f268ad33af8cfa8194ca5b8b8e448b1c775bf9153a2de734579', + audience_ids: ['23856594064540489'], + }, + { + id: 'fb40adc7debbf40e7b45b0a4a91886785dff1a28809276f95f1c44f7045f9b4d', + audience_ids: ['23856594064540489'], + }, + { + id: 'e6bbdf34c5f3472f31b2923a26811560a599233f3dea4c9971595c3bb7b1e8dc', + audience_ids: ['23856594064540489'], + }, + { + id: '661125f7d337811256c5b55996b22c89047804dcec494db72659e4be71e03091', + audience_ids: ['23856594064540489'], + }, + {}, + { + id: '94162773066d6ae88b2658dc58ca2317', + audience_ids: ['23856594064540489'], + }, + ], + ], + id_schema: [ + 'EMAIL_SHA256', + 'PHONE_SHA256', + 'IDFA_SHA256', + 'AAID_SHA256', + 'AAID_MD', + 'IDFA_MD5', + ], + advertiser_ids: ['dummyAdverTiserID'], + action: 'add', + }, + JSON_ARRAY: {}, + XML: {}, + FORM: {}, + }, + files: {}, + }, + batched: true, + destination: { + DestinationDefinition: { + Config: { + cdkV2Enabled: true, + }, + }, + Config: { + isHashRequired: true, + registerDeviceOrBrowserApiKey: true, + apiKey: 'intercomApiKey', + appId: '9e9cdea1-78fa-4829-a9b2-5d7f7e96d1a0', + collectContext: false, + }, + }, + metadata: [ + { + jobId: 3, + secret: { + accessToken: 'dummyAccessToken', + advertiserIds: ['dummyAdverTiserID'], + }, + }, + ], + statusCode: 200, + }, + { + batchedRequest: { + version: '1', + type: 'REST', + method: 'POST', + endpoint: 'https://business-api.tiktok.com/open_api/v1.3/segment/mapping/', + headers: { + 'Access-Token': 'dummyAccessToken', + 'Content-Type': 'application/json', + }, + params: {}, + body: { + JSON: { + batch_data: [ + [ + { + id: 'ac0f1baec38a9ef3cfcb56db981df7d9bab2568c7f53ef3776d1c059ec58e72b', + audience_ids: ['23856594064540489'], + }, + { + id: '31e78a3bf9ce2b43316f64fe883a531d6266938091e94e2f2480272481163dee', + audience_ids: ['23856594064540489'], + }, + {}, + {}, + {}, + { + id: '32ee3d063320815a13e0058c2498ff76', + audience_ids: ['23856594064540489'], + }, + ], + [ + { + id: '49eaeca26c878f268ad33af8cfa8194ca5b8b8e448b1c775bf9153a2de734579', + audience_ids: ['23856594064540489'], + }, + {}, + {}, + { + id: '661125f7d337811256c5b55996b22c89047804dcec494db72659e4be71e03091', + audience_ids: ['23856594064540489'], + }, + {}, + { + id: '94162773066d6ae88b2658dc58ca2317', + audience_ids: ['23856594064540489'], + }, + ], + ], + id_schema: [ + 'EMAIL_SHA256', + 'PHONE_SHA256', + 'IDFA_SHA256', + 'AAID_SHA256', + 'AAID_MD', + 'IDFA_MD5', + ], + advertiser_ids: ['dummyAdverTiserID'], + action: 'add', + }, + JSON_ARRAY: {}, + XML: {}, + FORM: {}, + }, + files: {}, + }, + batched: true, + destination: { + DestinationDefinition: { + Config: { + cdkV2Enabled: true, + }, + }, + Config: { + isHashRequired: true, + registerDeviceOrBrowserApiKey: true, + apiKey: 'intercomApiKey', + appId: '9e9cdea1-78fa-4829-a9b2-5d7f7e96d1a0', + collectContext: false, + }, + }, + metadata: [ + { + jobId: 4, + secret: { + accessToken: 'dummyAccessToken', + advertiserIds: ['dummyAdverTiserID'], + }, + }, + ], + statusCode: 200, + }, + ], + }, + }, + }, + }, +]; diff --git a/test/integrations/sources/revenuecat/data.ts b/test/integrations/sources/revenuecat/data.ts new file mode 100644 index 0000000000..4963781763 --- /dev/null +++ b/test/integrations/sources/revenuecat/data.ts @@ -0,0 +1,286 @@ +export const data = [ + { + name: 'revenuecat', + description: 'Simple track call', + module: 'source', + version: 'v0', + input: { + request: { + body: [ + { + api_version: '1.0', + event: { + aliases: [ + 'f8e14f51-0c76-49ba-8d67-c229f1875dd9', + '389ad6dd-bb40-4c03-9471-1353da2d55ec', + ], + app_user_id: 'f8e14f51-0c76-49ba-8d67-c229f1875dd9', + commission_percentage: null, + country_code: 'US', + currency: null, + entitlement_id: null, + entitlement_ids: null, + environment: 'SANDBOX', + event_timestamp_ms: 1698617217232, + expiration_at_ms: 1698624417232, + id: '8CF0CD6C-CAF3-41FB-968A-661938235AF0', + is_family_share: null, + offer_code: null, + original_app_user_id: 'f8e14f51-0c76-49ba-8d67-c229f1875dd9', + original_transaction_id: null, + period_type: 'NORMAL', + presented_offering_id: null, + price: null, + price_in_purchased_currency: null, + product_id: 'test_product', + purchased_at_ms: 1698617217232, + store: 'APP_STORE', + subscriber_attributes: { + $displayName: { + updated_at_ms: 1698617217232, + value: 'Mister Mistoffelees', + }, + $email: { + updated_at_ms: 1698617217232, + value: 'tuxedo@revenuecat.com', + }, + $phoneNumber: { + updated_at_ms: 1698617217232, + value: '+19795551234', + }, + my_custom_attribute_1: { + updated_at_ms: 1698617217232, + value: 'catnip', + }, + }, + takehome_percentage: null, + tax_percentage: null, + transaction_id: null, + type: 'TEST', + }, + }, + ], + method: 'POST', + headers: { + 'Content-Type': 'application/json', + }, + }, + pathSuffix: '', + }, + output: { + response: { + status: 200, + body: [ + { + output: { + batch: [ + { + context: { + library: { + name: 'unknown', + version: 'unknown', + }, + integration: { + name: 'RevenueCat', + }, + externalId: [ + { + type: 'revenuecatAppUserId', + id: 'f8e14f51-0c76-49ba-8d67-c229f1875dd9', + }, + ], + }, + integrations: { + RevenueCat: false, + }, + type: 'track', + properties: { + aliases: [ + 'f8e14f51-0c76-49ba-8d67-c229f1875dd9', + '389ad6dd-bb40-4c03-9471-1353da2d55ec', + ], + appUserId: 'f8e14f51-0c76-49ba-8d67-c229f1875dd9', + commissionPercentage: null, + countryCode: 'US', + currency: null, + entitlementId: null, + entitlementIds: null, + environment: 'SANDBOX', + eventTimestampMs: 1698617217232, + expirationAtMs: 1698624417232, + id: '8CF0CD6C-CAF3-41FB-968A-661938235AF0', + isFamilyShare: null, + offerCode: null, + originalAppUserId: 'f8e14f51-0c76-49ba-8d67-c229f1875dd9', + originalTransactionId: null, + periodType: 'NORMAL', + presentedOfferingId: null, + price: null, + priceInPurchasedCurrency: null, + productId: 'test_product', + purchasedAtMs: 1698617217232, + store: 'APP_STORE', + subscriberAttributes: { + $displayName: { + updated_at_ms: 1698617217232, + value: 'Mister Mistoffelees', + }, + $email: { + updated_at_ms: 1698617217232, + value: 'tuxedo@revenuecat.com', + }, + $phoneNumber: { + updated_at_ms: 1698617217232, + value: '+19795551234', + }, + my_custom_attribute_1: { + updated_at_ms: 1698617217232, + value: 'catnip', + }, + }, + takehomePercentage: null, + taxPercentage: null, + transactionId: null, + type: 'TEST', + }, + event: 'TEST', + messageId: '8CF0CD6C-CAF3-41FB-968A-661938235AF0', + originalTimestamp: '2023-10-29T22:06:57.232Z', + sentAt: '2023-10-29T22:06:57.232Z', + }, + ], + }, + }, + ], + }, + }, + }, + { + name: 'revenuecat', + description: 'Initial purchase event', + module: 'source', + version: 'v0', + input: { + request: { + body: [ + { + api_version: '1.0', + event: { + aliases: ['yourCustomerAliasedID', 'yourCustomerAliasedID'], + app_id: 'yourAppID', + app_user_id: 'yourCustomerAppUserID', + commission_percentage: 0.3, + country_code: 'US', + currency: 'USD', + entitlement_id: 'pro_cat', + entitlement_ids: ['pro_cat'], + environment: 'PRODUCTION', + event_timestamp_ms: 1591121855319, + expiration_at_ms: 1591726653000, + id: 'UniqueIdentifierOfEvent', + is_family_share: false, + offer_code: 'free_month', + original_app_user_id: 'OriginalAppUserID', + original_transaction_id: '1530648507000', + period_type: 'NORMAL', + presented_offering_id: 'OfferingID', + price: 2.49, + price_in_purchased_currency: 2.49, + product_id: 'onemonth_no_trial', + purchased_at_ms: 1591121853000, + store: 'APP_STORE', + subscriber_attributes: { + '$Favorite Cat': { + updated_at_ms: 1581121853000, + value: 'Garfield', + }, + }, + takehome_percentage: 0.7, + tax_percentage: 0.3, + transaction_id: '170000869511114', + type: 'INITIAL_PURCHASE', + }, + }, + ], + method: 'POST', + headers: { + 'Content-Type': 'application/json', + }, + }, + pathSuffix: '', + }, + output: { + response: { + status: 200, + body: [ + { + output: { + batch: [ + { + context: { + library: { + name: 'unknown', + version: 'unknown', + }, + integration: { + name: 'RevenueCat', + }, + externalId: [ + { + type: 'revenuecatAppUserId', + id: 'yourCustomerAppUserID', + }, + ], + }, + integrations: { + RevenueCat: false, + }, + type: 'track', + properties: { + aliases: ['yourCustomerAliasedID', 'yourCustomerAliasedID'], + appId: 'yourAppID', + appUserId: 'yourCustomerAppUserID', + commissionPercentage: 0.3, + countryCode: 'US', + currency: 'USD', + entitlementId: 'pro_cat', + entitlementIds: ['pro_cat'], + environment: 'PRODUCTION', + eventTimestampMs: 1591121855319, + expirationAtMs: 1591726653000, + id: 'UniqueIdentifierOfEvent', + isFamilyShare: false, + offerCode: 'free_month', + originalAppUserId: 'OriginalAppUserID', + originalTransactionId: '1530648507000', + periodType: 'NORMAL', + presentedOfferingId: 'OfferingID', + price: 2.49, + priceInPurchasedCurrency: 2.49, + productId: 'onemonth_no_trial', + purchasedAtMs: 1591121853000, + store: 'APP_STORE', + subscriberAttributes: { + '$Favorite Cat': { + updated_at_ms: 1581121853000, + value: 'Garfield', + }, + }, + takehomePercentage: 0.7, + taxPercentage: 0.3, + transactionId: '170000869511114', + type: 'INITIAL_PURCHASE', + }, + event: 'INITIAL_PURCHASE', + messageId: 'UniqueIdentifierOfEvent', + originalTimestamp: '2020-06-02T18:17:35.319Z', + sentAt: '2020-06-02T18:17:35.319Z', + }, + ], + }, + }, + ], + }, + }, + }, +];