diff --git a/src/services/userTransform.ts b/src/services/userTransform.ts index 74d9203188..433d27e78d 100644 --- a/src/services/userTransform.ts +++ b/src/services/userTransform.ts @@ -8,18 +8,21 @@ import { UserTransformationResponse, UserTransformationServiceResponse, } from '../types/index'; -import { RespStatusError, RetryRequestError, extractStackTraceUptoLastSubstringMatch } from '../util/utils'; +import { + RespStatusError, + RetryRequestError, + extractStackTraceUptoLastSubstringMatch, +} from '../util/utils'; import { getMetadata, isNonFuncObject } from '../v0/util'; import { SUPPORTED_FUNC_NAMES } from '../util/ivmFactory'; import logger from '../logger'; import stats from '../util/stats'; +import { CommonUtils } from '../util/common'; export default class UserTransformService { public static async transformRoutine( events: ProcessorTransformationRequest[], ): Promise { - - const startTime = new Date(); let retryStatus = 200; const groupedEvents: Object = groupBy( events, @@ -43,6 +46,8 @@ export default class UserTransformService { const transformationVersionId = eventsToProcess[0]?.destination?.Transformations[0]?.VersionID; const messageIds = eventsToProcess.map((ev) => ev.metadata?.messageId); + const messageIdsSet = new Set(messageIds); + const messageIdsInOutputSet = new Set(); const commonMetadata = { sourceId: eventsToProcess[0]?.metadata?.sourceId, @@ -76,31 +81,57 @@ export default class UserTransformService { transformationVersionId, librariesVersionIDs, ); - transformedEvents.push( - ...destTransformedEvents.map((ev) => { - if (ev.error) { - return { - statusCode: 400, - error: ev.error, - metadata: isEmpty(ev.metadata) ? commonMetadata : ev.metadata, - } as ProcessorTransformationResponse; - } - if (!isNonFuncObject(ev.transformedEvent)) { - return { - statusCode: 400, - error: `returned event in events from user transformation is not an object. transformationVersionId:${transformationVersionId} and returned event: ${JSON.stringify( - ev.transformedEvent, - )}`, - metadata: isEmpty(ev.metadata) ? commonMetadata : ev.metadata, - } as ProcessorTransformationResponse; - } - return { - output: ev.transformedEvent, + + const transformedEventsWithMetadata: ProcessorTransformationResponse[] = []; + destTransformedEvents.forEach((ev) => { + if (ev.error) { + transformedEventsWithMetadata.push({ + statusCode: 400, + error: ev.error, metadata: isEmpty(ev.metadata) ? commonMetadata : ev.metadata, - statusCode: 200, - } as ProcessorTransformationResponse; - }), - ); + } as ProcessorTransformationResponse); + return; + } + if (!isNonFuncObject(ev.transformedEvent)) { + transformedEventsWithMetadata.push({ + statusCode: 400, + error: `returned event in events from user transformation is not an object. transformationVersionId:${transformationVersionId} and returned event: ${JSON.stringify( + ev.transformedEvent, + )}`, + metadata: isEmpty(ev.metadata) ? commonMetadata : ev.metadata, + } as ProcessorTransformationResponse); + return; + } + // add messageId to output set + if (ev.metadata?.messageId) { + messageIdsInOutputSet.add(ev.metadata.messageId); + } else if (ev.metadata?.messageIds) { + ev.metadata.messageIds.forEach((id) => messageIdsInOutputSet.add(id)); + } + transformedEventsWithMetadata.push({ + output: ev.transformedEvent, + metadata: isEmpty(ev.metadata) ? commonMetadata : ev.metadata, + statusCode: 200, + } as ProcessorTransformationResponse); + }); + + // TODO: add feature flag based on rudder-server version to do this + // find difference between input and output messageIds + logger.debug(`messageIdsInOutputSet: ${messageIdsInOutputSet.keys()}`); + logger.debug(`messageIdsSet: ${messageIdsSet.keys()}`); + const messageIdsNotInOutput = CommonUtils.setDiff(messageIdsSet, messageIdsInOutputSet); + logger.debug(`messageIdsNotInOutput: ${messageIdsNotInOutput}`); + const droppedEvents = messageIdsNotInOutput.map((id) => ({ + statusCode: 298, + metadata: { + ...commonMetadata, + messageId: id, + messageIds: null, + }, + })); + transformedEvents.push(...droppedEvents); + + transformedEvents.push(...transformedEventsWithMetadata); } catch (error: any) { logger.error(error); let status = 400; @@ -172,7 +203,9 @@ export default class UserTransformService { response.status = 200; } catch (error: any) { response.status = 400; - response.body = { error: extractStackTraceUptoLastSubstringMatch(error.stack, SUPPORTED_FUNC_NAMES) }; + response.body = { + error: extractStackTraceUptoLastSubstringMatch(error.stack, SUPPORTED_FUNC_NAMES), + }; } return response; } diff --git a/src/util/common.js b/src/util/common.js index 5d732d34d1..8bf34f2eca 100644 --- a/src/util/common.js +++ b/src/util/common.js @@ -20,6 +20,10 @@ const CommonUtils = { } return [obj]; }, + + setDiff(mainSet, comparisionSet) { + return [...mainSet].filter((item) => !comparisionSet.has(item)); + }, }; module.exports = { diff --git a/test/__tests__/data/user_transformation_service_filter_input.json b/test/__tests__/data/user_transformation_service_filter_input.json new file mode 100644 index 0000000000..3b8425562d --- /dev/null +++ b/test/__tests__/data/user_transformation_service_filter_input.json @@ -0,0 +1,185 @@ +[ + { + "message": { + "channel": "web", + "context": { + "app": { + "build": "1.0.0", + "name": "RudderLabs JavaScript SDK", + "namespace": "com.rudderlabs.javascript", + "version": "1.0.0" + }, + "traits": { + "anonymousId": "123456" + }, + "library": { + "name": "RudderLabs JavaScript SDK", + "version": "1.0.0" + }, + "userAgent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.90 Safari/537.36", + "locale": "en-US", + "ip": "0.0.0.0", + "os": { + "name": "", + "version": "" + }, + "screen": { + "density": 2 + } + }, + "type": "identify", + "messageId": "84e26acc-56a5-4835-8233-591137fca468", + "originalTimestamp": "2019-10-14T09:03:17.562Z", + "anonymousId": "123456", + "userId": "123456", + "integrations": { + "All": true + }, + "sentAt": "2019-10-14T09:03:22.563Z" + }, + "metadata": { + "messageId": "84e26acc-56a5-4835-8233-591137fca468", + "sourceId": "s1", + "destinationId": "d1" + }, + "destination": { + "ID": 2, + "Config": { + "trackingID": "abcd" + }, + "Enabled": true, + "Transformations": [ + { + "VersionID": "24" + } + ] + } + }, + { + "message": { + "channel": "web", + "context": { + "app": { + "build": "1.0.0", + "name": "RudderLabs JavaScript SDK", + "namespace": "com.rudderlabs.javascript", + "version": "1.0.0" + }, + "traits": { + "email": "test@rudderstack.com", + "anonymousId": "12345" + }, + "library": { + "name": "RudderLabs JavaScript SDK", + "version": "1.0.0" + }, + "userAgent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.90 Safari/537.36", + "locale": "en-US", + "ip": "0.0.0.0", + "os": { + "name": "", + "version": "" + }, + "screen": { + "density": 2 + } + }, + "type": "page", + "messageId": "5e10d13a-bf9a-44bf-b884-43a9e591ea71", + "originalTimestamp": "2019-10-14T11:15:18.299Z", + "anonymousId": "00000000000000000000000000", + "userId": "12345", + "properties": { + "path": "/abc", + "referrer": "", + "search": "", + "title": "", + "url": "" + }, + "integrations": { + "All": true + }, + "name": "ApplicationLoaded", + "sentAt": "2019-10-14T11:15:53.296Z" + }, + "metadata": { + "messageId": "5e10d13a-bf9a-44bf-b884-43a9e591ea71", + "sourceId": "s1", + "destinationId": "d1" + }, + "destination": { + "ID": 2, + "Config": { + "trackingID": "abcd" + }, + "Enabled": true, + "Transformations": [ + { + "VersionID": "24" + } + ] + } + }, + { + "message": { + "channel": "web", + "context": { + "app": { + "build": "1.0.0", + "name": "RudderLabs JavaScript SDK", + "namespace": "com.rudderlabs.javascript", + "version": "1.0.0" + }, + "traits": { + "email": "test@rudderstack.com", + "anonymousId": "12345" + }, + "library": { + "name": "RudderLabs JavaScript SDK", + "version": "1.0.0" + }, + "userAgent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.90 Safari/537.36", + "locale": "en-US", + "ip": "0.0.0.0", + "os": { + "name": "", + "version": "" + }, + "screen": { + "density": 2 + } + }, + "type": "track", + "messageId": "ec5481b6-a926-4d2e-b293-0b3a77c4d3be", + "originalTimestamp": "2019-10-14T11:15:18.300Z", + "anonymousId": "00000000000000000000000000", + "userId": "12345", + "event": "test track event GA3", + "properties": { + "user_actual_role": "system_admin, system_user", + "user_actual_id": 12345 + }, + "integrations": { + "All": true + }, + "sentAt": "2019-10-14T11:15:53.296Z" + }, + "metadata": { + "messageId": "ec5481b6-a926-4d2e-b293-0b3a77c4d3be", + "sourceId": "s1", + "destinationId": "d1" + }, + "destination": { + "ID": 2, + "Config": { + "trackingID": "abcd" + }, + "Enabled": true, + "Transformations": [ + { + "VersionID": "24" + } + ] + } + } +] diff --git a/test/__tests__/data/user_transformation_service_filter_output.json b/test/__tests__/data/user_transformation_service_filter_output.json new file mode 100644 index 0000000000..64d8fd9261 --- /dev/null +++ b/test/__tests__/data/user_transformation_service_filter_output.json @@ -0,0 +1,74 @@ +{ + "transformedEvents": [ + { + "statusCode": 298, + "metadata": { + "sourceId": "s1", + "destinationId": "d1", + "messageIds": null, + "messageId": "84e26acc-56a5-4835-8233-591137fca468" + } + }, + { + "statusCode": 298, + "metadata": { + "sourceId": "s1", + "destinationId": "d1", + "messageIds": null, + "messageId": "5e10d13a-bf9a-44bf-b884-43a9e591ea71" + } + }, + { + "output": { + "channel": "web", + "context": { + "app": { + "build": "1.0.0", + "name": "RudderLabs JavaScript SDK", + "namespace": "com.rudderlabs.javascript", + "version": "1.0.0" + }, + "traits": { + "email": "test@rudderstack.com", + "anonymousId": "12345" + }, + "library": { + "name": "RudderLabs JavaScript SDK", + "version": "1.0.0" + }, + "userAgent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.90 Safari/537.36", + "locale": "en-US", + "ip": "0.0.0.0", + "os": { + "name": "", + "version": "" + }, + "screen": { + "density": 2 + } + }, + "type": "track", + "messageId": "ec5481b6-a926-4d2e-b293-0b3a77c4d3be", + "originalTimestamp": "2019-10-14T11:15:18.300Z", + "anonymousId": "00000000000000000000000000", + "userId": "12345", + "event": "test track event GA3", + "properties": { + "user_actual_role": "system_admin, system_user", + "user_actual_id": 12345 + }, + "integrations": { + "All": true + }, + "sentAt": "2019-10-14T11:15:53.296Z" + }, + "metadata": { + "messageId": "ec5481b6-a926-4d2e-b293-0b3a77c4d3be", + "sourceId": "s1", + "destinationId": "d1" + }, + "statusCode": 200 + } + ], + "retryStatus": 200 +} diff --git a/test/__tests__/user_transformation_ts.test.ts b/test/__tests__/user_transformation_ts.test.ts new file mode 100644 index 0000000000..6db46a4704 --- /dev/null +++ b/test/__tests__/user_transformation_ts.test.ts @@ -0,0 +1,39 @@ +import UserTransformService from '../../src/services/userTransform'; +import fetch from 'node-fetch'; +jest.mock('node-fetch', () => jest.fn()); + +const integration = 'user_transformation_service'; +const name = 'User Transformations'; +const randomID = () => Math.random().toString(36).substring(2, 15); + +describe('User Transform Service', () => { + fit(`Filtering ${name} Test`, async () => { + const versionId = '24'; // set in input file + const inputData = require(`./data/${integration}_filter_input.json`); + const expectedData = require(`./data/${integration}_filter_output.json`); + + const respBody = { + codeVersion: '1', + name, + versionId: versionId, + code: `export async function transformEvent(event, metadata) { + const eventType = event.type; + log(eventType); + log(eventType.match(/track/g)); + if(eventType && !eventType.match(/track/g)) return; + return event; + } + `, + }; + (fetch as jest.MockedFunction).mockResolvedValue({ + status: 200, + json: jest.fn().mockResolvedValue(respBody), + }); + const output = await UserTransformService.transformRoutine(inputData); + expect(fetch).toHaveBeenCalledWith( + `https://api.rudderlabs.com/transformation/getByVersionId?versionId=${versionId}`, + ); + + expect(output).toEqual(expectedData); + }); +});