diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index 5b716b7e9f..fe308ce8b9 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -2,21 +2,15 @@ Write a brief explainer on your code changes. -## Please explain the objectives of your changes below - -Put down any required details on the broader aspect of your changes. If there are any dependent changes, **mandatorily** mention them here - -### Type of change +## What is the related Linear task? -If the pull request is a **bug-fix**, **enhancement** or a **refactor**, please fill in the details on the changes made. +Resolves INT-XXX -- Existing capabilities/behavior - -- New capabilities/behavior +## Please explain the objectives of your changes below -If the pull request is a **new feature**, +Put down any required details on the broader aspect of your changes. If there are any dependent changes, **mandatorily** mention them here -### Any technical or performance related pointers to consider with the change? +### Any changes to existing capabilities/behaviour, mention the reason & what are the changes ? N/A @@ -28,11 +22,7 @@ N/A N/A -### If the PR has changes in more than 10 files, please mention why the changes were not split into multiple PRs. - -N/A - -### If multiple linear tasks are associated with the PR changes, please elaborate on the reason: +### Any technical or performance related pointers to consider with the change? N/A @@ -40,13 +30,19 @@ N/A ### Developer checklist +- [ ] My code follows the style guidelines of this project + - [ ] **No breaking changes are being introduced.** -- [ ] Are all related docs linked with the PR? +- [ ] All related docs linked with the PR? + +- [ ] All changes manually tested? + +- [ ] Any documentation changes needed with this change? -- [ ] Are all changes manually tested? +- [ ] Is the PR limited to 10 file changes? -- [ ] Does this change require any documentation changes? +- [ ] Is the PR limited to one linear task? - [ ] Are relevant unit and component test-cases added? diff --git a/.nvmrc b/.nvmrc index 6d80269a4f..a9d087399d 100644 --- a/.nvmrc +++ b/.nvmrc @@ -1 +1 @@ -18.16.0 +18.19.0 diff --git a/CHANGELOG.md b/CHANGELOG.md index f6bca95652..8933728ccf 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,13 @@ All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines. +### [1.52.4](https://github.com/rudderlabs/rudder-transformer/compare/v1.52.3...v1.52.4) (2023-12-27) + + +### Bug Fixes + +* send empty string instead of null when the schema data is undefined ([#2955](https://github.com/rudderlabs/rudder-transformer/issues/2955)) ([511741e](https://github.com/rudderlabs/rudder-transformer/commit/511741ed356b365f52e0335ce6a1fc953ccbc465)) + ### [1.52.3](https://github.com/rudderlabs/rudder-transformer/compare/v1.52.2...v1.52.3) (2023-12-18) diff --git a/Dockerfile b/Dockerfile index a568ce95c7..6bd03c9515 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,5 @@ # syntax=docker/dockerfile:1.4 -FROM node:18.17-alpine3.17 AS base +FROM node:18.19.0-alpine3.18 AS base ENV HUSKY 0 RUN apk update diff --git a/package-lock.json b/package-lock.json index 3fd630c9ec..d004b56151 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "rudder-transformer", - "version": "1.52.3", + "version": "1.52.4", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "rudder-transformer", - "version": "1.52.3", + "version": "1.52.4", "license": "ISC", "dependencies": { "@amplitude/ua-parser-js": "^0.7.24", @@ -3710,41 +3710,6 @@ "tslib": "^2.6.2" } }, - "node_modules/@rudderstack/workflow-engine/node_modules/@smithy/is-array-buffer": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/@smithy/is-array-buffer/-/is-array-buffer-2.0.0.tgz", - "integrity": "sha512-z3PjFjMyZNI98JFRJi/U0nGoLWMSJlDjAW4QUX2WNZLas5C0CmVV6LJ01JI0k90l7FvpmixjWxPFmENSClQ7ug==", - "dependencies": { - "tslib": "^2.5.0" - }, - "engines": { - "node": ">=14.0.0" - } - }, - "node_modules/@rudderstack/workflow-engine/node_modules/@smithy/util-buffer-from": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/@smithy/util-buffer-from/-/util-buffer-from-2.0.0.tgz", - "integrity": "sha512-/YNnLoHsR+4W4Vf2wL5lGv0ksg8Bmk3GEGxn2vEQt52AQaPSCuaO5PM5VM7lP1K9qHRKHwrPGktqVoAHKWHxzw==", - "dependencies": { - "@smithy/is-array-buffer": "^2.0.0", - "tslib": "^2.5.0" - }, - "engines": { - "node": ">=14.0.0" - } - }, - "node_modules/@rudderstack/workflow-engine/node_modules/@smithy/util-utf8": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/@smithy/util-utf8/-/util-utf8-2.0.0.tgz", - "integrity": "sha512-rctU1VkziY84n5OXe3bPNpKR001ZCME2JCaBBFgtiM2hfKbHFudc/BkMuPab8hRbLd0j3vbnBTTZ1igBf0wgiQ==", - "dependencies": { - "@smithy/util-buffer-from": "^2.0.0", - "tslib": "^2.5.0" - }, - "engines": { - "node": ">=14.0.0" - } - }, "node_modules/@sideway/address": { "version": "4.1.4", "license": "BSD-3-Clause", diff --git a/package.json b/package.json index dae8a4cbe0..069fc644b7 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "rudder-transformer", - "version": "1.52.3", + "version": "1.52.4", "description": "", "homepage": "https://github.com/rudderlabs/rudder-transformer#readme", "bugs": { diff --git a/src/v0/destinations/am/transform.js b/src/v0/destinations/am/transform.js index 05a130d6e0..911ec51be0 100644 --- a/src/v0/destinations/am/transform.js +++ b/src/v0/destinations/am/transform.js @@ -297,6 +297,15 @@ const identifyBuilder = (message, destination, rawPayload) => { } }); } + // update identify call request with unset fields + // AM docs https://www.docs.developers.amplitude.com/analytics/apis/http-v2-api/#keys-for-the-event-argument:~:text=exceed%2040%20layers.-,user_properties,-Optional.%20Object.%20A + const unsetObject = AMUtils.getUnsetObj(message); + if (unsetObject) { + // Example unsetObject = { + // "testObj.del1": "-" + // } + set(rawPayload, `user_properties.$unset`, unsetObject); + } return rawPayload; }; @@ -334,7 +343,7 @@ const getResponseData = (evType, destination, rawPayload, message, groupInfo) => case EventType.IDENTIFY: // event_type for identify event is $identify rawPayload.event_type = IDENTIFY_AM; - identifyBuilder(message, destination, rawPayload); + rawPayload = identifyBuilder(message, destination, rawPayload); break; case EventType.GROUP: // event_type for identify event is $identify diff --git a/src/v0/destinations/am/util.test.js b/src/v0/destinations/am/util.test.js new file mode 100644 index 0000000000..faaa9170f0 --- /dev/null +++ b/src/v0/destinations/am/util.test.js @@ -0,0 +1,66 @@ +const { getUnsetObj } = require('./utils'); + +describe('getUnsetObj', () => { + it("should return undefined when 'message.integrations.Amplitude.fieldsToUnset' is not array", () => { + const message = { + integrations: { + Amplitude: { fieldsToUnset: 'field_name' }, + }, + }; + const result = getUnsetObj(message); + expect(result).toBeUndefined(); + }); + it("should return undefined when 'message.integrations.Amplitude.fieldsToUnset' is undefined", () => { + const message = { + integrations: { + Amplitude: {}, + }, + }; + const result = getUnsetObj(message); + expect(result).toBeUndefined(); + }); + + it("should return an empty objecty when 'message.integrations.Amplitude.fieldsToUnset' is an empty array", () => { + const message = { + integrations: { + Amplitude: { fieldsToUnset: [] }, + }, + }; + const result = getUnsetObj(message); + expect(result).toEqual({}); + }); + + it("should return an object with keys and values set to '-' when 'message.integrations.Amplitude.fieldsToUnset' is an array of strings", () => { + const message = { + integrations: { + Amplitude: { fieldsToUnset: ['Unset1', 'Unset2'] }, + }, + }; + const result = getUnsetObj(message); + expect(result).toEqual({ + Unset1: '-', + Unset2: '-', + }); + }); + + it("should handle missing 'message' parameter", () => { + const result = getUnsetObj(); + expect(result).toBeUndefined(); + }); + + // Should handle missing 'integrations' property in 'message' parameter + it("should handle missing 'integrations' property in 'message' parameter", () => { + const message = {}; + const result = getUnsetObj(message); + expect(result).toBeUndefined(); + }); + + // Should handle missing 'Amplitude' property in 'message.integrations' parameter + it("should handle missing 'Amplitude' property in 'message.integrations' parameter", () => { + const message = { + integrations: {}, + }; + const result = getUnsetObj(message); + expect(result).toBeUndefined(); + }); +}); diff --git a/src/v0/destinations/am/utils.js b/src/v0/destinations/am/utils.js index b9925c20d8..71fe0ab459 100644 --- a/src/v0/destinations/am/utils.js +++ b/src/v0/destinations/am/utils.js @@ -82,6 +82,32 @@ const getEventId = (payload, sourceKey) => { return undefined; }; +/** + * generates the unsetObject and returns it + * @param {*} message + * @returns + * + * Example message = { + integrations: { + Amplitude: { fieldsToUnset: ['Unset1', 'Unset2'] }, + All: true, + }, + }; + return unsetObj = { + "Unset1": "-", + "Unset2": "-" + } + AM docs: https://www.docs.developers.amplitude.com/analytics/apis/http-v2-api/#keys-for-the-event-argument:~:text=exceed%2040%20layers.-,user_properties,-Optional.%20Object.%20A + */ +const getUnsetObj = (message) => { + const fieldsToUnset = get(message, 'integrations.Amplitude.fieldsToUnset'); + let unsetObject; + if (Array.isArray(fieldsToUnset)) { + unsetObject = Object.fromEntries(fieldsToUnset.map((field) => [field, '-'])); + } + + return unsetObject; +}; module.exports = { getOSName, getOSVersion, @@ -90,4 +116,5 @@ module.exports = { getPlatform, getBrand, getEventId, + getUnsetObj, }; diff --git a/src/v0/destinations/fb_custom_audience/util.js b/src/v0/destinations/fb_custom_audience/util.js index 47ccb9bf7d..6c53ed2814 100644 --- a/src/v0/destinations/fb_custom_audience/util.js +++ b/src/v0/destinations/fb_custom_audience/util.js @@ -131,21 +131,21 @@ const getUpdatedDataElement = (dataElement, isHashRequired, eachProperty, update /** * hash the original value for the properties apart from 'MADID' && 'EXTERN_ID as hashing is not required for them * ref: https://developers.facebook.com/docs/marketing-api/audiences/guides/custom-audiences#hash - * sending null values for the properties for which user hasn't provided any value + * sending empty string for the properties for which user hasn't provided any value */ if (isHashRequired && eachProperty !== 'MADID' && eachProperty !== 'EXTERN_ID') { if (tmpUpdatedProperty) { tmpUpdatedProperty = `${tmpUpdatedProperty}`; dataElement.push(sha256(tmpUpdatedProperty)); } else { - dataElement.push(null); + dataElement.push(''); } } - // if property name is MADID or EXTERN_ID if the value is undefined send null + // if property name is MADID or EXTERN_ID if the value is undefined send empty string else if (!tmpUpdatedProperty && (eachProperty === 'MADID' || eachProperty === 'EXTERN_ID')) { - dataElement.push(null); + dataElement.push(''); } else { - dataElement.push(tmpUpdatedProperty); + dataElement.push(tmpUpdatedProperty || ''); } return dataElement; }; diff --git a/test/integrations/destinations/am/processor/data.ts b/test/integrations/destinations/am/processor/data.ts index 5e941e07f8..f28606da0c 100644 --- a/test/integrations/destinations/am/processor/data.ts +++ b/test/integrations/destinations/am/processor/data.ts @@ -488,6 +488,10 @@ export const data = [ { message: { channel: 'web', + integrations: { + Amplitude: { fieldsToUnset: ['email'] }, + All: true, + }, context: { externalId: [ { @@ -562,9 +566,6 @@ export const data = [ originalTimestamp: '2019-10-14T09:03:17.562Z', anonymousId: '123456', userId: '123456', - integrations: { - All: true, - }, sentAt: '2019-10-14T09:03:22.563Z', }, metadata: { @@ -616,6 +617,9 @@ export const data = [ insert_id: '84e26acc-56a5-4835-8233-591137fca468', ip: '0.0.0.0', user_properties: { + $unset: { + email: '-', + }, initial_referrer: 'https://docs.rudderstack.com', initial_referring_domain: 'docs.rudderstack.com', anonymousId: '123456', @@ -743,6 +747,7 @@ export const data = [ anonymousId: '123456', userId: '123456', integrations: { + Amplitude: { fieldsToUnset: ['testObj.unsetField1'] }, All: true, }, sentAt: '2019-10-14T09:03:22.563Z', @@ -791,6 +796,9 @@ export const data = [ insert_id: '84e26acc-56a5-4835-8233-591137fca468', ip: '0.0.0.0', user_properties: { + $unset: { + 'testObj.unsetField1': '-', + }, initial_referrer: 'https://docs.rudderstack.com', initial_referring_domain: 'docs.rudderstack.com', anonymousId: '123456', diff --git a/test/integrations/destinations/fb_custom_audience/processor/data.ts b/test/integrations/destinations/fb_custom_audience/processor/data.ts index 4f892f6fef..267b966865 100644 --- a/test/integrations/destinations/fb_custom_audience/processor/data.ts +++ b/test/integrations/destinations/fb_custom_audience/processor/data.ts @@ -1987,7 +1987,7 @@ export const data = [ 'a953f09a1b6b6725b81956e9ad0b1eb49e3ad40004c04307ef8af6246a054116', '3fdba35f04dc8c462986c992bcf875546257113072a909c162f7e470e581e278', '7931aa2a1bed855457d1ddf6bc06ab4406a9fba0579045a4d6ff78f9c07c440f', - null, + '', '252f10c83610ebca1a059c0bae8255eba2f95be4d1d7bcfa89d7248a82d9f111', 'db0683221aebc02cc034b65ebcf7d1bddd1eb199e33fd23a31931947d13a11bc', 'abc', @@ -2123,7 +2123,7 @@ export const data = [ 'ST', 'COUNTRY', ], - data: [[null, null, null, null, null, null, null, null, null, null, null]], + data: [['', '', '', '', '', '', '', '', '', '', '']], }, }, body: { @@ -2256,7 +2256,7 @@ export const data = [ 'a953f09a1b6b6725b81956e9ad0b1eb49e3ad40004c04307ef8af6246a054116', '3fdba35f04dc8c462986c992bcf875546257113072a909c162f7e470e581e278', '7931aa2a1bed855457d1ddf6bc06ab4406a9fba0579045a4d6ff78f9c07c440f', - null, + '', '252f10c83610ebca1a059c0bae8255eba2f95be4d1d7bcfa89d7248a82d9f111', 'db0683221aebc02cc034b65ebcf7d1bddd1eb199e33fd23a31931947d13a11bc', 'abc', diff --git a/test/integrations/destinations/fb_custom_audience/router/data.ts b/test/integrations/destinations/fb_custom_audience/router/data.ts index efefb80a89..fd099fe33c 100644 --- a/test/integrations/destinations/fb_custom_audience/router/data.ts +++ b/test/integrations/destinations/fb_custom_audience/router/data.ts @@ -26757,16 +26757,16 @@ export const data = [ payload: { schema: ['EMAIL', 'FN'], data: [ - ['7625cab24612c37df6d2f724721bb38a25095d0295e29b807238ee188b8aca43', null], - ['b2b4abadd72190af54305c0d3abf1977fec4935016bb13ff28040d5712318dfd', null], - ['c4b007d1c3c9a5d31bd4082237a913e8e0db1767225c2a5ef33be2716df005fa', null], - ['94639be1bd9f17c05820164e9d71ef78558f117a9e8bfab43cf8015e08aa0b27', null], - ['39b456cfb4bb07f9e6bb18698aa173171ca49c731fccc4790e9ecea808d24ae6', null], - ['769f73387add781a481ca08300008a08fb2f1816aaed196137efc2e05976d711', null], - ['48ddb93f0b30c475423fe177832912c5bcdce3cc72872f8051627967ef278e08', null], - ['da2d431121cd10578fd81f8f80344b06db59ea2d05a7b5d27536c8789ddae8f0', null], - ['b100c2ec0718fe6b4805b623aeec6710719d042ceea55f5c8135b010ec1c7b36', null], - ['0c1d1b0ba547a742013366d6fbc8f71dd77f566d94e41ed9f828a74b96928161', null], + ['7625cab24612c37df6d2f724721bb38a25095d0295e29b807238ee188b8aca43', ''], + ['b2b4abadd72190af54305c0d3abf1977fec4935016bb13ff28040d5712318dfd', ''], + ['c4b007d1c3c9a5d31bd4082237a913e8e0db1767225c2a5ef33be2716df005fa', ''], + ['94639be1bd9f17c05820164e9d71ef78558f117a9e8bfab43cf8015e08aa0b27', ''], + ['39b456cfb4bb07f9e6bb18698aa173171ca49c731fccc4790e9ecea808d24ae6', ''], + ['769f73387add781a481ca08300008a08fb2f1816aaed196137efc2e05976d711', ''], + ['48ddb93f0b30c475423fe177832912c5bcdce3cc72872f8051627967ef278e08', ''], + ['da2d431121cd10578fd81f8f80344b06db59ea2d05a7b5d27536c8789ddae8f0', ''], + ['b100c2ec0718fe6b4805b623aeec6710719d042ceea55f5c8135b010ec1c7b36', ''], + ['0c1d1b0ba547a742013366d6fbc8f71dd77f566d94e41ed9f828a74b96928161', ''], ], }, },