chore(release): pull release/v1.85.0 into main

.github/workflows/verify.yml

@@ -1,4 +1,4 @@
 name: Verify
 
 on:
   pull_request:
@@ -33,9 +33,19 @@
         with:
           token: ${{ secrets.GITHUB_TOKEN }}
 
+      - name: Filter JS/TS Files
+        run: |
+          echo "${{ steps.files.outputs.added_modified }}" | tr ' ' '\n' | grep -E '\.(js|ts|jsx|tsx)$' > changed_files.txt
+          if [ ! -s changed_files.txt ]; then
+            echo "No JS/TS files to format or lint."
+            exit 0
+          fi
+
       - name: Run format Checks
         run: |
-          npx prettier ${{steps.files.outputs.added_modified}} --write
+          if [ -s changed_files.txt ]; then
+              npx prettier --write $(cat changed_files.txt)
+          fi
 
       - run: git diff --exit-code
 

.gitignore

@@ -1,4 +1,4 @@
 # Logs
 logs
 *.log
 npm-debug.log*
@@ -122,6 +122,7 @@
 
 # Stores VSCode versions used for testing VSCode extensions
 .vscode-test
+.vscode
 
 # yarn v2
 .yarn/cache
@@ -133,9 +134,9 @@
 # Others
 **/.DS_Store
 .dccache
-
+.python-version
 .idea
 
 # component test report
 test_reports/
-temp/
+temp/

CHANGELOG.md

@@ -2,6 +2,30 @@
 
 All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines.
 
+## [1.85.0](https://github.com/rudderlabs/rudder-transformer/compare/v1.84.0...v1.85.0) (2024-11-18)
+
+
+### Features
+
+* added support to eu/us2 datacenter for gainsight px destination ([#3871](https://github.com/rudderlabs/rudder-transformer/issues/3871)) ([12ac3de](https://github.com/rudderlabs/rudder-transformer/commit/12ac3de6e7cc91a6cd52c33bc342f74bbaa8a631))
+* iterable EUDC ([#3828](https://github.com/rudderlabs/rudder-transformer/issues/3828)) ([1c134f8](https://github.com/rudderlabs/rudder-transformer/commit/1c134f84601aaea78581078137cb9955de576f9e))
+* iterable EUDC deleteUsers ([#3881](https://github.com/rudderlabs/rudder-transformer/issues/3881)) ([becb4fa](https://github.com/rudderlabs/rudder-transformer/commit/becb4fa54e9093ed69779f54c36864cb9d28d321))
+* moved userSchema to connection config in GARL vdmv2 ([#3870](https://github.com/rudderlabs/rudder-transformer/issues/3870)) ([640a11e](https://github.com/rudderlabs/rudder-transformer/commit/640a11eb3dca5735fed3ad9ad5bd058974b069d6))
+* now getting consent related fields from connection config from retl for GARL ([#3877](https://github.com/rudderlabs/rudder-transformer/issues/3877)) ([51bbc02](https://github.com/rudderlabs/rudder-transformer/commit/51bbc02d5b00ce1b8fe8c91b4a7041e926bae9bd))
+* onboard linkedin audience destination ([#3857](https://github.com/rudderlabs/rudder-transformer/issues/3857)) ([f3ff409](https://github.com/rudderlabs/rudder-transformer/commit/f3ff4092d455508dd3354ffb22d345fa97f4d1f2))
+* onboarding intercom v2 retl support ([#3843](https://github.com/rudderlabs/rudder-transformer/issues/3843)) ([3d7db73](https://github.com/rudderlabs/rudder-transformer/commit/3d7db7366e30df31c37cc473e344da82b49ed885))
+* sources v2 spec support along with adapters ([04c0694](https://github.com/rudderlabs/rudder-transformer/commit/04c069486bdd3c101906fa6c621e983090fcab25))
+* sources v2 spec support along with adapters ([#3810](https://github.com/rudderlabs/rudder-transformer/issues/3810)) ([c51cfbb](https://github.com/rudderlabs/rudder-transformer/commit/c51cfbb4664a8531dce23b2d06fe40997f95697e))
+* update pinterest_tag single product events with new mapping ([#3858](https://github.com/rudderlabs/rudder-transformer/issues/3858)) ([8520278](https://github.com/rudderlabs/rudder-transformer/commit/85202781de3464bd46fe910159d2b143cd4209e8))
+
+
+### Bug Fixes
+
+* adding logger for undefined source event ([#3879](https://github.com/rudderlabs/rudder-transformer/issues/3879)) ([79e5979](https://github.com/rudderlabs/rudder-transformer/commit/79e597907eee126b4187e4534b2aa2253d1431da))
+* adding uuid transformation for airship ([#3884](https://github.com/rudderlabs/rudder-transformer/issues/3884)) ([a80f874](https://github.com/rudderlabs/rudder-transformer/commit/a80f87486dc93b423e4fe6efbee6f4cb8330ba02))
+* handling invalid timestamp for adjust source ([#3866](https://github.com/rudderlabs/rudder-transformer/issues/3866)) ([d57f48e](https://github.com/rudderlabs/rudder-transformer/commit/d57f48e989d18d469bea0de94293bc685300945b))
+* revert gaec changes ([#3885](https://github.com/rudderlabs/rudder-transformer/issues/3885)) ([0aeaa39](https://github.com/rudderlabs/rudder-transformer/commit/0aeaa391b025fc68de6e3d63a6721f067c5be318))
+
 ## [1.84.0](https://github.com/rudderlabs/rudder-transformer/compare/v1.83.2...v1.84.0) (2024-11-11)
 
 

package.json

@@ -1,6 +1,6 @@
 {
   "name": "rudder-transformer",
-  "version": "1.84.0",
+  "version": "1.85.0",
   "description": "",
   "homepage": "https://github.com/rudderlabs/rudder-transformer#readme",
   "bugs": {
@@ -64,7 +64,7 @@
     "@koa/router": "^12.0.0",
     "@ndhoule/extend": "^2.0.0",
     "@pyroscope/nodejs": "^0.2.9",
-    "@rudderstack/integrations-lib": "^0.2.12",
+    "@rudderstack/integrations-lib": "^0.2.13",
     "@rudderstack/json-template-engine": "^0.18.0",
     "@rudderstack/workflow-engine": "^0.8.13",
     "@shopify/jest-koa-mocks": "^5.1.1",
@@ -118,7 +118,7 @@
     "truncate-utf8-bytes": "^1.0.2",
     "ua-parser-js": "^1.0.37",
     "unset-value": "^2.0.1",
-    "uuid": "^9.0.0",
+    "uuid": "^9.0.1",
     "valid-url": "^1.0.9",
     "zod": "^3.22.4"
   },

src/cdk/v2/destinations/linkedin_audience/config.ts

@@ -0,0 +1,10 @@
+export const SUPPORTED_EVENT_TYPE = 'record';
+export const ACTION_TYPES = ['insert', 'delete'];
+export const BASE_ENDPOINT = 'https://api.linkedin.com/rest';
+export const USER_ENDPOINT = '/dmpSegments/audienceId/users';
+export const COMPANY_ENDPOINT = '/dmpSegments/audienceId/companies';
+export const FIELD_MAP = {
+  sha256Email: 'SHA256_EMAIL',
+  sha512Email: 'SHA512_EMAIL',
+  googleAid: 'GOOGLE_AID',
+};

src/cdk/v2/destinations/linkedin_audience/procWorkflow.yaml

@@ -0,0 +1,89 @@
+bindings:
+  - path: ./config
+    exportAll: true
+  - path: ./utils
+    exportAll: true
+  - name: defaultRequestConfig
+    path: ../../../../v0/util
+
+steps:
+  - name: validateInput
+    description: Validate input, if all the required fields are available or not.
+    template: |
+      const config = .connection.config.destination;
+      const secret = .metadata.secret;
+      let messageType = .message.type;
+      $.assertConfig(config.audienceId, "Audience Id is not present. Aborting");
+      $.assertConfig(secret.accessToken, "Access Token is not present. Aborting");
+      $.assertConfig(config.audienceType, "audienceType is not present. Aborting");
+      $.assert(messageType, "Message Type is not present. Aborting message.");
+      $.assert(messageType.toLowerCase() === $.SUPPORTED_EVENT_TYPE, `Event type ${.message.type.toLowerCase()} is not supported. Aborting message.`);
+      $.assert(.message.fields, "`fields` is not present. Aborting message.");
+      $.assert(.message.identifiers, "`identifiers` is not present inside properties. Aborting message.");
+      $.assert($.containsAll([.message.action], $.ACTION_TYPES), "Unsupported action type. Aborting message.")
+
+  - name: getConfigs
+    description: This step fetches the configs from different places and combines them.
+    template: |
+      const config = .connection.config.destination;
+      {
+        audienceType: config.audienceType,
+        audienceId: config.audienceId,
+        accessToken: .metadata.secret.accessToken,
+        isHashRequired: config.isHashRequired,
+      }
+
+  - name: prepareUserTypeBasePayload
+    condition: $.outputs.getConfigs.audienceType === 'user'
+    steps:
+      - name: prepareUserIds
+        description: Prepare user ids for user audience type
+        template: |
+          const identifiers = $.outputs.getConfigs.isHashRequired === true ? 
+            $.hashIdentifiers(.message.identifiers) :
+            .message.identifiers;
+          $.prepareUserIds(identifiers)
+
+      - name: preparePayload
+        description: Prepare base payload for user audiences
+        template: |
+          const payload = {
+            'elements': [
+              {
+                'action': $.generateActionType(.message.action),
+                'userIds': $.outputs.prepareUserTypeBasePayload.prepareUserIds,
+                ....message.fields
+              }
+            ]
+          }
+          payload;
+
+  - name: prepareCompanyTypeBasePayload
+    description: Prepare base payload for company audiences
+    condition: $.outputs.getConfigs.audienceType === 'company'
+    template: |
+      const payload = {
+        'elements': [
+          {
+            'action': $.generateActionType(.message.action),
+            ....message.identifiers,
+            ....message.fields
+          }
+        ]
+      }
+      payload;
+
+  - name: buildResponseForProcessTransformation
+    description: build response depending upon batch size
+    template: |
+      const response = $.defaultRequestConfig();
+      response.body.JSON = {...$.outputs.prepareUserTypeBasePayload, ...$.outputs.prepareCompanyTypeBasePayload};
+      response.endpoint = $.generateEndpoint($.outputs.getConfigs.audienceType, $.outputs.getConfigs.audienceId);
+      response.headers = {
+          "Authorization": "Bearer " + $.outputs.getConfigs.accessToken,
+          "Content-Type": "application/json",
+          "X-RestLi-Method": "BATCH_CREATE",
+          "X-Restli-Protocol-Version": "2.0.0",
+          "LinkedIn-Version": "202409"
+        };
+      response;

src/cdk/v2/destinations/linkedin_audience/rtWorkflow.yaml

@@ -0,0 +1,40 @@
+bindings:
+  - path: ./utils
+  - name: handleRtTfSingleEventError
+    path: ../../../../v0/util/index
+
+steps:
+  - name: validateInput
+    template: |
+      $.assert(Array.isArray(^) && ^.length > 0, "Invalid event array")
+
+  - name: transform
+    externalWorkflow:
+      path: ./procWorkflow.yaml
+      bindings:
+        - name: batchMode
+          value: true
+    loopOverInput: true
+
+  - name: successfulEvents
+    template: |
+      $.outputs.transform#idx.output.({
+        "message": .[],
+        "destination": ^ [idx].destination,
+        "metadata": ^ [idx].metadata
+      })[]
+
+  - name: failedEvents
+    template: |
+      $.outputs.transform#idx.error.(
+        $.handleRtTfSingleEventError(^[idx], .originalError ?? ., {})
+      )[]
+
+  - name: batchSuccessfulEvents
+    description: Batches the successfulEvents
+    template: |
+      $.batchResponseBuilder($.outputs.successfulEvents);
+
+  - name: finalPayload
+    template: |
+      [...$.outputs.batchSuccessfulEvents, ...$.outputs.failedEvents]

src/cdk/v2/destinations/linkedin_audience/utils.ts

@@ -0,0 +1,87 @@
+import lodash from 'lodash';
+import { hashToSha256 } from '@rudderstack/integrations-lib';
+import { createHash } from 'crypto';
+import { BASE_ENDPOINT, COMPANY_ENDPOINT, FIELD_MAP, USER_ENDPOINT } from './config';
+
+export function hashIdentifiers(identifiers: string[]): Record<string, string> {
+  const hashedIdentifiers = {};
+  Object.keys(identifiers).forEach((key) => {
+    if (key === 'sha256Email') {
+      hashedIdentifiers[key] = hashToSha256(identifiers[key]);
+    } else if (key === 'sha512Email') {
+      hashedIdentifiers[key] = createHash('sha512').update(identifiers[key]).digest('hex');
+    } else {
+      hashedIdentifiers[key] = identifiers[key];
+    }
+  });
+  return hashedIdentifiers;
+}
+
+export function prepareUserIds(
+  identifiers: Record<string, string>,
+): { idType: string; idValue: string }[] {
+  const userIds: { idType: string; idValue: string }[] = [];
+  Object.keys(identifiers).forEach((key) => {
+    userIds.push({ idType: FIELD_MAP[key], idValue: identifiers[key] });
+  });
+  return userIds;
+}
+
+export function generateEndpoint(audienceType: string, audienceId: string) {
+  if (audienceType === 'user') {
+    return BASE_ENDPOINT + USER_ENDPOINT.replace('audienceId', audienceId);
+  }
+  return BASE_ENDPOINT + COMPANY_ENDPOINT.replace('audienceId', audienceId);
+}
+
+export function batchResponseBuilder(successfulEvents) {
+  const chunkOnActionType = lodash.groupBy(
+    successfulEvents,
+    (event) => event.message[0].body.JSON.elements[0].action,
+  );
+  const result: any = [];
+  Object.keys(chunkOnActionType).forEach((actionType) => {
+    const firstEvent = chunkOnActionType[actionType][0];
+    const { method, endpoint, headers, type, version } = firstEvent.message[0];
+    const batchEvent = {
+      batchedRequest: {
+        body: {
+          JSON: { elements: firstEvent.message[0].body.JSON.elements },
+          JSON_ARRAY: {},
+          XML: {},
+          FORM: {},
+        },
+        version,
+        type,
+        method,
+        endpoint,
+        headers,
+        params: {},
+        files: {},
+      },
+      metadata: [firstEvent.metadata],
+      batched: true,
+      statusCode: 200,
+      destination: firstEvent.destination,
+    };
+    firstEvent.metadata = [firstEvent.metadata];
+    chunkOnActionType[actionType].forEach((element, index) => {
+      if (index !== 0) {
+        batchEvent.batchedRequest.body.JSON.elements.push(element.message[0].body.JSON.elements[0]);
+        batchEvent.metadata.push(element.metadata);
+      }
+    });
+    result.push(batchEvent);
+  });
+  return result;
+}
+
+export const generateActionType = (actionType: string): string => {
+  if (actionType === 'insert') {
+    return 'ADD';
+  }
+  if (actionType === 'delete') {
+    return 'REMOVE';
+  }
+  return actionType;
+};