Merge branch 'enhance.reddit' of github.com:rudderlabs/rudder-transfo…

…rmer into enhance.reddit
rudderlabs · Apr 8, 2024 · 7a12986 · 7a12986
2 parents 71f85c3 + e2346e4
commit 7a12986
Show file tree

Hide file tree

Showing 45 changed files with 221 additions and 67 deletions.
diff --git a/.github/workflows/publish-new-release.yml b/.github/workflows/publish-new-release.yml
@@ -99,7 +99,7 @@ jobs:
           channel-id: ${{ secrets.SLACK_RELEASE_CHANNEL_ID }}
           payload: |
             {
-              "text": "*<${{env.RELEASES_URL}}v${{ steps.extract-version.outputs.release_version }}|v${{ steps.extract-version.outputs.release_version }}>*\nCC: <@U03KG4BK1L1> <@U02AE5GMMHV> <@U01LVJ30QEB>",
+              "text": "*<${{env.RELEASES_URL}}v${{ steps.extract-version.outputs.release_version }}|v${{ steps.extract-version.outputs.release_version }}>*\nCC: <@U03KG4BK1L1> <@U024YF8CR53> <@U01LVJ30QEB>",
               "blocks": [
                 {
                   "type": "header",
@@ -115,7 +115,7 @@ jobs:
                   "type": "section",
                   "text": {
                     "type": "mrkdwn",
-                    "text": "*<${{env.RELEASES_URL}}v${{ steps.extract-version.outputs.release_version }}|v${{ steps.extract-version.outputs.release_version }}>*\nCC: <@U03KG4BK1L1> <@U02AE5GMMHV> <@U01LVJ30QEB>"
+                    "text": "*<${{env.RELEASES_URL}}v${{ steps.extract-version.outputs.release_version }}|v${{ steps.extract-version.outputs.release_version }}>*\nCC: <@U03KG4BK1L1> <@U024YF8CR53> <@U01LVJ30QEB>"
                   }
                 }
               ]

diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -2,6 +2,27 @@
 
 All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines.
 
+### [1.61.1](https://github.com/rudderlabs/rudder-transformer/compare/v1.61.0...v1.61.1) (2024-04-03)
+
+## [1.61.0](https://github.com/rudderlabs/rudder-transformer/compare/v1.60.0...v1.61.0) (2024-04-02)
+
+
+### Features
+
+* consent field support for ga4 ([#3213](https://github.com/rudderlabs/rudder-transformer/issues/3213)) ([92515a5](https://github.com/rudderlabs/rudder-transformer/commit/92515a5fd8a2798c48010078f62b360ec6a49979))
+* consent field support for gaoc and upgrade the api version from v14 to v16 ([#3121](https://github.com/rudderlabs/rudder-transformer/issues/3121)) ([2aac2a6](https://github.com/rudderlabs/rudder-transformer/commit/2aac2a62547b7a7c617735fc3d6e88e0a1bed76e)), closes [#3190](https://github.com/rudderlabs/rudder-transformer/issues/3190)
+* onboard new destination bloomreach ([#3185](https://github.com/rudderlabs/rudder-transformer/issues/3185)) ([d9b7e1f](https://github.com/rudderlabs/rudder-transformer/commit/d9b7e1f70565d59979aee3e62f60e39edb9a23c7))
+* onboarding linkedin conversion api ([#3194](https://github.com/rudderlabs/rudder-transformer/issues/3194)) ([eb7b197](https://github.com/rudderlabs/rudder-transformer/commit/eb7b197322c617b14c2579de8cb4d4dacf8e1df3))
+* update movable ink batch size ([#3223](https://github.com/rudderlabs/rudder-transformer/issues/3223)) ([667095f](https://github.com/rudderlabs/rudder-transformer/commit/667095fa8316cd95a066f15b848ad503c6b4af80))
+
+
+### Bug Fixes
+
+* fixed userId mapping, now mapping to uid instead of id ([#3192](https://github.com/rudderlabs/rudder-transformer/issues/3192)) ([70a468b](https://github.com/rudderlabs/rudder-transformer/commit/70a468bf16ecd5ee0b6fecee4b837895d19c525f))
+* ninetailed: remove  page support ([#3218](https://github.com/rudderlabs/rudder-transformer/issues/3218)) ([2f30c56](https://github.com/rudderlabs/rudder-transformer/commit/2f30c56af62e983d09b5d4f2da9a0ba22f5c1612))
+* shopify invalid_event metric prometheus label ([#3200](https://github.com/rudderlabs/rudder-transformer/issues/3200)) ([345c87d](https://github.com/rudderlabs/rudder-transformer/commit/345c87d7c530c621ae3fd6c504d64e5a14e31f22))
+* fix: snapchat conversion: add event level_complete ([#3231](https://github.com/rudderlabs/rudder-transformer/issues/3231)) ([39368a0](https://github.com/rudderlabs/rudder-transformer/commit/39368a09e48acc324faa855186bc623e5c347881))
+
 ## [1.60.0](https://github.com/rudderlabs/rudder-transformer/compare/v1.57.1...v1.60.0) (2024-03-20)
 
 

diff --git a/package-lock.json b/package-lock.json
diff --git a/package.json b/package.json
@@ -1,6 +1,6 @@
 {
   "name": "rudder-transformer",
-  "version": "1.60.0",
+  "version": "1.61.1",
   "description": "",
   "homepage": "https://github.com/rudderlabs/rudder-transformer#readme",
   "bugs": {
@@ -63,7 +63,7 @@
     "@datadog/pprof": "^3.1.0",
     "@koa/router": "^12.0.0",
     "@ndhoule/extend": "^2.0.0",
-    "@pyroscope/nodejs": "^0.2.6",
+    "@pyroscope/nodejs": "^0.2.9",
     "@rudderstack/integrations-lib": "^0.2.7",
     "@rudderstack/workflow-engine": "^0.7.5",
     "@shopify/jest-koa-mocks": "^5.1.1",

diff --git a/src/cdk/v2/destinations/bluecore/data/bluecoreCommonConfig.json b/src/cdk/v2/destinations/bluecore/data/bluecoreCommonConfig.json
@@ -35,7 +35,7 @@
   },
   {
     "destKey": "properties.customer.email",
-    "sourceKeys": "email",
+    "sourceKeys": "emailOnly",
     "required": false,
     "sourceFromGenericMap": true
   },

diff --git a/src/cdk/v2/destinations/fullstory/procWorkflow.yaml b/src/cdk/v2/destinations/fullstory/procWorkflow.yaml
@@ -76,7 +76,7 @@ steps:
         "use_most_recent": .message.properties.useMostRecent,
       };
       $.context.payload.user = {
-        "uid": .message.properties.userId ?? .message.userId,
+        "id": .message.properties.userId ?? .message.userId,
       }
 
   - name: cleanPayload

diff --git a/src/cdk/v2/destinations/linkedin_ads/rtWorkflow.yaml b/src/cdk/v2/destinations/linkedin_ads/rtWorkflow.yaml
@@ -1,6 +1,8 @@
 bindings:
   - path: ./utils
   - path: ./config
+  - name: handleRtTfSingleEventError
+    path: ../../../../v0/util/index
 
 steps:
   - name: validateInput
@@ -23,14 +25,9 @@ steps:
       })[]
   - name: failedEvents
     template: |
-      $.outputs.transform#idx.error.({
-        "metadata": ^[idx].metadata[],
-        "destination": ^[idx].destination,
-        "batched": false,
-        "statusCode": .status,
-        "error": .message,
-        "statTags": .originalError.statTags
-      })[]
+      $.outputs.transform#idx.error.(
+        $.handleRtTfSingleEventError(^[idx], .originalError ?? ., {})
+      )[]
 
   - name: batchSuccessfulEvents
     description: Batches the successfulEvents

diff --git a/src/cdk/v2/destinations/linkedin_ads/utils.js b/src/cdk/v2/destinations/linkedin_ads/utils.js
@@ -162,6 +162,9 @@ const fetchAndVerifyConversionHappenedAt = (message) => {
 };
 
 function batchResponseBuilder(successfulEvents) {
+  if (successfulEvents.length === 0) {
+    return [];
+  }
   const constants = {
     version: successfulEvents[0].message[0].version,
     type: successfulEvents[0].message[0].type,

diff --git a/src/util/openfaas/index.js b/src/util/openfaas/index.js
@@ -27,6 +27,7 @@ const FAAS_AST_VID = 'ast';
 const FAAS_AST_FN_NAME = 'fn-ast';
 const CUSTOM_NETWORK_POLICY_WORKSPACE_IDS = process.env.CUSTOM_NETWORK_POLICY_WORKSPACE_IDS || '';
 const customNetworkPolicyWorkspaceIds = CUSTOM_NETWORK_POLICY_WORKSPACE_IDS.split(',');
+const CUSTOMER_TIER = process.env.CUSTOMER_TIER || 'shared';
 
 // Initialise node cache
 const functionListCache = new NodeCache();
@@ -151,6 +152,10 @@ const deployFaasFunction = async (
       'com.openfaas.scale.min': FAAS_MIN_PODS_IN_TEXT,
       transformationId: trMetadata.transformationId,
       workspaceId: trMetadata.workspaceId,
+      team: 'data-management',
+      service: 'openfaas-fn',
+      customer: 'shared',
+      'customer-tier': CUSTOMER_TIER,
     };
     if (
       trMetadata.workspaceId &&

diff --git a/src/util/redis/testData/shopify_source.json b/src/util/redis/testData/shopify_source.json
@@ -65,11 +65,8 @@
       }
     },
     "output": {
-      "outputToSource": {
-        "body": "T0s=",
-        "contentType": "text/plain"
-      },
-      "statusCode": 200
+      "error": "Error: Error setting value in Redis due Error: Connection is Closed",
+      "statusCode": 500
     }
   },
   {

diff --git a/src/v0/destinations/active_campaign/data/ACIdentify.json b/src/v0/destinations/active_campaign/data/ACIdentify.json
@@ -1,7 +1,7 @@
 [
   {
     "destKey": "email",
-    "sourceKeys": "email",
+    "sourceKeys": "emailOnly",
     "sourceFromGenericMap": true,
     "required": true
   },

diff --git a/src/v0/destinations/blueshift/data/blueshiftGroupConfig.json b/src/v0/destinations/blueshift/data/blueshiftGroupConfig.json
@@ -13,7 +13,7 @@
   },
   {
     "destKey": "email",
-    "sourceKeys": "email",
+    "sourceKeys": "emailOnly",
     "sourceFromGenericMap": true,
     "required": false
   },

diff --git a/src/v0/destinations/blueshift/data/blueshiftIdentifyConfig.json b/src/v0/destinations/blueshift/data/blueshiftIdentifyConfig.json
@@ -1,7 +1,7 @@
 [
   {
     "destKey": "email",
-    "sourceKeys": "email",
+    "sourceKeys": "emailOnly",
     "sourceFromGenericMap": true,
     "required": true
   },

diff --git a/src/v0/destinations/blueshift/data/blueshiftTrackConfig.json b/src/v0/destinations/blueshift/data/blueshiftTrackConfig.json
@@ -7,7 +7,7 @@
   },
   {
     "destKey": "email",
-    "sourceKeys": "email",
+    "sourceKeys": "emailOnly",
     "sourceFromGenericMap": true,
     "required": false
   },

diff --git a/src/v0/destinations/custify/data/CUSTIFYIdentifyConfig.json b/src/v0/destinations/custify/data/CUSTIFYIdentifyConfig.json
@@ -7,7 +7,7 @@
   },
   {
     "destKey": "email",
-    "sourceKeys": "email",
+    "sourceKeys": "emailOnly",
     "sourceFromGenericMap": true,
     "required": false
   },

diff --git a/src/v0/destinations/custify/data/CUSTIFYTrackConfig.json b/src/v0/destinations/custify/data/CUSTIFYTrackConfig.json
@@ -7,7 +7,7 @@
   },
   {
     "destKey": "email",
-    "sourceKeys": "email",
+    "sourceKeys": "emailOnly",
     "sourceFromGenericMap": true,
     "required": false
   },

diff --git a/src/v0/destinations/facebook_offline_conversions/data/FbOfflineConversionsTrackConfig.json b/src/v0/destinations/facebook_offline_conversions/data/FbOfflineConversionsTrackConfig.json
@@ -39,7 +39,7 @@
   },
   {
     "destKey": "email",
-    "sourceKeys": "email",
+    "sourceKeys": "emailOnly",
     "required": false,
     "sourceFromGenericMap": true
   },

diff --git a/src/v0/destinations/freshmarketer/data/FRESHMARKETERIdentifyConfig.json b/src/v0/destinations/freshmarketer/data/FRESHMARKETERIdentifyConfig.json
@@ -1,7 +1,7 @@
 [
   {
     "destKey": "emails",
-    "sourceKeys": "email",
+    "sourceKeys": "emailOnly",
     "sourceFromGenericMap": true,
     "required": true
   },

diff --git a/src/v0/destinations/freshsales/data/identifyConfig.json b/src/v0/destinations/freshsales/data/identifyConfig.json
@@ -1,7 +1,7 @@
 [
   {
     "destKey": "emails",
-    "sourceKeys": "email",
+    "sourceKeys": "emailOnly",
     "sourceFromGenericMap": true,
     "required": true
   },

diff --git a/src/v0/destinations/google_adwords_enhanced_conversions/data/trackConfig.json b/src/v0/destinations/google_adwords_enhanced_conversions/data/trackConfig.json
@@ -51,7 +51,7 @@
   },
   {
     "destKey": "conversionAdjustments[0].userIdentifiers[0].hashedEmail",
-    "sourceKeys": "email",
+    "sourceKeys": "emailOnly",
     "sourceFromGenericMap": true,
     "required": false,
     "metadata": {

diff --git a/src/v0/destinations/impact/data/ImpactConversionConfig.json b/src/v0/destinations/impact/data/ImpactConversionConfig.json
@@ -120,7 +120,7 @@
   },
   {
     "destKey": "CustomerEmail",
-    "sourceKeys": "email",
+    "sourceKeys": "emailOnly",
     "sourceFromGenericMap": true,
     "required": false
   },

diff --git a/src/v0/destinations/impact/data/ImpactPageLoadConfig.json b/src/v0/destinations/impact/data/ImpactPageLoadConfig.json
@@ -7,7 +7,7 @@
   },
   {
     "destKey": "CustomerEmail",
-    "sourceKeys": "email",
+    "sourceKeys": "emailOnly",
     "sourceFromGenericMap": true,
     "required": false
   },

diff --git a/src/v0/destinations/klaviyo/data/KlaviyoGroup.json b/src/v0/destinations/klaviyo/data/KlaviyoGroup.json
@@ -1,7 +1,7 @@
 [
   {
     "destKey": "email",
-    "sourceKeys": "email",
+    "sourceKeys": "emailOnly",
     "required": false,
     "sourceFromGenericMap": true
   },

diff --git a/src/v0/destinations/klaviyo/data/KlaviyoIdentify.json b/src/v0/destinations/klaviyo/data/KlaviyoIdentify.json
@@ -7,7 +7,7 @@
   },
   {
     "destKey": "email",
-    "sourceKeys": "email",
+    "sourceKeys": "emailOnly",
     "required": false,
     "sourceFromGenericMap": true
   },

diff --git a/src/v0/destinations/klaviyo/data/KlaviyoProfile.json b/src/v0/destinations/klaviyo/data/KlaviyoProfile.json
@@ -1,7 +1,7 @@
 [
   {
     "destKey": "$email",
-    "sourceKeys": "email",
+    "sourceKeys": "emailOnly",
     "required": false,
     "sourceFromGenericMap": true
   },

diff --git a/src/v0/destinations/mailjet/data/MailJetIdentifyConfig.json b/src/v0/destinations/mailjet/data/MailJetIdentifyConfig.json
@@ -1,7 +1,7 @@
 [
   {
     "destKey": "email",
-    "sourceKeys": "email",
+    "sourceKeys": "emailOnly",
     "required": true,
     "sourceFromGenericMap": true
   },

diff --git a/src/v0/destinations/mp/data/MPIdentifyConfig.json b/src/v0/destinations/mp/data/MPIdentifyConfig.json
@@ -7,7 +7,7 @@
   },
   {
     "destKey": "$email",
-    "sourceKeys": "email",
+    "sourceKeys": "emailOnly",
     "required": false,
     "sourceFromGenericMap": true
   },

diff --git a/src/v0/destinations/pinterest_tag/data/pinterestUserConfig.json b/src/v0/destinations/pinterest_tag/data/pinterestUserConfig.json
@@ -1,7 +1,7 @@
 [
   {
     "destKey": "em",
-    "sourceKeys": "email",
+    "sourceKeys": "emailOnly",
     "sourceFromGenericMap": true,
     "required": false
   },

diff --git a/src/v0/destinations/revenue_cat/data/RCIdentifyConfig.json b/src/v0/destinations/revenue_cat/data/RCIdentifyConfig.json
@@ -13,7 +13,7 @@
   },
   {
     "destKey": "$email.value",
-    "sourceKeys": "email",
+    "sourceKeys": "emailOnly",
     "sourceFromGenericMap": true,
     "required": false
   },

diff --git a/src/v0/destinations/rockerbox/data/RockerboxTrackConfig.json b/src/v0/destinations/rockerbox/data/RockerboxTrackConfig.json
@@ -12,7 +12,7 @@
     "sourceFromGenericMap": false
   },
   {
-    "sourceKeys": "email",
+    "sourceKeys": "emailOnly",
     "destKey": "email",
     "required": false,
     "sourceFromGenericMap": true

diff --git a/src/v0/destinations/sendgrid/data/SendgridIdentify.json b/src/v0/destinations/sendgrid/data/SendgridIdentify.json
@@ -1,7 +1,7 @@
 [
   {
     "destKey": "email",
-    "sourceKeys": "email",
+    "sourceKeys": "emailOnly",
     "required": true,
     "sourceFromGenericMap": true
   },

diff --git a/src/v0/destinations/serenytics/data/SerenyticsIdentifyConfig.json b/src/v0/destinations/serenytics/data/SerenyticsIdentifyConfig.json
@@ -35,7 +35,7 @@
   },
   {
     "destKey": "email",
-    "sourceKeys": "email",
+    "sourceKeys": "emailOnly",
     "sourceFromGenericMap": true,
     "required": false
   },

diff --git a/src/v0/destinations/snapchat_conversion/config.js b/src/v0/destinations/snapchat_conversion/config.js
@@ -55,6 +55,7 @@ const eventNameMapping = {
   save: 'SAVE',
   subscribe: 'SUBSCRIBE',
   complete_tutorial: 'COMPLETE_TUTORIAL',
+  level_complete: 'LEVEL_COMPLETE',
   invite: 'INVITE',
   login: 'LOGIN',
   share: 'SHARE',