chore: added credentials to javascript transformations

src/controllers/userTransform.ts

@@ -33,11 +33,12 @@ export class UserTransformController {
       '(User transform - router:/transformation/test ):: Request to transformer',
       ctx.request.body,
     );
-    const { events, trRevCode, libraryVersionIDs = [] } = ctx.request.body as any;
+    const { events, trRevCode, libraryVersionIDs = [], credentials = [] } = ctx.request.body as any;
     const response = await UserTransformService.testTransformRoutine(
       events,
       trRevCode,
       libraryVersionIDs,
+      credentials,
     );
     ctx.body = response.body;
     ControllerUtility.postProcess(ctx, response.status);

src/legacy/router.js

@@ -678,6 +678,7 @@ if (transformerTestModeEnabled) {
         trRevCode.versionId,
         libraryVersionIDs,
         trRevCode,
+        null,
         true,
       );
       logger.debug(`[CT] Test Output Events: ${JSON.stringify(res.transformedEvents)}`);

src/services/userTransform.ts

@@ -203,7 +203,7 @@
     } as UserTransformationServiceResponse;
   }
 
-  public static async testTransformRoutine(events, trRevCode, libraryVersionIDs) {
+  public static async testTransformRoutine(events, trRevCode, libraryVersionIDs, credentials) {
     const response: FixMe = {};
     try {
       if (!trRevCode || !trRevCode.code || !trRevCode.codeVersion) {
@@ -221,6 +221,7 @@
         trRevCode.versionId,
         libraryVersionIDs,
         trRevCode,
+        credentials,
         true,
       );
       logger.debug(`[CT] Test Output Events: ${JSON.stringify(response.body.transformedEvents)}`);

src/types/index.ts

@@ -139,12 +139,20 @@ type UserTransformationLibrary = {
   VersionID: string;
 };
 
+type Credential = {
+  id: string;
+  key: string;
+  value: string;
+  isSecret: boolean;
+};
+
 type ProcessorTransformationRequest = {
   request?: object;
   message: object;
   metadata: Metadata;
   destination: Destination;
   libraries?: UserTransformationLibrary[];
+  credentials?: Credential[];
 };
 
 type RouterTransformationRequestData = {

src/util/customTransformer-v1.js

@@ -51,6 +51,7 @@ function calculateMsFromIvmTime(value) {
 async function userTransformHandlerV1(
   events,
   userTransformation,
+  credentials,
   libraryVersionIds,
   testMode = false,
 ) {
@@ -62,6 +63,9 @@ async function userTransformHandlerV1(
     userTransformation.code,
     libraryVersionIds,
     userTransformation.versionId,
+    userTransformation.id,
+    userTransformation.workspaceId,
+    credentials,
     userTransformation.secrets || {},
     testMode,
   );

src/util/customTransformer.js

@@ -8,7 +8,7 @@
 const stats = require('./stats');
 const { fetchWithDnsWrapper } = require('./utils');
 const { getMetadata, getTransformationMetadata } = require('../v0/util');
-
+const { isNil } = require('lodash');
 const ISOLATE_VM_MEMORY = parseInt(process.env.ISOLATE_VM_MEMORY || '128', 10);
 const GEOLOCATION_TIMEOUT_IN_MS = parseInt(process.env.GEOLOCATION_TIMEOUT_IN_MS || '1000', 10);
 
@@ -274,6 +274,7 @@
   versionId,
   libraryVersionIDs,
   trRevCode = {},
+  credentials = [],
   testMode = false,
 ) {
   if (versionId) {
@@ -286,12 +287,27 @@
       events.forEach((ev) => {
         eventsMetadata[ev.message.messageId] = ev.metadata;
       });
-
+      const credentialsMap = {};
+      if (testMode === false) {
+        (events[0]?.credentials || []).forEach((cred) => {
+          credentialsMap[cred.key] = cred.value;
+        });
+      } else {
+        (credentials || []).forEach((cred) => {
+          credentialsMap[cred.key] = cred.value;
+        });
+        events.forEach((ev) => {
+          if (isNil(ev.credentials)) {
+            ev.credentials = credentials;
+          }
+        });
+      }
       let userTransformedEvents = [];
       let result;
       if (res.codeVersion && res.codeVersion === '1') {
         result = await UserTransformHandlerFactory(res).runUserTransfrom(
           events,
+          credentialsMap,
           testMode,
           libraryVersionIDs,
         );

src/util/customTransformerFactory.js

@@ -13,14 +13,20 @@ const UserTransformHandlerFactory = (userTransformation) => {
       }
     },
 
-    runUserTransfrom: async (events, testMode, libraryVersionIds) => {
+    runUserTransfrom: async (events, credentials, testMode, libraryVersionIds) => {
       switch (userTransformation.language) {
         case 'pythonfaas':
         case 'python':
           return runOpenFaasUserTransform(events, userTransformation, libraryVersionIds, testMode);
 
         default:
-          return userTransformHandlerV1(events, userTransformation, libraryVersionIds, testMode);
+          return userTransformHandlerV1(
+            events,
+            userTransformation,
+            credentials,
+            libraryVersionIds,
+            testMode,
+          );
       }
     },
   };

src/util/ivmFactory.js

@@ -1,6 +1,6 @@
 const ivm = require('isolated-vm');
 const fetch = require('node-fetch');
-const _ = require('lodash');
+const { isNil, isObject, camelCase } = require('lodash');
 
 const { getLibraryCodeV1, getRudderLibByImportName } = require('./customTransforrmationsStore-v1');
 const { extractStackTraceUptoLastSubstringMatch } = require('./utils');
@@ -30,7 +30,16 @@
   return module;
 }
 
-async function createIvm(code, libraryVersionIds, versionId, secrets, testMode) {
+async function createIvm(
+  code,
+  libraryVersionIds,
+  versionId,
+  transformationId,
+  workspaceId,
+  credentials,
+  secrets,
+  testMode,
+) {
   const createIvmStartTime = new Date();
   const logs = [];
   const libraries = await Promise.all(
@@ -51,7 +60,7 @@
 
     // TODO: Check if this should this be &&
     libraries.forEach((library) => {
-      const libHandleName = _.camelCase(library.name);
+      const libHandleName = camelCase(library.name);
       if (extractedLibraries.includes(libHandleName)) {
         librariesMap[libHandleName] = library.code;
       }
@@ -243,6 +252,15 @@
     }),
   );
 
+  await jail.set('_credential', function (key) {
+    if (isNil(credentials) || !isObject(credentials)) {
+      logger.error('Error fetching credentials map', versionId);
+      stats.increment('credential_errors', { transformationId, workspaceId });
+      return undefined;
+    }
+    return credentials[key];
+  });
+
   await jail.set('_rsSecrets', function (...args) {
     if (args.length == 0 || !secrets || !secrets[args[0]]) return 'ERROR';
     return secrets[args[0]];
@@ -321,6 +339,16 @@
         ]);
       };
 
+      let credential = _credential;
+      delete _credential;
+      global.credential = function(...args) {
+        const key = args[0];
+        if (key === null || key === undefined) {
+          throw new Error('Key should be valid and defined'+ JSON.stringify(args));
+        }
+        return credential(new ivm.ExternalCopy(key).copyInto());
+      };
+
       return new ivm.Reference(function forwardMainPromise(
         fnRef,
         resolve,
@@ -411,10 +439,28 @@
   return evaluateModule(isolate, context, code);
 }
 
-async function getFactory(code, libraryVersionIds, versionId, secrets, testMode) {
+async function getFactory(
+  code,
+  libraryVersionIds,
+  transformationId,
+  workspaceId,
+  versionId,
+  credentials,
+  secrets,
+  testMode,
+) {
   const factory = {
     create: async () => {
-      return createIvm(code, libraryVersionIds, versionId, secrets, testMode);
+      return createIvm(
+        code,
+        libraryVersionIds,
+        versionId,
+        transformationId,
+        workspaceId,
+        credentials,
+        secrets,
+        testMode,
+      );
     },
     destroy: async (client) => {
       client.fnRef.release();

src/util/prometheus.js

@@ -569,6 +569,12 @@ class Prometheus {
           'module',
         ],
       },
+      {
+        name: 'credential_errors',
+        help: 'Error in fetching credentials count',
+        type: 'counter',
+        labelNames: ['transformationId', 'workspaceId'],
+      },
 
       // Gauges
       {

test/__tests__/data/user_transformation_input_credentials.json

@@ -0,0 +1,67 @@
+[
+  {
+    "message": {
+      "channel": "web",
+      "context": {
+        "app": {
+          "build": "1.0.0",
+          "name": "RudderLabs JavaScript SDK",
+          "namespace": "com.rudderlabs.javascript",
+          "version": "1.0.0"
+        },
+        "traits": {
+          "email": "[email protected]",
+          "anonymousId": "12345"
+        },
+        "library": {
+          "name": "RudderLabs JavaScript SDK",
+          "version": "1.0.0"
+        },
+        "userAgent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.90 Safari/537.36",
+        "locale": "en-US",
+        "ip": "0.0.0.0",
+        "os": {
+          "name": "",
+          "version": ""
+        },
+        "screen": {
+          "density": 2
+        }
+      },
+      "type": "track",
+      "messageId": "ec5481b6-a926-4d2e-b293-0b3a77c4d3be",
+      "originalTimestamp": "2019-10-14T11:15:18.300Z",
+      "anonymousId": "00000000000000000000000000",
+      "userId": "12345",
+      "event": "test track event GA3",
+      "properties": {
+        "user_actual_role": "system_admin, system_user",
+        "user_actual_id": 12345
+      },
+      "integrations": {
+        "All": true
+      },
+      "sentAt": "2019-10-14T11:15:53.296Z"
+    },
+    "destination": {
+      "Config": {
+        "trackingID": "UA-149602794-1"
+      },
+      "Enabled": true
+    },
+    "credentials": [
+      {
+        "id": "id1",
+        "key": "key1",
+        "value": "value1",
+        "isSecret": false
+      },
+      {
+        "id": "id2",
+        "key": "key2",
+        "value": "value2",
+        "isSecret": true
+      }
+    ]
+  }
+]

test/__tests__/user_transformation.integration.test.js

@@ -159,13 +159,14 @@ describe("Function invocation & creation tests", () => {
       versionId,
       [],
       trRevCode,
+      [],
       true
     );
     expect(response).toEqual(outputEvents);
 
     // Test with language python; should return same output
     trRevCode = contructTrRevCode(versionId, 'python');
-    response = await userTransformHandler(inputEvents, versionId, [], trRevCode, true);
+    response = await userTransformHandler(inputEvents, versionId, [], trRevCode, [], true);
     expect(response).toEqual(outputEvents);
   });