diff --git a/src/util/customTransforrmationsStore-v1.js b/src/util/customTransforrmationsStore-v1.js index 40a7b006c9..894981d889 100644 --- a/src/util/customTransforrmationsStore-v1.js +++ b/src/util/customTransforrmationsStore-v1.js @@ -5,11 +5,13 @@ const { responseStatusHandler } = require('./utils'); const transformationCache = {}; const libraryCache = {}; +const rudderLibraryCache = {}; -// const CONFIG_BACKEND_URL = "http://localhost:5000"; +// const CONFIG_BACKEND_URL = 'http://localhost:5000'; const CONFIG_BACKEND_URL = process.env.CONFIG_BACKEND_URL || 'https://api.rudderlabs.com'; const getTransformationURL = `${CONFIG_BACKEND_URL}/transformation/getByVersionId`; const getLibrariesUrl = `${CONFIG_BACKEND_URL}/transformationLibrary/getByVersionId`; +const getRudderLibrariesUrl = `${CONFIG_BACKEND_URL}/rudderstackTransformationLibraries`; // Gets the transformation from config backend. // Stores the transformation object in memory with time to live after which it expires. @@ -64,4 +66,31 @@ async function getLibraryCodeV1(versionId) { } } -module.exports = { getTransformationCodeV1, getLibraryCodeV1 }; +async function getRudderLibByImportName(importName) { + const rudderLibrary = rudderLibraryCache[importName]; + if (rudderLibrary) return rudderLibrary; + const tags = { + libraryVersionId: importName, + version: 1, + type: 'rudderlibrary', + }; + try { + const [name, version] = importName.split('/').slice(-2); + const url = `${getRudderLibrariesUrl}/${name}?version=${version}`; + const startTime = new Date(); + const response = await fetchWithProxy(url); + + responseStatusHandler(response.status, 'Rudder Library', importName, url); + stats.increment('get_libraries_code.success', tags); + stats.timing('get_libraries_code', startTime, tags); + const myJson = await response.json(); + rudderLibraryCache[importName] = myJson; + return myJson; + } catch (error) { + logger.error(error); + stats.increment('get_libraries_code.error', tags); + throw error; + } +} + +module.exports = { getTransformationCodeV1, getLibraryCodeV1, getRudderLibByImportName }; diff --git a/src/util/ivmFactory.js b/src/util/ivmFactory.js index 3c90e2d761..a987806db6 100644 --- a/src/util/ivmFactory.js +++ b/src/util/ivmFactory.js @@ -3,10 +3,12 @@ const fetch = require('node-fetch'); const _ = require('lodash'); const stats = require('./stats'); -const { getLibraryCodeV1 } = require('./customTransforrmationsStore-v1'); +const { getLibraryCodeV1, getRudderLibByImportName } = require('./customTransforrmationsStore-v1'); const { parserForImport } = require('./parser'); const logger = require('../logger'); +const RUDDER_LIBRARY_REGEX = /^@rs\/[A-Za-z]+\/v[0-9]{1,3}$/; + const isolateVmMem = 128; async function evaluateModule(isolate, context, moduleCode) { const module = await isolate.compileModule(moduleCode); @@ -25,18 +27,29 @@ async function createIvm(code, libraryVersionIds, versionId, testMode) { const createIvmStartTime = new Date(); const logs = []; const libraries = await Promise.all( - libraryVersionIds.map(async (libraryVersionId) => getLibraryCodeV1(libraryVersionId)), + libraryVersionIds.map(async (libraryVersionId) => await getLibraryCodeV1(libraryVersionId)), ); const librariesMap = {}; if (code && libraries) { - const extractedLibraries = Object.keys(await parserForImport(code)); - // TODO: Check if this should this be && + const extractedLibImportNames = Object.keys(await parserForImport(code)); + libraries.forEach((library) => { const libHandleName = _.camelCase(library.name); - if (extractedLibraries.includes(libHandleName)) { + if (extractedLibImportNames.includes(libHandleName)) { librariesMap[libHandleName] = library.code; } }); + + // Extract ruddder libraries from import names + const rudderLibImportNames = extractedLibImportNames.filter((name) => + RUDDER_LIBRARY_REGEX.test(name), + ); + const rudderLibraries = await Promise.all( + rudderLibImportNames.map(async (importName) => await getRudderLibByImportName(importName)), + ); + rudderLibraries.forEach((library) => { + librariesMap[library.importName] = library.code; + }); } const codeWithWrapper = diff --git a/test/__tests__/user_transformation.test.js b/test/__tests__/user_transformation.test.js index b715ed62ea..1237891d33 100644 --- a/test/__tests__/user_transformation.test.js +++ b/test/__tests__/user_transformation.test.js @@ -1100,6 +1100,67 @@ describe("Timeout tests", () => { }); }); +describe("Rudder library tests", () => { + beforeEach(() => {}); + it(`Simple ${name} async test for V1 transformation - with rudder library urlParser `, async () => { + const versionId = randomID(); + const rudderLibraryImportName = '@rs/urlParser/v1'; + const [name, version] = rudderLibraryImportName.split('/').slice(-2); + const inputData = require(`./data/${integration}_input_large.json`); + const expectedData = require(`./data/${integration}_async_output_large.json`); + + const respBody = { + code: ` + import url from '@rs/urlParser/v1'; + async function foo() { + return 'resolved'; + } + export async function transformEvent(event, metadata) { + const pr = await foo(); + if(event.properties && event.properties.url){ + const x = new url.URLSearchParams(event.properties.url).get("client"); + } + event.promise = pr; + return event; + } + `, + name: "urlParser", + codeVersion: "1" + }; + respBody.versionId = versionId; + const transformerUrl = `https://api.rudderlabs.com/transformation/getByVersionId?versionId=${versionId}`; + when(fetch) + .calledWith(transformerUrl) + .mockResolvedValue({ + status: 200, + json: jest.fn().mockResolvedValue(respBody) + }); + + const urlCode = `${fs.readFileSync( + path.resolve(__dirname, "../../src/util/url-search-params.min.js"), + "utf8" + )}; + export default self; + `; + + const rudderLibraryUrl = `https://api.rudderlabs.com/rudderstackTransformationLibraries/${name}?version=${version}`; + when(fetch) + .calledWith(rudderLibraryUrl) + .mockResolvedValue({ + status: 200, + json: jest.fn().mockResolvedValue({ code: urlCode, name: "urlParser", importName: rudderLibraryImportName }) + }); + + const output = await userTransformHandler(inputData, versionId, []); + + expect(fetch).toHaveBeenCalledWith( + `https://api.rudderlabs.com/transformation/getByVersionId?versionId=${versionId}` + ); + + expect(output).toEqual(expectedData); + }); +}); + // Running tests for python transformations with openfaas mocks describe("Python transformations", () => { beforeEach(() => {