Merge branch 'develop' into feat.hs-handle-partial-error

.github/workflows/verify-server-start.yml

@@ -0,0 +1,34 @@
+name: Verify Server start
+
+on:
+  pull_request:
+    types: ['opened', 'reopened', 'synchronize']
+
+jobs:
+  check-health:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout
+        uses: actions/[email protected]
+        with:
+          fetch-depth: 1
+
+      - name: Setup Node
+        uses: actions/[email protected]
+        with:
+          node-version-file: '.nvmrc'
+          cache: 'npm'
+
+      - name: Install Dependencies
+        run: npm ci
+
+      - name: Start server
+        run: npm run build:start &
+
+      - name: Wait for server to start
+        run: sleep 10 # Adjust the time as necessary for your server to start
+
+      - name: Check server health
+        run: |
+          curl --fail http://localhost:9090/health || exit 1

benchmark/metaLogger.js

@@ -2,30 +2,30 @@
 
 const logger = require('../src/logger');
 
-logger.setLogLevel(Number.POSITIVE_INFINITY);
+logger.setLogLevel('random');
 
 const debug = (...args) => {
-  logger.setLogLevel(logger.levelDebug);
+  logger.setLogLevel('debug');
   logger.debug(...args);
-  logger.setLogLevel(Number.POSITIVE_INFINITY);
+  logger.setLogLevel('random');
 };
 
 const info = (...args) => {
-  logger.setLogLevel(logger.levelInfo);
+  logger.setLogLevel('info');
   logger.info(...args);
-  logger.setLogLevel(Number.POSITIVE_INFINITY);
+  logger.setLogLevel('random');
 };
 
 const warn = (...args) => {
-  logger.setLogLevel(logger.levelWarn);
+  logger.setLogLevel('warn');
   logger.warn(...args);
-  logger.setLogLevel(Number.POSITIVE_INFINITY);
+  logger.setLogLevel('random');
 };
 
 const error = (...args) => {
-  logger.setLogLevel(logger.levelError);
+  logger.setLogLevel('error');
   logger.error(...args);
-  logger.setLogLevel(Number.POSITIVE_INFINITY);
+  logger.setLogLevel('random');
 };
 
 module.exports = {

package.json

@@ -64,9 +64,9 @@
     "@koa/router": "^12.0.0",
     "@ndhoule/extend": "^2.0.0",
     "@pyroscope/nodejs": "^0.2.9",
-    "@rudderstack/integrations-lib": "^0.2.8",
-    "@rudderstack/json-template-engine": "^0.13.0",
-    "@rudderstack/workflow-engine": "^0.8.1",
+    "@rudderstack/integrations-lib": "^0.2.10",
+    "@rudderstack/json-template-engine": "^0.14.1",
+    "@rudderstack/workflow-engine": "^0.8.8",
     "@shopify/jest-koa-mocks": "^5.1.1",
     "ajv": "^8.12.0",
     "ajv-draft-04": "^1.0.0",
@@ -93,6 +93,7 @@
     "koa": "^2.14.1",
     "koa-bodyparser": "^4.4.0",
     "koa2-swagger-ui": "^5.7.0",
+    "libphonenumber-js": "^1.11.1",
     "lodash": "^4.17.21",
     "match-json": "^1.3.5",
     "md5": "^2.3.0",

src/adapters/network.js

@@ -45,21 +45,28 @@ const networkClientConfigs = {
   httpsAgent: new https.Agent({ keepAlive: true }),
 };
 
-const fireHTTPStats = (clientResponse, startTime, statTags) => {
-  const destType = statTags.destType ? statTags.destType : '';
-  const feature = statTags.feature ? statTags.feature : '';
-  const endpointPath = statTags.endpointPath ? statTags.endpointPath : '';
-  const requestMethod = statTags.requestMethod ? statTags.requestMethod : '';
-  const module = statTags.module ? statTags.module : '';
-  const statusCode = clientResponse.success ? clientResponse.response.status : '';
+const fireOutgoingReqStats = ({
+  destType,
+  feature,
+  endpointPath,
+  requestMethod,
+  module,
+  metadata = {},
+  startTime,
+  statusCode,
+  clientResponse,
+}) => {
+  const logMetaInfo = log.getLogMetadata(metadata);
   stats.timing('outgoing_request_latency', startTime, {
+    ...logMetaInfo,
     feature,
     destType,
     endpointPath,
     requestMethod,
     module,
   });
   stats.counter('outgoing_request_count', 1, {
+    ...logMetaInfo,
     feature,
     destType,
     endpointPath,
@@ -70,6 +77,36 @@ const fireHTTPStats = (clientResponse, startTime, statTags) => {
   });
 };
 
+const fireHTTPStats = (clientResponse, startTime, statTags) => {
+  const destType = statTags.destType ? statTags.destType : '';
+  const feature = statTags.feature ? statTags.feature : '';
+  const endpointPath = statTags.endpointPath ? statTags.endpointPath : '';
+  const requestMethod = statTags.requestMethod ? statTags.requestMethod : '';
+  const module = statTags.module ? statTags.module : '';
+  const statusCode = clientResponse.success ? clientResponse.response.status : '';
+  const defArgs = {
+    destType,
+    endpointPath,
+    feature,
+    module,
+    requestMethod,
+    statusCode,
+    startTime,
+    clientResponse,
+  };
+  if (statTags?.metadata) {
+    const metadata = !Array.isArray(statTags?.metadata) ? [statTags.metadata] : statTags.metadata;
+    metadata?.forEach((m) => {
+      fireOutgoingReqStats({
+        ...defArgs,
+        metadata: m,
+      });
+    });
+    return;
+  }
+  fireOutgoingReqStats(defArgs);
+};
+
 const enhanceRequestOptions = (options) => {
   const requestOptions = {
     ...networkClientConfigs,
@@ -322,25 +359,6 @@ const prepareProxyRequest = (request) => {
   return removeUndefinedValues({ endpoint, data, params, headers, method, config });
 };
 
-/**
- * depricating: handles proxying requests to destinations from server, expects requsts in "defaultRequestConfig"
- * note: needed for test api
- * @param {*} request
- * @returns
- */
-const proxyRequest = async (request, destType) => {
-  const { endpoint, data, method, params, headers } = prepareProxyRequest(request);
-  const requestOptions = {
-    url: endpoint,
-    data,
-    params,
-    headers,
-    method,
-  };
-  const response = await httpSend(requestOptions, { feature: 'proxy', destType });
-  return response;
-};
-
 /**
  * handles http request and sends the response in a simple format that is followed in transformer
  *
@@ -392,6 +410,38 @@ const handleHttpRequest = async (requestType = 'post', ...httpArgs) => {
   return { httpResponse, processedResponse };
 };
 
+/**
+ * depricating: handles proxying requests to destinations from server, expects requsts in "defaultRequestConfig"
+ * note: needed for test api
+ * @param {*} request
+ * @returns
+ */
+const proxyRequest = async (request, destType) => {
+  const { metadata } = request;
+  const { endpoint, data, method, params, headers } = prepareProxyRequest(request);
+  const requestOptions = {
+    url: endpoint,
+    data,
+    params,
+    headers,
+    method,
+  };
+  log.requestLog(`[${destType.toUpperCase()}] delivering data`, {
+    metadata,
+    requestDetails: {
+      body: data,
+      url: endpoint,
+      method,
+    },
+  });
+  const response = await httpSend(requestOptions, {
+    feature: 'proxy',
+    destType,
+    metadata,
+  });
+  return response;
+};
+
 module.exports = {
   httpSend,
   httpGET,

src/adapters/utils/networkUtils.js

@@ -144,10 +144,11 @@ const processAxiosResponse = (clientResponse) => {
     }
     // non 2xx status handling for axios response
     if (response) {
-      const { data, status } = response;
+      const { data, status, headers } = response;
       return {
         response: data || '',
         status: status || 500,
+        ...(isDefinedAndNotNullAndNotEmpty(headers) ? { headers } : {}),
       };
     }
     // (edge case) response and code is not present
@@ -157,10 +158,11 @@ const processAxiosResponse = (clientResponse) => {
     };
   }
   // success(2xx) axios response
-  const { data, status } = clientResponse.response;
+  const { data, status, headers } = clientResponse.response;
   return {
     response: data || '',
     status: status || 500,
+    ...(isDefinedAndNotNullAndNotEmpty(headers) ? { headers } : {}),
   };
 };
 

src/cdk/v2/destinations/gladly/procWorkflow.yaml

@@ -61,7 +61,8 @@ steps:
         headers: $.getHeaders(.destination)
       }
       const endpoint = $.getEndpoint(.destination) + "?" + $.getQueryParams($.context.payload);
-      const rawResponse = await $.httpGET(endpoint,requestOptions)
+      const reqStats = {metadata:.metadata, module: 'router',feature: "transformation", destType:"gladly",requestMethod:"get",endpointPath:"/api/v1/customer-profiles"}
+      const rawResponse = await $.httpGET(endpoint,requestOptions, reqStats)
       const processedResponse = $.processAxiosResponse(rawResponse)
       processedResponse