fix((INT-360): enhancement: shopify ecom fix and enhancement

src/v0/sources/shopify V2/commonUtils.js

@@ -0,0 +1,127 @@
+/* eslint-disable camelcase */
+const sha256 = require('sha256');
+const stats = require('../../../util/stats');
+const { constructPayload, extractCustomFields, flattenJson } = require('../../util');
+const { RedisDB } = require('../../../util/redis/redisConnector');
+const logger = require('../../../logger');
+const {
+  lineItemsMappingJSON,
+  productMappingJSON,
+  LINE_ITEM_EXCLUSION_FIELDS,
+  PRODUCT_MAPPING_EXCLUSION_FIELDS,
+  SHOPIFY_TRACK_MAP,
+} = require('./config');
+const { TransformationError } = require('../../util/errorTypes');
+
+const getCartToken = (message) => {
+  const { event } = message;
+  if (event === SHOPIFY_TRACK_MAP.carts_update) {
+    return message.properties?.id || message.properties?.token;
+  }
+  return message.properties?.cart_token || null;
+};
+
+const getDataFromRedis = async (key, metricMetadata) => {
+  try {
+    stats.increment('shopify_redis_calls', {
+      type: 'get',
+      field: 'all',
+      ...metricMetadata,
+    });
+    const redisData = await RedisDB.getVal(key);
+    if (redisData === null) {
+      stats.increment('shopify_redis_no_val', {
+        ...metricMetadata,
+      });
+    }
+    return redisData;
+  } catch (e) {
+    logger.debug(`{{SHOPIFY::}} Get call Failed due redis error ${e}`);
+    stats.increment('shopify_redis_failures', {
+      type: 'get',
+      ...metricMetadata,
+    });
+  }
+  return null;
+};
+
+/**
+ * query_parameters : { topic: ['<shopify_topic>'], ...}
+ * Throws error otherwise
+ * @param {*} event
+ * @returns
+ */
+const getShopifyTopic = (event) => {
+  const { query_parameters: qParams } = event;
+  logger.debug(`[Shopify] Input event: query_params: ${JSON.stringify(qParams)}`);
+  if (!qParams) {
+    throw new TransformationError('Query_parameters is missing');
+  }
+  const { topic } = qParams;
+  if (!topic || !Array.isArray(topic)) {
+    throw new TransformationError('Invalid topic in query_parameters');
+  }
+  if (topic.length === 0) {
+    throw new TransformationError('Topic not found');
+  }
+  return topic[0];
+};
+
+const getHashLineItems = (cart) => {
+  if (cart && cart?.line_items && cart.line_items.length > 0) {
+    return sha256(JSON.stringify(cart.line_items));
+  }
+  return 'EMPTY';
+};
+
+const getVariantString = (lineItem) => {
+  const { variant_id, variant_price, variant_title } = lineItem;
+  return `${variant_id || ''} ${variant_price || ''} ${variant_title || ''}`;
+};
+
+const getProductsListFromLineItems = (lineItems) => {
+  if (!lineItems || lineItems.length === 0) {
+    return [];
+  }
+  const products = [];
+  lineItems.forEach((lineItem) => {
+    const product = constructPayload(lineItem, lineItemsMappingJSON);
+    extractCustomFields(lineItem, product, 'root', LINE_ITEM_EXCLUSION_FIELDS);
+    product.variant = getVariantString(lineItem);
+    products.push(product);
+  });
+  return products;
+};
+
+const createPropertiesForEcomEvent = (message) => {
+  const { line_items: lineItems } = message;
+  const productsList = getProductsListFromLineItems(lineItems);
+  const mappedPayload = constructPayload(message, productMappingJSON);
+  extractCustomFields(message, mappedPayload, 'root', PRODUCT_MAPPING_EXCLUSION_FIELDS);
+  mappedPayload.products = productsList;
+  return mappedPayload;
+};
+
+const extractEmailFromPayload = (event) => {
+  const flattenedPayload = flattenJson(event);
+  let email;
+  const regex_email = /\bemail\b/i;
+  Object.entries(flattenedPayload).some(([key, value]) => {
+    if (regex_email.test(key)) {
+      email = value;
+      return true;
+    }
+    return false;
+  });
+  return email;
+};
+
+module.exports = {
+  getCartToken,
+  getShopifyTopic,
+  getProductsListFromLineItems,
+  createPropertiesForEcomEvent,
+  extractEmailFromPayload,
+  getHashLineItems,
+  getDataFromRedis,
+};

src/v0/sources/shopify V2/config.js

@@ -0,0 +1,130 @@
+const path = require('path');
+const fs = require('fs');
+const { EventType } = require('../../../constants');
+
+const INTEGERATION = 'SHOPIFY';
+
+const NO_OPERATION_SUCCESS = {
+  outputToSource: {
+    body: Buffer.from('OK').toString('base64'),
+    contentType: 'text/plain',
+  },
+  statusCode: 200,
+};
+
+const identifierEvents = ['rudderIdentifier', 'rudderSessionIdentifier'];
+
+const IDENTIFY_TOPICS = {
+  CUSTOMERS_CREATE: 'customers_create',
+  CUSTOMERS_UPDATE: 'customers_update',
+};
+
+const RUDDER_ECOM_MAP = {   // TOBEUPDATED:
+  checkouts_create: 'Checkout Started',
+  checkouts_update: 'Checkout Updated',
+  orders_updated: 'Order Updated',
+  orders_create: 'Order Created',
+  carts_update: 'Cart Updated' // This will split into Product Added and Product Removed
+};
+
+const SHOPIFY_TO_RUDDER_ECOM_EVENTS_MAP = ['Cart Update', 'Checkout Updated'];
+
+const SHOPIFY_ADMIN_ONLY_EVENTS = ['Order Deleted', 'Fulfillments Create', 'Fulfillments Update'];
+
+const SHOPIFY_TRACK_MAP = {
+  checkouts_delete: 'Checkout Deleted',
+  carts_update: 'Cart Update',
+  customers_enable: 'Customer Enabled',
+  customers_disable: 'Customer Disabled',
+  fulfillments_create: 'Fulfillments Create',
+  fulfillments_update: 'Fulfillments Update',
+  orders_delete: 'Order Deleted',
+  orders_edited: 'Order Edited',
+  orders_cancelled: 'Order Cancelled',
+  orders_fulfilled: 'Order Fulfilled',
+  orders_paid: 'Order Paid',
+  orders_partially_fullfilled: 'Order Partially Fulfilled',
+};
+
+const identifyMappingJSON = JSON.parse(
+  fs.readFileSync(path.resolve(__dirname, 'data', 'identifyMapping.json')),
+);
+
+const productMappingJSON = JSON.parse(
+  fs.readFileSync(path.resolve(__dirname, 'data', 'productMapping.json')),
+);
+
+const lineItemsMappingJSON = JSON.parse(
+  fs.readFileSync(path.resolve(__dirname, 'data', 'lineItemsMapping.json')),
+);
+
+const MAPPING_CATEGORIES = {
+  [EventType.IDENTIFY]: identifyMappingJSON,
+  [EventType.TRACK]: productMappingJSON,
+  // update it for every ECOM ma[ong and genera mapping]
+};
+
+const LINE_ITEM_EXCLUSION_FIELDS = [
+  'product_id',
+  'sku',
+  'name',
+  'price',
+  'vendor',
+  'quantity',
+  'variant_id',
+  'variant_price',
+  'variant_title',
+];
+
+const PRODUCT_MAPPING_EXCLUSION_FIELDS = [
+  'id',
+  'total_price',
+  'total_tax',
+  'currency',
+  'line_items',
+  'customer',
+  'shipping_address',
+  'billing_address',
+];
+
+/**
+ * list of events name supported as generic track calls
+ * track events not belonging to this list or ecom events will
+ * be discarded.
+ */
+const NON_ECOM_SUPPORTED_EVENTS = [ // to be updated 
+  'checkouts_delete',
+  'checkouts_update',
+  'customers_disable',
+  'customers_enable',
+  'carts_update',
+  'fulfillments_create',
+  'fulfillments_update',
+  'orders_create',
+  'orders_delete',
+  'orders_edited',
+  'orders_cancelled',
+  'orders_fulfilled',
+  'orders_paid',
+  'orders_partially_fullfilled',
+];
+
+const maxTimeToIdentifyRSGeneratedCall = 10000; // in ms
+
+module.exports = {
+  NO_OPERATION_SUCCESS,
+  identifierEvents,
+  IDENTIFY_TOPICS,
+  INTEGERATION,
+  SHOPIFY_TO_RUDDER_ECOM_EVENTS_MAP,
+  MAPPING_CATEGORIES,
+  RUDDER_ECOM_MAP,
+  lineItemsMappingJSON,
+  productMappingJSON,
+  LINE_ITEM_EXCLUSION_FIELDS,
+  PRODUCT_MAPPING_EXCLUSION_FIELDS,
+  NON_ECOM_SUPPORTED_EVENTS,
+  SHOPIFY_TRACK_MAP,
+  SHOPIFY_ADMIN_ONLY_EVENTS,
+  maxTimeToIdentifyRSGeneratedCall
+};

src/v0/sources/shopify V2/data/identifyMapping.json

@@ -0,0 +1,112 @@
+[
+  {
+    "sourceKeys": "id",
+    "destKeys": "userId"
+  },
+  {
+    "sourceKeys": "email",
+    "destKeys": "traits.email"
+  },
+
+  {
+    "sourceKeys": "first_name",
+    "destKeys": "traits.firstName"
+  },
+
+  {
+    "sourceKeys": "last_name",
+    "destKeys": "traits.lastName"
+  },
+
+  {
+    "sourceKeys": "phone",
+    "destKeys": "traits.phone"
+  },
+
+  {
+    "sourceKeys": "addresses",
+    "destKeys": "traits.addressList"
+  },
+
+  {
+    "sourceKeys": "default_address",
+    "destKeys": "traits.address"
+  },
+
+  {
+    "sourceKeys": "shipping_address",
+    "destKeys": "traits.shippingAddress"
+  },
+
+  {
+    "sourceKeys": "billing_address",
+    "destKeys": "traits.billingAddress"
+  },
+
+  {
+    "sourceKeys": "accepts_marketing",
+    "destKeys": "traits.acceptsMarketing"
+  },
+
+  {
+    "sourceKeys": "orders_count",
+    "destKeys": "traits.orderCount"
+  },
+
+  {
+    "sourceKeys": "state",
+    "destKeys": "traits.state"
+  },
+  {
+    "sourceKeys": "total_spent",
+    "destKeys": "traits.totalSpent"
+  },
+  {
+    "sourceKeys": "note",
+    "destKeys": "traits.note"
+  },
+  {
+    "sourceKeys": "verified_email",
+    "destKeys": "traits.verifiedEmail"
+  },
+  {
+    "sourceKeys": "multipass_identifier",
+    "destKeys": "traits.multipassIdentifier"
+  },
+  {
+    "sourceKeys": "tax_exempt",
+    "destKeys": "traits.taxExempt"
+  },
+  {
+    "sourceKeys": "tags",
+    "destKeys": "traits.tags"
+  },
+  {
+    "sourceKeys": "last_order_name",
+    "destKeys": "traits.lastOrderName"
+  },
+  {
+    "sourceKeys": "currency",
+    "destKeys": "traits.currency"
+  },
+  {
+    "sourceKeys": "marketing_opt_in_level",
+    "destKeys": "traits.marketingOptInLevel"
+  },
+  {
+    "sourceKeys": "tax_exemptions",
+    "destKeys": "traits.taxExemptions"
+  },
+  {
+    "sourceKeys": "sms_marketing_consent",
+    "destKeys": "traits.smsMarketingConsent"
+  },
+  {
+    "sourceKeys": "admin_graphql_api_id",
+    "destKeys": "traits.adminGraphqlApiId"
+  },
+  {
+    "sourceKeys": "accepts_marketing_updated_at",
+    "destKeys": "traits.acceptsMarketingUpdatedAt"
+  }
+]

src/v0/sources/shopify V2/data/lineItemsMapping.json

@@ -0,0 +1,26 @@
+[
+  {
+    "sourceKeys": "product_id",
+    "destKey": "product_id"
+  },
+  {
+    "sourceKeys": "sku",
+    "destKey": "sku"
+  },
+  {
+    "sourceKeys": "name",
+    "destKey": "title"
+  },
+  {
+    "sourceKeys": "price",
+    "destKey": "price"
+  },
+  {
+    "sourceKeys": "vendor",
+    "destKey": "brand"
+  },
+  {
+    "sourceKeys": "quantity",
+    "destKey": "quantity"
+  }
+]