Skip to content

Commit

Permalink
Merge branch 'develop' into feat.sprig-user-deletion
Browse files Browse the repository at this point in the history
  • Loading branch information
@mihir-4116
mihir-4116 authored Jan 4, 2024
2 parents 0b142ca + e002093 commit 7174320
Show file tree
Hide file tree
Showing 6 changed files with 118 additions and 20 deletions.
3 changes: 3 additions & 0 deletions .gitignore
View file
Original file line number Diff line number Diff line change
Expand Up @@ -132,3 +132,6 @@ dist

# Others
**/.DS_Store


.idea
9 changes: 8 additions & 1 deletion src/warehouse/config/helpers.js
View file
Original file line number Diff line number Diff line change
@@ -1,12 +1,18 @@
const _ = require('lodash');
const get = require('get-value');
const logger = require('../../logger');

const isNull = (x) => {
return x === null || x === undefined;
};

const isBlank = (value) => {
return _.isEmpty(_.toString(value));
try {
return _.isEmpty(_.toString(value));
} catch (e) {
logger.error(`Error in isBlank: ${e.message}`);
return false;
}
};

const getFirstValidValue = (message, props) => {
Expand All @@ -27,6 +33,7 @@ function isDataLakeProvider(provider) {

module.exports = {
isNull,
isBlank,
getFirstValidValue,
isDataLakeProvider,
};
3 changes: 1 addition & 2 deletions src/warehouse/index.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,6 @@ const { v4: uuidv4 } = require('uuid');

const {
isObject,
isBlank,
isValidJsonPathKey,
isValidLegacyJsonPathKey,
keysFromJsonPaths,
Expand All @@ -24,7 +23,7 @@ const whPageColumnMappingRules = require('./config/WHPageConfig.js');
const whScreenColumnMappingRules = require('./config/WHScreenConfig.js');
const whGroupColumnMappingRules = require('./config/WHGroupConfig.js');
const whAliasColumnMappingRules = require('./config/WHAliasConfig.js');
const { isDataLakeProvider } = require('./config/helpers');
const {isDataLakeProvider, isBlank} = require('./config/helpers');
const { InstrumentationError } = require('@rudderstack/integrations-lib');
const whExtractEventTableColumnMappingRules = require('./config/WHExtractEventTableConfig.js');

Expand Down
9 changes: 4 additions & 5 deletions src/warehouse/util.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ const get = require('get-value');
const v0 = require('./v0/util');
const v1 = require('./v1/util');
const { PlatformError, InstrumentationError } = require('@rudderstack/integrations-lib');
const {isBlank} = require('./config/helpers');

const minTimeInMs = Date.parse('0001-01-01T00:00:00Z');
const maxTimeInMs = Date.parse('9999-12-31T23:59:59.999Z');
Expand All @@ -22,10 +23,6 @@ const isValidLegacyJsonPathKey = (eventType, key, level, jsonKeys = {}) => {
return eventType === 'track' && jsonKeys[key] === level;
};

const isBlank = (value) => {
return _.isEmpty(_.toString(value));
};

/*
This function takes in an array of json paths and returns an object with keys as the json path and value as the position of the key in the json path
Example:
Expand Down Expand Up @@ -97,6 +94,9 @@ const timestampRegex = new RegExp(
);

function validTimestamp(input) {
if (typeof input !== 'string') {
return false;
}
if (timestampRegex.test(input)) {
// check if date value lies in between min time and max time. if not then it's not a valid timestamp
const d = new Date(input);
Expand Down Expand Up @@ -147,7 +147,6 @@ const getRecordIDForExtract = (message) => {

module.exports = {
isObject,
isBlank,
isValidJsonPathKey,
isValidLegacyJsonPathKey,
keysFromJsonPaths,
Expand Down
34 changes: 30 additions & 4 deletions test/__tests__/data/warehouse/events.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -60,7 +60,17 @@ const sampleEvents = {
originalTimestamp: "2020-01-24T06:29:02.364Z",
properties: {
currency: "USD",
revenue: 50
revenue: 50,
stack: {
history: {
errorDetails: [
{
"message": "Cannot set headers after they are sent to the client",
"toString": "[function]"
}
]
}
}
},
receivedAt: "2020-01-24T11:59:02.403+05:30",
request_ip: "[::1]:53708",
Expand Down Expand Up @@ -180,6 +190,12 @@ const sampleEvents = {
data: {
currency: "USD",
revenue: 50,
stack_history_error_details: [
{
"message": "Cannot set headers after they are sent to the client",
"toString": "[function]"
}
],
context_app_build: "1.0.0",
context_app_name: "RudderLabs JavaScript SDK",
context_app_namespace: "com.rudderlabs.javascript",
Expand Down Expand Up @@ -314,6 +330,12 @@ const sampleEvents = {
data: {
CURRENCY: "USD",
REVENUE: 50,
STACK_HISTORY_ERROR_DETAILS: [
{
"message": "Cannot set headers after they are sent to the client",
"toString": "[function]"
}
],
CONTEXT_APP_BUILD: "1.0.0",
CONTEXT_APP_NAME: "RudderLabs JavaScript SDK",
CONTEXT_APP_NAMESPACE: "com.rudderlabs.javascript",
Expand All @@ -340,7 +362,7 @@ const sampleEvents = {
RECEIVED_AT: "2020-01-24T06:29:02.403Z",
ORIGINAL_TIMESTAMP: "2020-01-24T06:29:02.364Z",
CHANNEL: "web",
EVENT: "button_clicked"
EVENT: "button_clicked",
}
}
],
Expand Down Expand Up @@ -448,6 +470,12 @@ const sampleEvents = {
data: {
currency: "USD",
revenue: 50,
stack_history_error_details: [
{
"message": "Cannot set headers after they are sent to the client",
"toString": "[function]"
}
],
context_app_build: "1.0.0",
context_app_name: "RudderLabs JavaScript SDK",
context_app_namespace: "com.rudderlabs.javascript",
Expand Down Expand Up @@ -2053,7 +2081,6 @@ const sampleEvents = {
id: "string",
user_id: "string",
received_at: "datetime",
event: "string"
},
receivedAt: "2020-01-24T11:59:02.403+05:30"
},
Expand Down Expand Up @@ -2121,7 +2148,6 @@ const sampleEvents = {
id: "string",
user_id: "string",
received_at: "datetime",
event: "string",
boolean_property: "boolean",
},
receivedAt: "2020-01-24T11:59:02.403+05:30"
Expand Down
80 changes: 72 additions & 8 deletions test/__tests__/warehouse.test.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@ const {
const {
validTimestamp
} = require("../../src/warehouse/util.js");
const {isBlank} = require("../../src/warehouse/config/helpers.js");

const version = "v0";
const integrations = [
Expand Down Expand Up @@ -1096,54 +1097,117 @@ describe("Integration options", () => {
describe("validTimestamp", () => {
const testCases = [
{
name: "undefined input should return false",
input: undefined,
expected: false,
},
{
name: "negative year and time input should return false #1",
input: '-0001-11-30T00:00:00+0000',
expected: false,
},
{
name: "negative year and time input should return false #2",
input: '-2023-06-14T05:23:59.244Z',
expected: false,
},
{
name: "negative year and time input should return false #3",
input: '-1900-06-14T05:23:59.244Z',
expected: false,
},
{
name: "positive year and time input should return false",
input: '+2023-06-14T05:23:59.244Z',
expected: false,
},
{
name: "valid timestamp input should return true",
input: '2023-06-14T05:23:59.244Z',
expected: true,
},
{
input: '-1900-06-14T05:23:59.244Z',
expected: false,
},
{
name: "non-date string input should return false",
input: 'abc',
expected: false,
},
{
input: '%u002e%u002e%u2216%u002e%u002e%u2216%u002e%u002e%u2216%u002e%u002e%u2216%u002e%u002e%u2216%u002e%u002e%u2216%u002e%u002e%u2216%u002e%u002e%u2216%u002e%u002e%u2216%u002e%u002e%u2216%u002e%u002e%u2216%u002e%u002e%u2216%u002e%u002e%u2216%u002e%u002e%u2216%u002e%u002e%u2216%u002e%u002e%u2216%u002e%u002e%u2216%u002e%u002e%u2216Windows%u2216win%u002ein',
name: "malicious string input should return false",
input: '%u002e%u002e%u2216%u002e%u002e%u2216%u002e%u002e%u2216%u002e%u002e%u2216%u002e%u002e%u2216%u002e%u002e%u2216%u002e%u002e%u2216%u002e%u002e%u2216%u002e%u002e%u2216%u002e%u002e%u2216%u002e%u002e%u2216%u002e%u002e%u2216%u002e%u002e%u2216%u002e%u002e%u2216%u002e%u002e%u2216%u002e%u002e%u2216%u002e%u002e%u2216%u002e%u002e%u2216%u002e%u002e%u2216%u002e%u002e%u2216Windows%u2216win%u002ein',
expected: false,
},
{
name: "empty string input should return false",
input: '',
expected: false,
},
{
name: "valid date input should return true",
input: '2023-06-14',
expected: true,
},
{
name: "time-only input should return false",
input: '05:23:59.244Z',
expected: false,
}
]
},
{
name: "non-string input should return false",
input: { abc: 123 },
expected: false,
},
{
name: "object with toString method input should return false",
input: {
toString: '2023-06-14T05:23:59.244Z'
},
expected: false,
},
];

for (const testCase of testCases) {
it(`should return ${testCase.expected} for ${testCase.input}`, () => {
it(`should return ${testCase.expected} for ${testCase.name}`, () => {
expect(validTimestamp(testCase.input)).toEqual(testCase.expected);
});
}
});



describe("isBlank", () => {
const testCases = [
{
name: "null",
input: null,
expected: true
},
{
name: "empty string",
input: "",
expected: true
},
{
name: "non-empty string",
input: "test",
expected: false
},
{
name: "numeric value",
input: 1634762544,
expected: false
},
{
name: "object with toString property",
input: {
toString: '2023-06-14T05:23:59.244Z'
},
expected: false
},
];

for (const testCase of testCases) {
it(`should return ${testCase.expected} for ${testCase.name}`, () => {
expect(isBlank(testCase.input)).toEqual(testCase.expected);
});
}
});

0 comments on commit 7174320

Please sign in to comment.