Skip to content

Commit

Permalink
feat: snowpipe streaming
Browse files Browse the repository at this point in the history
  • Loading branch information
achettyiitr committed Oct 19, 2024
1 parent 921a988 commit 18b1f3f
Show file tree
Hide file tree
Showing 2 changed files with 158 additions and 115 deletions.
34 changes: 34 additions & 0 deletions src/v0/destinations/snowpipe_streaming/transform.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,34 @@
const { processWarehouseMessage } = require('../../../warehouse');

const provider = 'snowflake';

function getDataTypeOverride(key, val, options, jsonKey = false) {
if (key === 'violationErrors' || jsonKey) {
return 'json';
}
return 'string';
}

function process(event) {
const whSchemaVersion = event.request.query.whSchemaVersion || 'v1';
const whIDResolve = event.request.query.whIDResolve === 'true' || false;
const whStoreEvent = event.destination.Config.storeFullEvent === true;
const destJsonPaths = event.destination?.Config?.jsonPaths || '';
return processWarehouseMessage(event.message, {
metadata: event.metadata,
whSchemaVersion,
whStoreEvent,
whIDResolve,
getDataTypeOverride,
provider,
sourceCategory: event.metadata ? event.metadata.sourceCategory : null,
destJsonPaths,
destConfig: event.destination?.Config,
});
}

module.exports = {
provider,
process,
getDataTypeOverride,
};
Loading

0 comments on commit 18b1f3f

Please sign in to comment.