diff --git a/core/adapters/bigquery.ts b/core/adapters/bigquery.ts index f2b7fb4ea..5c4f028e8 100644 --- a/core/adapters/bigquery.ts +++ b/core/adapters/bigquery.ts @@ -80,7 +80,11 @@ export class BigQueryAdapter extends Adapter implements IAdapter { return tasks; } - public createOrReplace(table: dataform.ITable) { + public dropIfExists(target: dataform.ITarget, type: dataform.TableMetadata.Type) { + return `drop ${this.tableTypeAsSql(type)} if exists ${this.resolveTarget(target)}`; + } + + private createOrReplace(table: dataform.ITable) { return `create or replace ${this.tableTypeAsSql( this.baseTableType(table.type) )} ${this.resolveTarget(table.target)} ${ @@ -94,16 +98,12 @@ export class BigQueryAdapter extends Adapter implements IAdapter { }as ${table.query}`; } - public createOrReplaceView(target: dataform.ITarget, query: string) { + private createOrReplaceView(target: dataform.ITarget, query: string) { return ` create or replace view ${this.resolveTarget(target)} as ${query}`; } - public dropIfExists(target: dataform.ITarget, type: dataform.TableMetadata.Type) { - return `drop ${this.tableTypeAsSql(type)} if exists ${this.resolveTarget(target)}`; - } - - public mergeInto( + private mergeInto( target: dataform.ITarget, columns: string[], query: string, diff --git a/core/adapters/redshift.ts b/core/adapters/redshift.ts index 1d0ab3793..1e4955531 100644 --- a/core/adapters/redshift.ts +++ b/core/adapters/redshift.ts @@ -38,7 +38,6 @@ export class RedshiftAdapter extends Adapter implements IAdapter { table.uniqueKey && table.uniqueKey.length > 0 ? this.mergeInto( table.target, - tableMetadata.fields.map(f => f.name), this.where(table.incrementalQuery || table.query, table.where), table.uniqueKey ) @@ -78,7 +77,15 @@ export class RedshiftAdapter extends Adapter implements IAdapter { .add(Task.assertion(`select sum(1) as row_count from ${this.resolveTarget(target)}`)); } - public createOrReplaceView(target: dataform.ITarget, query: string, bind: boolean) { + public dropIfExists(target: dataform.ITarget, type: dataform.TableMetadata.Type) { + const query = `drop ${this.tableTypeAsSql(type)} if exists ${this.resolveTarget(target)}`; + if (!this.isBindSupported()) { + return query; + } + return `${query} cascade`; + } + + private createOrReplaceView(target: dataform.ITarget, query: string, bind: boolean) { const createQuery = `create or replace view ${this.resolveTarget(target)} as ${query}`; // Postgres doesn't support with no schema binding. if (bind || this.project.warehouse === "postgres") { @@ -87,7 +94,7 @@ export class RedshiftAdapter extends Adapter implements IAdapter { return `${createQuery} with no schema binding`; } - public createOrReplace(table: dataform.ITable) { + private createOrReplace(table: dataform.ITable) { if (table.type === "view") { const isBindDefined = table.redshift && table.redshift.hasOwnProperty("bind"); const bindDefaultValue = semver.gte(this.dataformCoreVersion, "1.4.1") ? false : true; @@ -116,7 +123,7 @@ export class RedshiftAdapter extends Adapter implements IAdapter { ); } - public createTable(table: dataform.ITable, target: dataform.ITarget) { + private createTable(table: dataform.ITable, target: dataform.ITarget) { if (table.redshift) { let query = `create table ${this.resolveTarget(target)}`; @@ -135,20 +142,7 @@ export class RedshiftAdapter extends Adapter implements IAdapter { return `create table ${this.resolveTarget(target)} as ${table.query}`; } - public dropIfExists(target: dataform.ITarget, type: dataform.TableMetadata.Type) { - const query = `drop ${this.tableTypeAsSql(type)} if exists ${this.resolveTarget(target)}`; - if (!this.isBindSupported()) { - return query; - } - return `${query} cascade`; - } - - public mergeInto( - target: dataform.ITarget, - columns: string[], - query: string, - uniqueKey: string[] - ) { + private mergeInto(target: dataform.ITarget, query: string, uniqueKey: string[]) { const finalTarget = this.resolveTarget(target); // Schema name not allowed for temporary tables. const tempTarget = `"${target.schema}__${target.name}_incremental_temp"`; diff --git a/core/adapters/snowflake.ts b/core/adapters/snowflake.ts index 083d86bdb..266527461 100644 --- a/core/adapters/snowflake.ts +++ b/core/adapters/snowflake.ts @@ -73,23 +73,25 @@ export class SnowflakeAdapter extends Adapter implements IAdapter { schema: projectConfig.assertionSchema, name: assertion.name }); - tasks.add(Task.statement(this.createOrReplaceView(target, assertion.query))); + tasks.add(Task.statement(this.createOrReplaceView(target, assertion.query, false))); tasks.add(Task.assertion(`select sum(1) as row_count from ${this.resolveTarget(target)}`)); return tasks; } - public createOrReplaceView(target: dataform.ITarget, query: string) { - return ` - create or replace view ${this.resolveTarget(target)} as ${query}`; + private createOrReplaceView(target: dataform.ITarget, query: string, secure: boolean) { + return `create or replace ${secure ? "secure " : ""}view ${this.resolveTarget( + target + )} as ${query}`; } - public createOrReplace(table: dataform.ITable) { - return `create or replace ${this.tableTypeAsSql( - this.baseTableType(table.type || "table") - )} ${this.resolveTarget(table.target)} as ${table.query}`; + private createOrReplace(table: dataform.ITable) { + if (table.type === "view") { + return this.createOrReplaceView(table.target, table.query, table.snowflake?.secure); + } + return `create or replace table ${this.resolveTarget(table.target)} as ${table.query}`; } - public mergeInto( + private mergeInto( target: dataform.ITarget, columns: string[], query: string, diff --git a/core/adapters/sqldatawarehouse.ts b/core/adapters/sqldatawarehouse.ts index f22e7b49b..acc9e7a2b 100644 --- a/core/adapters/sqldatawarehouse.ts +++ b/core/adapters/sqldatawarehouse.ts @@ -90,7 +90,16 @@ export class SQLDataWarehouseAdapter extends Adapter implements IAdapter { )}','U') is not null drop table ${this.resolveTarget(target)}`; } - public createOrReplace(table: dataform.ITable, alreadyExists: boolean) { + public insertInto(target: dataform.ITarget, columns: string[], query: string) { + return ` +insert into ${this.resolveTarget(target)} +(${columns.join(",")}) +select ${columns.join(",")} +from (${query} +) as insertions`; + } + + private createOrReplace(table: dataform.ITable, alreadyExists: boolean) { if (table.type === "view") { return Tasks.create().add( Task.statement( @@ -116,7 +125,7 @@ export class SQLDataWarehouseAdapter extends Adapter implements IAdapter { ); } - public createTable(table: dataform.ITable, target: dataform.ITarget) { + private createTable(table: dataform.ITable, target: dataform.ITarget) { const distribution = table.sqlDataWarehouse && table.sqlDataWarehouse.distribution ? table.sqlDataWarehouse.distribution @@ -127,13 +136,4 @@ export class SQLDataWarehouseAdapter extends Adapter implements IAdapter { ) as ${table.query}`; } - - public insertInto(target: dataform.ITarget, columns: string[], query: string) { - return ` -insert into ${this.resolveTarget(target)} -(${columns.join(",")}) -select ${columns.join(",")} -from (${query} -) as insertions`; - } } diff --git a/core/session.ts b/core/session.ts index e3ee4c134..1f517184c 100644 --- a/core/session.ts +++ b/core/session.ts @@ -572,6 +572,17 @@ export class Session { ); } + // snowflake config + if (!!table.snowflake) { + if (table.snowflake.secure && table.type !== "view") { + this.compileError( + new Error(`The 'secure' option is only valid for Snowflake views`), + table.fileName, + table.name + ); + } + } + // sqldatawarehouse config if (!!table.sqlDataWarehouse) { if (!!table.uniqueKey && table.uniqueKey.length > 0) { diff --git a/core/table.ts b/core/table.ts index 08310c105..ffa08dedd 100644 --- a/core/table.ts +++ b/core/table.ts @@ -59,7 +59,7 @@ export const SortStyleType = ["compound", "interleaved"] as const; export type SortStyleType = typeof SortStyleType[number]; /** - * Redshift specific warehouse options. + * Redshift-specific warehouse options. */ export interface IRedshiftOptions { /** @@ -95,7 +95,20 @@ const IRedshiftOptionsProperties = () => strictKeysOf()(["distKey", "distStyle", "sortKeys", "sortStyle"]); /** - * Options for creating tables within Azure SQL Data Warehouse projects. + * Snowflake-specific warehouse options. + */ +export interface ISnowflakeOptions { + /** + * If set to true, a secure view will be created. + * + * For more information, read the [Snowflake Secure Views docs](https://docs.snowflake.com/en/user-guide/views-secure.html). + */ + secure?: boolean; +} +const ISnowflakeOptionsProperties = () => strictKeysOf()(["secure"]); + +/** + * Azure SQL Data Warehouse-specific warehouse options. */ export interface ISQLDataWarehouseOptions { /** @@ -109,7 +122,7 @@ const ISQLDataWarehouseOptionsProperties = () => strictKeysOf()(["distribution"]); /** - * Options for creating tables within BigQuery projects. + * BigQuery-specific warehouse options. */ export interface IBigQueryOptions { /** @@ -206,6 +219,11 @@ export interface ITableConfig */ bigquery?: IBigQueryOptions; + /** + * Snowflake-specific options. + */ + snowflake?: ISnowflakeOptions; + /** * Azure SQL Data Warehouse-specific options. */ @@ -235,6 +253,7 @@ export const ITableConfigProperties = () => "name", "redshift", "bigquery", + "snowflake", "sqldatawarehouse", "tags", "uniqueKey", @@ -274,6 +293,7 @@ export class Table { inline: [ "bigquery", "redshift", + "snowflake", "sqlDataWarehouse", "preOps", "postOps", @@ -326,6 +346,9 @@ export class Table { if (config.bigquery) { this.bigquery(config.bigquery); } + if (config.snowflake) { + this.snowflake(config.snowflake); + } if (config.sqldatawarehouse) { this.sqldatawarehouse(config.sqldatawarehouse); } @@ -393,6 +416,17 @@ export class Table { this.proto.uniqueKey = uniqueKey; } + public snowflake(snowflake: dataform.ISnowflakeOptions) { + checkExcessProperties( + (e: Error) => this.session.compileError(e), + snowflake, + ISnowflakeOptionsProperties(), + "snowflake config" + ); + this.proto.snowflake = dataform.SnowflakeOptions.create(snowflake); + return this; + } + public sqldatawarehouse(sqlDataWarehouse: dataform.ISQLDataWarehouseOptions) { checkExcessProperties( (e: Error) => this.session.compileError(e), diff --git a/protos/core.proto b/protos/core.proto index 0f32a8a98..e211a1333 100644 --- a/protos/core.proto +++ b/protos/core.proto @@ -164,6 +164,10 @@ message RedshiftOptions { bool bind = 5 [deprecated = true]; } +message SnowflakeOptions { + bool secure = 1; +} + message SQLDataWarehouseOptions { string distribution = 1; } @@ -285,6 +289,7 @@ message Table { // Warehouse specific features. RedshiftOptions redshift = 21; BigQueryOptions bigquery = 22; + SnowflakeOptions snowflake = 33; SQLDataWarehouseOptions sql_data_warehouse = 25; // Generated. diff --git a/tests/integration/snowflake_project/definitions/example_view.sqlx b/tests/integration/snowflake_project/definitions/example_view.sqlx index a2d97a2a4..858074b3b 100644 --- a/tests/integration/snowflake_project/definitions/example_view.sqlx +++ b/tests/integration/snowflake_project/definitions/example_view.sqlx @@ -3,6 +3,9 @@ config { description: "An example view", columns: { val: "val doc", + }, + snowflake: { + secure: true } } select * from ${ref("sample_data")} diff --git a/version.bzl b/version.bzl index f3e386909..019ac3016 100644 --- a/version.bzl +++ b/version.bzl @@ -1,3 +1,3 @@ # NOTE: If you change the format of this line, you must change the bash command # in /scripts/publish to extract the version string correctly. -DF_VERSION = "1.11.1" +DF_VERSION = "1.12.0"