Skip to content

Commit

Permalink
Always uppercase all Snowflake identifiers. (#409)
Browse files Browse the repository at this point in the history
  • Loading branch information
@BenBirt
BenBirt authored Aug 30, 2019
1 parent 90630aa commit 965aea1
Show file tree
Hide file tree
Showing 7 changed files with 96 additions and 75 deletions.
32 changes: 18 additions & 14 deletions core/adapters/base.ts
View file
Original file line number Diff line number Diff line change
@@ -1,35 +1,39 @@
import { dataform } from "@dataform/protos";

export abstract class Adapter {
public where(query: string, where: string) {
return `select * from (${query}) as subquery
where ${where}`;
public abstract resolveTarget(target: dataform.ITarget): string;

public normalizeIdentifier(identifier: string) {
return identifier;
}

public dropIfExists(target: dataform.ITarget, type: string) {
return `drop ${this.baseTableType(type)} if exists ${this.resolveTarget(target)} ${
this.baseTableType(type) == "table" ? "cascade" : ""
}`;
}

public baseTableType(type: string) {
if (type == "incremental") {
if (type === "incremental") {
return "table";
}
return type;
}

public oppositeTableType(type: string) {
return this.baseTableType(type) == "table" ? "view" : "table";
}

public insertInto(target: dataform.ITarget, columns: string[], query: string) {
protected insertInto(target: dataform.ITarget, columns: string[], query: string) {
return `
insert into ${this.resolveTarget(target)}
(${columns.join(",")})
select ${columns.join(",")}
from (${query}) as insertions`;
}

public dropIfExists(target: dataform.ITarget, type: string) {
return `drop ${this.baseTableType(type)} if exists ${this.resolveTarget(target)} ${
this.baseTableType(type) == "table" ? "cascade" : ""
}`;
protected oppositeTableType(type: string) {
return this.baseTableType(type) === "table" ? "view" : "table";
}

public abstract resolveTarget(target: dataform.ITarget): string;
protected where(query: string, where: string) {
return `select * from (${query}) as subquery
where ${where}`;
}
}
1 change: 1 addition & 0 deletions core/adapters/index.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ import { dataform } from "@dataform/protos";

export interface IAdapter {
resolveTarget(target: dataform.ITarget): string;
normalizeIdentifier(identifier: string): string;

publishTasks(
table: dataform.ITable,
Expand Down
4 changes: 4 additions & 0 deletions core/adapters/snowflake.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,10 @@ export class SnowflakeAdapter extends Adapter implements IAdapter {
return `"${target.schema}"."${target.name}"`;
}

public normalizeIdentifier(identifier: string) {
return identifier.toUpperCase();
}

public publishTasks(
table: dataform.ITable,
runConfig: dataform.IRunConfig,
Expand Down
20 changes: 15 additions & 5 deletions core/session.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -212,13 +212,19 @@ export class Session {
}

public target(target: string, defaultSchema?: string): dataform.ITarget {
const adapter = this.adapter();
if (target.includes(".")) {
const [schema, name] = target.split(".");
return dataform.Target.create({ name, schema: schema + this.getSuffixWithUnderscore() });
return dataform.Target.create({
name: adapter.normalizeIdentifier(name),
schema: adapter.normalizeIdentifier(schema + this.getSuffixWithUnderscore())
});
}
return dataform.Target.create({
name: target,
schema: (defaultSchema || this.config.defaultSchema) + this.getSuffixWithUnderscore()
name: adapter.normalizeIdentifier(target),
schema: adapter.normalizeIdentifier(
(defaultSchema || this.config.defaultSchema) + this.getSuffixWithUnderscore()
)
});
}

Expand Down Expand Up @@ -421,11 +427,15 @@ export class Session {
}

public findActions(res: Resolvable) {
const adapter = this.adapter();
return this.actions.filter(action => {
if (typeof res === "string") {
return action.proto.target.name === res;
return action.proto.target.name === adapter.normalizeIdentifier(res);
}
return action.proto.target.schema === res.schema && action.proto.target.name === res.name;
return (
action.proto.target.schema === adapter.normalizeIdentifier(res.schema) &&
action.proto.target.name === adapter.normalizeIdentifier(res.name)
);
});
}

Expand Down
91 changes: 46 additions & 45 deletions tests/api/api.spec.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -147,17 +147,17 @@ describe("@dataform/api", () => {
.to.be.an("array").that.is.not.empty;

graph.tables.forEach((t: dataform.ITable) => {
const action = executedGraph.actions.find(item => item.name == t.name);
const action = executedGraph.actions.find(item => item.name === t.name);
expect(action).to.include({ type: "table", target: t.target, tableType: t.type });
});

graph.operations.forEach((o: dataform.IOperation) => {
const action = executedGraph.actions.find(item => item.name == o.name);
const action = executedGraph.actions.find(item => item.name === o.name);
expect(action).to.include({ type: "operation", target: o.target });
});

graph.assertions.forEach((a: dataform.IAssertion) => {
const action = executedGraph.actions.find(item => item.name == a.name);
const action = executedGraph.actions.find(item => item.name === a.name);
expect(action).to.include({ type: "assertion" });
});
});
Expand Down Expand Up @@ -302,9 +302,9 @@ describe("@dataform/api", () => {
]
});
const executionGraph = new Builder(graph, {}, state).build();
expect(executionGraph.actions.filter(n => n.name == "incremental")).is.not.empty;
expect(executionGraph.actions.filter(n => n.name === "incremental")).is.not.empty;
expect(
cleanSql(executionGraph.actions.filter(n => n.name == "incremental")[0].tasks[0].statement)
cleanSql(executionGraph.actions.filter(n => n.name === "incremental")[0].tasks[0].statement)
).equals(
cleanSql(
`insert into \`schema.incremental\` (existing_field)
Expand Down Expand Up @@ -537,15 +537,15 @@ describe("@dataform/api", () => {
// Check JS blocks get processed.
expect(tableNames).includes("df_integration_test.example_js_blocks");
const exampleJsBlocks = graph.tables.filter(
(t: dataform.ITable) => t.name == "df_integration_test.example_js_blocks"
(t: dataform.ITable) => t.name === "df_integration_test.example_js_blocks"
)[0];
expect(exampleJsBlocks.type).equals("table");
expect(exampleJsBlocks.query).equals("select 1 as foo");

// Check we can import and use an external package.
expect(tableNames).includes("df_integration_test.example_incremental");
const exampleIncremental = graph.tables.filter(
(t: dataform.ITable) => t.name == "df_integration_test.example_incremental"
(t: dataform.ITable) => t.name === "df_integration_test.example_incremental"
)[0];
expect(exampleIncremental.query).equals("select current_timestamp() as ts");
expect(exampleIncremental.where.trim()).equals(
Expand All @@ -558,7 +558,7 @@ describe("@dataform/api", () => {
// Check SQL files with raw back-ticks get escaped.
expect(tableNames).includes("df_integration_test.example_backticks");
const exampleBackticks = graph.tables.filter(
(t: dataform.ITable) => t.name == "df_integration_test.example_backticks"
(t: dataform.ITable) => t.name === "df_integration_test.example_backticks"
)[0];
expect(cleanSql(exampleBackticks.query)).equals(
"select * from `tada-analytics.df_integration_test.sample_data`"
Expand All @@ -567,14 +567,14 @@ describe("@dataform/api", () => {
// Check deferred calls to table resolve to the correct definitions file.
expect(tableNames).includes("df_integration_test.example_deferred");
const exampleDeferred = graph.tables.filter(
(t: dataform.ITable) => t.name == "df_integration_test.example_deferred"
(t: dataform.ITable) => t.name === "df_integration_test.example_deferred"
)[0];
expect(exampleDeferred.fileName).includes("definitions/example_deferred.js");

// Check inline tables
expect(tableNames).includes("df_integration_test.example_inline");
const exampleInline = graph.tables.filter(
(t: dataform.ITable) => t.name == "df_integration_test.example_inline"
(t: dataform.ITable) => t.name === "df_integration_test.example_inline"
)[0];
expect(exampleInline.type).equals("inline");
expect(exampleInline.query).equals(
Expand All @@ -584,7 +584,7 @@ describe("@dataform/api", () => {

expect(tableNames).includes("df_integration_test.example_using_inline");
const exampleUsingInline = graph.tables.filter(
(t: dataform.ITable) => t.name == "df_integration_test.example_using_inline"
(t: dataform.ITable) => t.name === "df_integration_test.example_using_inline"
)[0];
expect(exampleUsingInline.type).equals("table");
expect(exampleUsingInline.query).equals(
Expand All @@ -595,7 +595,7 @@ describe("@dataform/api", () => {
// Check view
expect(tableNames).includes("df_integration_test.example_view");
const exampleView = graph.tables.filter(
(t: dataform.ITable) => t.name == "df_integration_test.example_view"
(t: dataform.ITable) => t.name === "df_integration_test.example_view"
)[0];
expect(exampleView.type).equals("view");
expect(exampleView.query).equals(
Expand All @@ -606,7 +606,7 @@ describe("@dataform/api", () => {
// Check table
expect(tableNames).includes("df_integration_test.example_table");
const exampleTable = graph.tables.filter(
(t: dataform.ITable) => t.name == "df_integration_test.example_table"
(t: dataform.ITable) => t.name === "df_integration_test.example_table"
)[0];
expect(exampleTable.type).equals("table");
expect(exampleTable.query).equals(
Expand All @@ -617,7 +617,7 @@ describe("@dataform/api", () => {
// Check sample data
expect(tableNames).includes("df_integration_test.sample_data");
const exampleSampleData = graph.tables.filter(
(t: dataform.ITable) => t.name == "df_integration_test.sample_data"
(t: dataform.ITable) => t.name === "df_integration_test.sample_data"
)[0];
expect(exampleSampleData.type).equals("view");
expect(exampleSampleData.query).equals(
Expand Down Expand Up @@ -1015,22 +1015,22 @@ describe("@dataform/api", () => {
// Check we can import and use an external package.
expect(tableNames).includes("df_integration_test.example_incremental");
const exampleIncremental = graph.tables.filter(
(t: dataform.ITable) => t.name == "df_integration_test.example_incremental"
(t: dataform.ITable) => t.name === "df_integration_test.example_incremental"
)[0];
expect(exampleIncremental.query).equals("select current_timestamp::timestamp as ts");

// Check inline tables
expect(tableNames).includes("df_integration_test.example_inline");
const exampleInline = graph.tables.filter(
(t: dataform.ITable) => t.name == "df_integration_test.example_inline"
(t: dataform.ITable) => t.name === "df_integration_test.example_inline"
)[0];
expect(exampleInline.type).equals("inline");
expect(exampleInline.query).equals('\nselect * from "df_integration_test"."sample_data"');
expect(exampleInline.dependencies).includes("df_integration_test.sample_data");

expect(tableNames).includes("df_integration_test.example_using_inline");
const exampleUsingInline = graph.tables.filter(
(t: dataform.ITable) => t.name == "df_integration_test.example_using_inline"
(t: dataform.ITable) => t.name === "df_integration_test.example_using_inline"
)[0];
expect(exampleUsingInline.type).equals("table");
expect(exampleUsingInline.query).equals(
Expand Down Expand Up @@ -1097,7 +1097,7 @@ describe("@dataform/api", () => {

// Make sure it compiles.
expect(tableNames).includes("example");
const example = graph.tables.filter((t: dataform.ITable) => t.name == "example")[0];
const example = graph.tables.filter((t: dataform.ITable) => t.name === "example")[0];
expect(example.type).equals("table");
expect(example.query.trim()).equals("select 1 as foo_bar");

Expand Down Expand Up @@ -1149,35 +1149,35 @@ describe("@dataform/api", () => {

const mNames = graph.tables.map((t: dataform.ITable) => t.name);

expect(mNames).includes("df_integration_test.example_incremental");
expect(mNames).includes("DF_INTEGRATION_TEST.EXAMPLE_INCREMENTAL");
const mIncremental = graph.tables.filter(
(t: dataform.ITable) => t.name == "df_integration_test.example_incremental"
(t: dataform.ITable) => t.name === "DF_INTEGRATION_TEST.EXAMPLE_INCREMENTAL"
)[0];
expect(mIncremental.type).equals("incremental");
expect(mIncremental.query).equals(
"select convert_timezone('UTC', current_timestamp())::timestamp as ts"
);
expect(mIncremental.dependencies).to.be.an("array").that.is.empty;

expect(mNames).includes("df_integration_test.example_table");
expect(mNames).includes("DF_INTEGRATION_TEST.EXAMPLE_TABLE");
const mTable = graph.tables.filter(
(t: dataform.ITable) => t.name == "df_integration_test.example_table"
(t: dataform.ITable) => t.name === "DF_INTEGRATION_TEST.EXAMPLE_TABLE"
)[0];
expect(mTable.type).equals("table");
expect(mTable.query).equals('\nselect * from "df_integration_test"."sample_data"');
expect(mTable.dependencies).deep.equals(["df_integration_test.sample_data"]);
expect(mTable.query).equals('\nselect * from "DF_INTEGRATION_TEST"."SAMPLE_DATA"');
expect(mTable.dependencies).deep.equals(["DF_INTEGRATION_TEST.SAMPLE_DATA"]);

expect(mNames).includes("df_integration_test.example_view");
expect(mNames).includes("DF_INTEGRATION_TEST.EXAMPLE_VIEW");
const mView = graph.tables.filter(
(t: dataform.ITable) => t.name == "df_integration_test.example_view"
(t: dataform.ITable) => t.name === "DF_INTEGRATION_TEST.EXAMPLE_VIEW"
)[0];
expect(mView.type).equals("view");
expect(mView.query).equals('\nselect * from "df_integration_test"."sample_data"');
expect(mView.dependencies).deep.equals(["df_integration_test.sample_data"]);
expect(mView.query).equals('\nselect * from "DF_INTEGRATION_TEST"."SAMPLE_DATA"');
expect(mView.dependencies).deep.equals(["DF_INTEGRATION_TEST.SAMPLE_DATA"]);

expect(mNames).includes("df_integration_test.sample_data");
expect(mNames).includes("DF_INTEGRATION_TEST.SAMPLE_DATA");
const mSampleData = graph.tables.filter(
(t: dataform.ITable) => t.name == "df_integration_test.sample_data"
(t: dataform.ITable) => t.name === "DF_INTEGRATION_TEST.SAMPLE_DATA"
)[0];
expect(mSampleData.type).equals("view");
expect(mSampleData.query).equals(
Expand All @@ -1186,38 +1186,39 @@ describe("@dataform/api", () => {
expect(mSampleData.dependencies).to.be.an("array").that.is.empty;

// Check inline tables
expect(mNames).includes("df_integration_test.example_inline");
expect(mNames).includes("DF_INTEGRATION_TEST.EXAMPLE_INLINE");
const exampleInline = graph.tables.filter(
(t: dataform.ITable) => t.name == "df_integration_test.example_inline"
(t: dataform.ITable) => t.name === "DF_INTEGRATION_TEST.EXAMPLE_INLINE"
)[0];
expect(exampleInline.type).equals("inline");
expect(exampleInline.query).equals('\nselect * from "df_integration_test"."sample_data"');
expect(exampleInline.dependencies).includes("df_integration_test.sample_data");
expect(exampleInline.query).equals('\nselect * from "DF_INTEGRATION_TEST"."SAMPLE_DATA"');
expect(exampleInline.dependencies).includes("DF_INTEGRATION_TEST.SAMPLE_DATA");

expect(mNames).includes("df_integration_test.example_using_inline");
expect(mNames).includes("DF_INTEGRATION_TEST.EXAMPLE_USING_INLINE");
const exampleUsingInline = graph.tables.filter(
(t: dataform.ITable) => t.name == "df_integration_test.example_using_inline"
(t: dataform.ITable) => t.name === "DF_INTEGRATION_TEST.EXAMPLE_USING_INLINE"
)[0];
expect(exampleUsingInline.type).equals("table");
expect(exampleUsingInline.query).equals(
'\nselect * from (\nselect * from "df_integration_test"."sample_data")\nwhere true'
'\nselect * from (\nselect * from "DF_INTEGRATION_TEST"."SAMPLE_DATA")\nwhere true'
);
expect(exampleUsingInline.dependencies).includes("df_integration_test.sample_data");
expect(exampleUsingInline.dependencies).includes("DF_INTEGRATION_TEST.SAMPLE_DATA");

const aNames = graph.assertions.map((a: dataform.IAssertion) => a.name);

expect(aNames).includes("df_integration_test_assertions.sample_data_assertion");
expect(aNames).includes("DF_INTEGRATION_TEST_ASSERTIONS.SAMPLE_DATA_ASSERTION");
const assertion = graph.assertions.filter(
(a: dataform.IAssertion) => a.name == "df_integration_test_assertions.sample_data_assertion"
(a: dataform.IAssertion) =>
a.name === "DF_INTEGRATION_TEST_ASSERTIONS.SAMPLE_DATA_ASSERTION"
)[0];
expect(assertion.query).equals(
'select * from "df_integration_test"."sample_data" where sample_column > 3'
'select * from "DF_INTEGRATION_TEST"."SAMPLE_DATA" where sample_column > 3'
);
expect(assertion.dependencies).to.include.members([
"df_integration_test.sample_data",
"df_integration_test.example_table",
"df_integration_test.example_incremental",
"df_integration_test.example_view"
"DF_INTEGRATION_TEST.SAMPLE_DATA",
"DF_INTEGRATION_TEST.EXAMPLE_TABLE",
"DF_INTEGRATION_TEST.EXAMPLE_INCREMENTAL",
"DF_INTEGRATION_TEST.EXAMPLE_VIEW"
]);
});

Expand Down
Loading

0 comments on commit 965aea1

Please sign in to comment.