From 9c2d57167ed5de9d3020d5c00c03b8be65f74771 Mon Sep 17 00:00:00 2001 From: BenBirt Date: Wed, 26 Jun 2019 14:29:14 +0100 Subject: [PATCH] Make schema suffixes work correctly for files with explicitly overridden schemas. (#281) * Make schema suffixes work correctly for files with explicitly overridden schemas. * cleanup * bump df version --- core/session.ts | 13 +- tests/api/api.spec.ts | 365 +++++++++++++++++++++++------------------- version.bzl | 2 +- 3 files changed, 205 insertions(+), 175 deletions(-) diff --git a/core/session.ts b/core/session.ts index 79e974d5f..8ec23398c 100644 --- a/core/session.ts +++ b/core/session.ts @@ -129,14 +129,11 @@ export class Session { } } })(); - if (actionOptions.sqlxConfig.schema) { - action.proto.target.schema = actionOptions.sqlxConfig.schema; - } - if (actionOptions.sqlxConfig.name) { - action.proto.name = actionOptions.sqlxConfig.name; - if (action.proto.target) { - action.proto.target.name = actionOptions.sqlxConfig.name; - } + if (action.proto.target) { + action.proto.target = this.target( + actionOptions.sqlxConfig.name, + actionOptions.sqlxConfig.schema + ); } return action; } diff --git a/tests/api/api.spec.ts b/tests/api/api.spec.ts index 91b6d602c..62d1bbaa1 100644 --- a/tests/api/api.spec.ts +++ b/tests/api/api.spec.ts @@ -532,183 +532,216 @@ describe("@dataform/api", () => { expect(exampleSampleData.dependencies).to.eql([]); }); - it("bigquery using v2 language compiles", async () => { - const graph = await compile({ projectDir: path.resolve("df/examples/bigquery_language_v2") }); - - expect(graph.graphErrors).to.eql( - dataform.GraphErrors.create({ - compilationErrors: [ - dataform.CompilationError.create({ - fileName: "definitions/has_compile_errors/assertion_with_bigquery.sqlx", - message: "Actions may only specify 'bigquery: { ... }' if they create a dataset." - }), - dataform.CompilationError.create({ - fileName: "definitions/has_compile_errors/assertion_with_output.sqlx", - message: - "Actions may only specify 'hasOutput: true' if they are of type 'operations' or create a dataset." - }), - dataform.CompilationError.create({ - fileName: "definitions/has_compile_errors/assertion_with_postops.sqlx", - message: "Actions may only include post_operations if they create a dataset." - }), - dataform.CompilationError.create({ - fileName: "definitions/has_compile_errors/assertion_with_preops.sqlx", - message: "Actions may only include pre_operations if they create a dataset." - }), - dataform.CompilationError.create({ - fileName: "definitions/has_compile_errors/assertion_with_redshift.sqlx", - message: "Actions may only specify 'redshift: { ... }' if they create a dataset." - }), - dataform.CompilationError.create({ - fileName: "definitions/has_compile_errors/disabled_assertion.sqlx", - message: "Actions may only specify 'disabled: true' if they create a dataset." - }), - dataform.CompilationError.create({ - fileName: "definitions/has_compile_errors/op_with_output_multiple_statements.sqlx", - message: "Operations with 'hasOutput: true' must contain exactly one SQL statement." - }), - dataform.CompilationError.create({ - fileName: "definitions/has_compile_errors/protected_assertion.sqlx", - message: - "Actions may only specify 'protected: true' if they are of type 'incremental'." - }), - dataform.CompilationError.create({ - fileName: "definitions/has_compile_errors/view_with_incremental.sqlx", - message: - "Actions may only include incremental_where if they are of type 'incremental'." - }), - dataform.CompilationError.create({ - fileName: "definitions/has_compile_errors/view_with_multiple_statements.sqlx", - message: - "Actions may only contain more than one SQL statement if they are of type 'operations'." - }) - ] - }) - ); - - // Check JS blocks get processed. - const exampleJsBlocks = graph.tables.find(t => t.name === "example_js_blocks"); - expect(exampleJsBlocks).to.not.be.undefined; - expect(exampleJsBlocks.type).equals("table"); - expect(exampleJsBlocks.query.trim()).equals("select 1 as foo"); + for (const schemaSuffix of ["", "suffix"]) { + const schemaWithSuffix = (schema: string) => + schemaSuffix ? `${schema}_${schemaSuffix}` : schema; + + it(`bigquery using v2 language compiles with suffix "${schemaSuffix}"`, async () => { + const graph = await compile({ + projectDir: path.resolve("df/examples/bigquery_language_v2"), + schemaSuffixOverride: schemaSuffix + }); - // Check we can import and use an external package. - const exampleIncremental = graph.tables.find(t => t.name === "example_incremental"); - expect(exampleIncremental).to.not.be.undefined; - expect(exampleIncremental.query.trim()).equals("select current_timestamp() as ts"); - expect(exampleIncremental.where.trim()).equals( - "ts > (select max(ts) from `tada-analytics.df_integration_test.example_incremental`) or (select max(ts) from `tada-analytics.df_integration_test.example_incremental`) is null" - ); + expect(graph.graphErrors).to.eql( + dataform.GraphErrors.create({ + compilationErrors: [ + dataform.CompilationError.create({ + fileName: "definitions/has_compile_errors/assertion_with_bigquery.sqlx", + message: "Actions may only specify 'bigquery: { ... }' if they create a dataset." + }), + dataform.CompilationError.create({ + fileName: "definitions/has_compile_errors/assertion_with_output.sqlx", + message: + "Actions may only specify 'hasOutput: true' if they are of type 'operations' or create a dataset." + }), + dataform.CompilationError.create({ + fileName: "definitions/has_compile_errors/assertion_with_postops.sqlx", + message: "Actions may only include post_operations if they create a dataset." + }), + dataform.CompilationError.create({ + fileName: "definitions/has_compile_errors/assertion_with_preops.sqlx", + message: "Actions may only include pre_operations if they create a dataset." + }), + dataform.CompilationError.create({ + fileName: "definitions/has_compile_errors/assertion_with_redshift.sqlx", + message: "Actions may only specify 'redshift: { ... }' if they create a dataset." + }), + dataform.CompilationError.create({ + fileName: "definitions/has_compile_errors/disabled_assertion.sqlx", + message: "Actions may only specify 'disabled: true' if they create a dataset." + }), + dataform.CompilationError.create({ + fileName: "definitions/has_compile_errors/op_with_output_multiple_statements.sqlx", + message: "Operations with 'hasOutput: true' must contain exactly one SQL statement." + }), + dataform.CompilationError.create({ + fileName: "definitions/has_compile_errors/protected_assertion.sqlx", + message: + "Actions may only specify 'protected: true' if they are of type 'incremental'." + }), + dataform.CompilationError.create({ + fileName: "definitions/has_compile_errors/view_with_incremental.sqlx", + message: + "Actions may only include incremental_where if they are of type 'incremental'." + }), + dataform.CompilationError.create({ + fileName: "definitions/has_compile_errors/view_with_multiple_statements.sqlx", + message: + "Actions may only contain more than one SQL statement if they are of type 'operations'." + }) + ] + }) + ); - // Check tables defined in includes are not included. - const exampleIgnore = graph.tables.find(t => t.name === "example_ignore"); - expect(exampleIgnore).to.be.undefined; + // Check JS blocks get processed. + const exampleJsBlocks = graph.tables.find(t => t.name === "example_js_blocks"); + expect(exampleJsBlocks).to.not.be.undefined; + expect(exampleJsBlocks.type).equals("table"); + expect(exampleJsBlocks.query.trim()).equals("select 1 as foo"); - // Check SQL files with raw back-ticks get escaped. - const exampleBackticks = graph.tables.find(t => t.name === "example_backticks"); - expect(exampleBackticks).to.not.be.undefined; - expect(cleanSql(exampleBackticks.query)).equals( - "select * from `tada-analytics.df_integration_test.sample_data`" - ); - expect(exampleBackticks.preOps).to.eql([ - '\n GRANT SELECT ON `tada-analytics.df_integration_test.sample_data` TO GROUP "allusers@dataform.co"\n' - ]); - expect(exampleBackticks.postOps).to.eql([]); + // Check we can import and use an external package. + const exampleIncremental = graph.tables.find(t => t.name === "example_incremental"); + expect(exampleIncremental).to.not.be.undefined; + expect(exampleIncremental.query.trim()).equals("select current_timestamp() as ts"); + expect(exampleIncremental.where.trim()).equals( + `ts > (select max(ts) from \`tada-analytics.${schemaWithSuffix( + "df_integration_test" + )}.example_incremental\`) or (select max(ts) from \`tada-analytics.${schemaWithSuffix( + "df_integration_test" + )}.example_incremental\`) is null` + ); - // Check deferred calls to table resolve to the correct definitions file. - const exampleDeferred = graph.tables.find(t => t.name === "example_deferred"); - expect(exampleDeferred).to.not.be.undefined; - expect(exampleDeferred.fileName).includes("definitions/example_deferred.js"); + // Check tables defined in includes are not included. + const exampleIgnore = graph.tables.find(t => t.name === "example_ignore"); + expect(exampleIgnore).to.be.undefined; - // Check inline tables - const exampleInline = graph.tables.find(t => t.name === "example_inline"); - expect(exampleInline).to.not.be.undefined; - expect(exampleInline.type).equals("inline"); - expect(exampleInline.query.trim()).equals( - "select * from `tada-analytics.df_integration_test.sample_data`" - ); - expect(exampleInline.dependencies).includes("sample_data"); + // Check SQL files with raw back-ticks get escaped. + const exampleBackticks = graph.tables.find(t => t.name === "example_backticks"); + expect(exampleBackticks).to.not.be.undefined; + expect(cleanSql(exampleBackticks.query)).equals( + "select * from `tada-analytics.df_integration_test.sample_data`" + ); + expect(exampleBackticks.preOps).to.eql([ + '\n GRANT SELECT ON `tada-analytics.df_integration_test.sample_data` TO GROUP "allusers@dataform.co"\n' + ]); + expect(exampleBackticks.postOps).to.eql([]); - const exampleUsingInline = graph.tables.find(t => t.name === "example_using_inline"); - expect(exampleUsingInline).to.not.be.undefined; - expect(exampleUsingInline.type).equals("table"); - expect(exampleUsingInline.query.trim()).equals( - "select * from (\n\nselect * from `tada-analytics.df_integration_test.sample_data`\n)\nwhere true" - ); - expect(exampleUsingInline.dependencies).includes("sample_data"); + // Check deferred calls to table resolve to the correct definitions file. + const exampleDeferred = graph.tables.find(t => t.name === "example_deferred"); + expect(exampleDeferred).to.not.be.undefined; + expect(exampleDeferred.fileName).includes("definitions/example_deferred.js"); - // Check view - const exampleView = graph.tables.find(t => t.name === "example_view"); - expect(exampleView).to.not.be.undefined; - expect(exampleView.type).equals("view"); - expect(exampleView.query.trim()).equals( - "select * from `tada-analytics.df_integration_test.sample_data`" - ); - expect(exampleView.dependencies).deep.equals(["sample_data"]); + // Check inline tables + const exampleInline = graph.tables.find(t => t.name === "example_inline"); + expect(exampleInline).to.not.be.undefined; + expect(exampleInline.type).equals("inline"); + expect(exampleInline.query.trim()).equals( + `select * from \`tada-analytics.${schemaWithSuffix("df_integration_test")}.sample_data\`` + ); + expect(exampleInline.dependencies).includes("sample_data"); - // Check table - const exampleTable = graph.tables.find(t => t.name === "example_table"); - expect(exampleTable).to.not.be.undefined; - expect(exampleTable.type).equals("table"); - expect(exampleTable.query.trim()).equals( - "select * from `tada-analytics.df_integration_test.sample_data`" - ); - expect(exampleTable.dependencies).deep.equals(["sample_data"]); - expect(exampleTable.preOps).to.eql([]); - expect(exampleTable.postOps).to.eql([ - '\n GRANT SELECT ON `tada-analytics.df_integration_test.example_table` TO GROUP "allusers@dataform.co"\n', - '\n GRANT SELECT ON `tada-analytics.df_integration_test.example_table` TO GROUP "otherusers@dataform.co"\n' - ]); + const exampleUsingInline = graph.tables.find(t => t.name === "example_using_inline"); + expect(exampleUsingInline).to.not.be.undefined; + expect(exampleUsingInline.type).equals("table"); + expect(exampleUsingInline.query.trim()).equals( + `select * from (\n\nselect * from \`tada-analytics.${schemaWithSuffix( + "df_integration_test" + )}.sample_data\`\n)\nwhere true` + ); + expect(exampleUsingInline.dependencies).includes("sample_data"); - // Check sample data - const exampleSampleData = graph.tables.find(t => t.name === "sample_data"); - expect(exampleSampleData).to.not.be.undefined; - expect(exampleSampleData.type).equals("view"); - expect(exampleSampleData.query.trim()).equals( - "select 1 as sample union all\nselect 2 as sample union all\nselect 3 as sample" - ); - expect(exampleSampleData.dependencies).to.eql([]); + // Check view + const exampleView = graph.tables.find(t => t.name === "example_view"); + expect(exampleView).to.not.be.undefined; + expect(exampleView.type).equals("view"); + expect(exampleView.query.trim()).equals( + `select * from \`tada-analytics.${schemaWithSuffix("df_integration_test")}.sample_data\`` + ); + expect(exampleView.dependencies).deep.equals(["sample_data"]); + + // Check table + const exampleTable = graph.tables.find(t => t.name === "example_table"); + expect(exampleTable).to.not.be.undefined; + expect(exampleTable.type).equals("table"); + expect(exampleTable.query.trim()).equals( + `select * from \`tada-analytics.${schemaWithSuffix("df_integration_test")}.sample_data\`` + ); + expect(exampleTable.dependencies).deep.equals(["sample_data"]); + expect(exampleTable.preOps).to.eql([]); + expect(exampleTable.postOps).to.eql([ + `\n GRANT SELECT ON \`tada-analytics.${schemaWithSuffix( + "df_integration_test" + )}.example_table\` TO GROUP "allusers@dataform.co"\n`, + `\n GRANT SELECT ON \`tada-analytics.${schemaWithSuffix( + "df_integration_test" + )}.example_table\` TO GROUP "otherusers@dataform.co"\n` + ]); + + // Check sample data + const exampleSampleData = graph.tables.find(t => t.name === "sample_data"); + expect(exampleSampleData).to.not.be.undefined; + expect(exampleSampleData.type).equals("view"); + expect(exampleSampleData.query.trim()).equals( + "select 1 as sample union all\nselect 2 as sample union all\nselect 3 as sample" + ); + expect(exampleSampleData.dependencies).to.eql([]); - // Check schema overrides defined in "config {}" - const exampleUsingOverriddenSchema = graph.tables.find( - t => t.name === "override_schema_example" - ); - expect(exampleUsingOverriddenSchema).to.not.be.undefined; - expect(exampleUsingOverriddenSchema.target.schema).equals("override_schema"); - expect(exampleUsingOverriddenSchema.type).equals("view"); - expect(exampleUsingOverriddenSchema.query.trim()).equals("select 1 as test_schema_override"); - - // Check assertion - const exampleAssertion = graph.assertions.find(t => t.name === "example_assertion"); - expect(exampleAssertion).to.not.be.undefined; - expect(exampleAssertion.target.schema).equals("hi_there"); - expect(exampleAssertion.query.trim()).equals( - "select * from `tada-analytics.df_integration_test.sample_data` where sample = 100" - ); - expect(exampleAssertion.dependencies).to.eql(["sample_data"]); - - // Check example operations file - const exampleOperations = graph.operations.find(o => o.name === "example_operations"); - expect(exampleOperations).to.not.be.undefined; - expect(exampleOperations.target).is.null; - expect(exampleOperations.queries).to.eql([ - "\n\nCREATE OR REPLACE VIEW someschema.someview AS (SELECT 1 AS test)\n", - "\nDROP VIEW IF EXISTS `tada-analytics.override_schema.override_schema_example`\n" - ]); - expect(exampleOperations.dependencies).to.eql(["example_inline", "override_schema_example"]); + // Check schema overrides defined in "config {}" + const exampleUsingOverriddenSchema = graph.tables.find( + t => t.name === "override_schema_example" + ); + expect(exampleUsingOverriddenSchema).to.not.be.undefined; + expect(exampleUsingOverriddenSchema.target.schema).equals( + schemaWithSuffix("override_schema") + ); + expect(exampleUsingOverriddenSchema.type).equals("view"); + expect(exampleUsingOverriddenSchema.query.trim()).equals( + "select 1 as test_schema_override" + ); - // Check example operation with output. - const exampleOperationWithOutput = graph.operations.find( - o => o.name === "example_operation_with_output" - ); - expect(exampleOperationWithOutput).to.not.be.undefined; - expect(exampleOperationWithOutput.target.schema).equals("df_integration_test"); - expect(exampleOperationWithOutput.target.name).equals("example_operation_with_output"); - expect(exampleOperationWithOutput.queries).to.eql([ - "\nCREATE OR REPLACE VIEW `tada-analytics.df_integration_test.example_operation_with_output` AS (SELECT 1 AS TEST)" - ]); - expect(exampleOperationWithOutput.dependencies).to.eql([]); - }); + // Check assertion + const exampleAssertion = graph.assertions.find(t => t.name === "example_assertion"); + expect(exampleAssertion).to.not.be.undefined; + expect(exampleAssertion.target.schema).equals(schemaWithSuffix("hi_there")); + expect(exampleAssertion.query.trim()).equals( + `select * from \`tada-analytics.${schemaWithSuffix( + "df_integration_test" + )}.sample_data\` where sample = 100` + ); + expect(exampleAssertion.dependencies).to.eql(["sample_data"]); + + // Check example operations file + const exampleOperations = graph.operations.find(o => o.name === "example_operations"); + expect(exampleOperations).to.not.be.undefined; + expect(exampleOperations.target).is.null; + expect(exampleOperations.queries).to.eql([ + "\n\nCREATE OR REPLACE VIEW someschema.someview AS (SELECT 1 AS test)\n", + `\nDROP VIEW IF EXISTS \`tada-analytics.${schemaWithSuffix( + "override_schema" + )}.override_schema_example\`\n` + ]); + expect(exampleOperations.dependencies).to.eql([ + "example_inline", + "override_schema_example" + ]); + + // Check example operation with output. + const exampleOperationWithOutput = graph.operations.find( + o => o.name === "example_operation_with_output" + ); + expect(exampleOperationWithOutput).to.not.be.undefined; + expect(exampleOperationWithOutput.target.schema).equals( + schemaWithSuffix("df_integration_test") + ); + expect(exampleOperationWithOutput.target.name).equals("example_operation_with_output"); + expect(exampleOperationWithOutput.queries).to.eql([ + `\nCREATE OR REPLACE VIEW \`tada-analytics.${schemaWithSuffix( + "df_integration_test" + )}.example_operation_with_output\` AS (SELECT 1 AS TEST)` + ]); + expect(exampleOperationWithOutput.dependencies).to.eql([]); + }); + } it("schema overrides", async () => { const graph = await compile({ diff --git a/version.bzl b/version.bzl index 895a29ae2..28c61a795 100644 --- a/version.bzl +++ b/version.bzl @@ -1 +1 @@ -DF_VERSION = "1.0.0-alpha.4" +DF_VERSION = "1.0.0-alpha.5"