From c04d7a0dd89526f611956f10fc10165fdc88b4f5 Mon Sep 17 00:00:00 2001 From: Claire Carroll <20294432+clrcrl@users.noreply.github.com> Date: Thu, 3 Sep 2020 16:32:49 -0400 Subject: [PATCH 1/3] Bump dbt to v0.18.0 --- dbt_project.yml | 4 ++-- packages.yml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/dbt_project.yml b/dbt_project.yml index a1b9b7d0..a147dfe3 100644 --- a/dbt_project.yml +++ b/dbt_project.yml @@ -3,10 +3,10 @@ version: '0.3.0' config-version: 2 -require-dbt-version: ">=0.17.0" +require-dbt-version: ">=0.18.0" source-paths: ["models"] -analysis-paths: ["analysis"] +analysis-paths: ["analysis"] test-paths: ["tests"] data-paths: ["data"] macro-paths: ["macros"] diff --git a/packages.yml b/packages.yml index ecb70730..2246ac8b 100644 --- a/packages.yml +++ b/packages.yml @@ -1,3 +1,3 @@ packages: - package: fishtown-analytics/dbt_utils - version: [">=0.4.0", "<0.6.0"] + version: [">=0.6.0", "<0.7.0"] From 793efb47d798163fcd000984129d2a2754da912e Mon Sep 17 00:00:00 2001 From: Claire Carroll <20294432+clrcrl@users.noreply.github.com> Date: Thu, 3 Sep 2020 16:35:33 -0400 Subject: [PATCH 2/3] Use dispatch macro --- macros/external/create_external_table.sql | 10 +++++----- macros/helpers/get_external_tables_namespace.sql | 4 ++++ 2 files changed, 9 insertions(+), 5 deletions(-) create mode 100644 macros/helpers/get_external_tables_namespace.sql diff --git a/macros/external/create_external_table.sql b/macros/external/create_external_table.sql index 2e10cb9f..df4e5018 100644 --- a/macros/external/create_external_table.sql +++ b/macros/external/create_external_table.sql @@ -1,5 +1,5 @@ {% macro create_external_table(source_node) %} - {{ adapter_macro('dbt_external_tables.create_external_table', source_node) }} + {{ adapter.dispatch('create_external_table', dbt_external_tables._get_dbt_external_tables_namespaces()) (source_node) }} {% endmacro %} {% macro default__create_external_table(source_node) %} @@ -30,7 +30,7 @@ {% if external.file_format -%} stored as {{external.file_format}} {%- endif %} {% if external.location -%} location '{{external.location}}' {%- endif %} {% if external.table_properties -%} table properties {{external.table_properties}} {%- endif %} - + {% endmacro %} {% macro spark__create_external_table(source_node) %} @@ -62,7 +62,7 @@ {%- set columns = source_node.columns.values() -%} {%- set external = source_node.external -%} {%- set partitions = external.partitions -%} - + {%- set is_csv = dbt_external_tables.is_csv(external.file_format) -%} {# https://docs.snowflake.net/manuals/sql-reference/sql/create-external-table.html #} @@ -93,12 +93,12 @@ {% macro bigquery__create_external_table(source_node) %} {{ exceptions.raise_compiler_error( - "BigQuery does not support creating external tables in SQL/DDL. + "BigQuery does not support creating external tables in SQL/DDL. Create it from the BQ console.") }} {% endmacro %} {% macro presto__create_external_table(source_node) %} {{ exceptions.raise_compiler_error( - "Presto does not support creating external tables with + "Presto does not support creating external tables with the Hive connector. Do so from Hive directly.") }} {% endmacro %} diff --git a/macros/helpers/get_external_tables_namespace.sql b/macros/helpers/get_external_tables_namespace.sql new file mode 100644 index 00000000..4dd60ca1 --- /dev/null +++ b/macros/helpers/get_external_tables_namespace.sql @@ -0,0 +1,4 @@ +{% macro _get_dbt_external_tables_namespaces() %} + {% set override_namespaces = var('dbt_external_tables_dispatch_list', []) %} + {% do return(override_namespaces + ['dbt_external_tables']) %} +{% endmacro %} From 90a58ea8aa2630f47a19bd24a91f4b656b996b72 Mon Sep 17 00:00:00 2001 From: Jeremy Cohen Date: Thu, 3 Sep 2020 18:18:43 -0400 Subject: [PATCH 3/3] Updates from local testing --- macros/external/create_external_table.sql | 4 +++- macros/external/refresh_external_table.sql | 4 +++- macros/external/stage_external_sources.sql | 9 +++++---- macros/helpers/common.sql | 2 +- macros/helpers/redshift/transaction.sql | 6 +++--- 5 files changed, 15 insertions(+), 10 deletions(-) diff --git a/macros/external/create_external_table.sql b/macros/external/create_external_table.sql index df4e5018..aa08e4cb 100644 --- a/macros/external/create_external_table.sql +++ b/macros/external/create_external_table.sql @@ -1,5 +1,7 @@ {% macro create_external_table(source_node) %} - {{ adapter.dispatch('create_external_table', dbt_external_tables._get_dbt_external_tables_namespaces()) (source_node) }} + {{ adapter.dispatch('create_external_table', + packages = dbt_external_tables._get_dbt_external_tables_namespaces()) + (source_node) }} {% endmacro %} {% macro default__create_external_table(source_node) %} diff --git a/macros/external/refresh_external_table.sql b/macros/external/refresh_external_table.sql index cdb0cb57..7d8c214d 100644 --- a/macros/external/refresh_external_table.sql +++ b/macros/external/refresh_external_table.sql @@ -1,5 +1,7 @@ {% macro refresh_external_table(source_node) %} - {{ return(adapter_macro('dbt_external_tables.refresh_external_table', source_node)) }} + {{ return(adapter.dispatch('refresh_external_table', + packages = dbt_external_tables._get_dbt_external_tables_namespaces()) + (source_node)) }} {% endmacro %} {% macro default__refresh_external_table(source_node) %} diff --git a/macros/external/stage_external_sources.sql b/macros/external/stage_external_sources.sql index 9ba215b2..1f24eb16 100644 --- a/macros/external/stage_external_sources.sql +++ b/macros/external/stage_external_sources.sql @@ -1,5 +1,7 @@ {% macro get_external_build_plan(source_node) %} - {{ return(adapter_macro('dbt_external_tables.get_external_build_plan', source_node)) }} + {{ return(adapter.dispatch('get_external_build_plan', + packages = dbt_external_tables._get_dbt_external_tables_namespaces()) + (source_node)) }} {% endmacro %} {% macro default__get_external_build_plan(source_node) %} @@ -116,17 +118,16 @@ {% do dbt_utils.log_info(loop_label ~ ' SKIP') if run_queue == [] %} - {% do dbt_external_tables.exit_transaction() %} - {% for q in run_queue %} {% set q_msg = q|trim %} {% set q_log = q_msg[:50] ~ '... ' if q_msg|length > 50 else q_msg %} {% do dbt_utils.log_info(loop_label ~ ' (' ~ loop.index ~ ') ' ~ q_log) %} + {% set exit_txn = dbt_external_tables.exit_transaction() %} {% call statement('runner', fetch_result = True, auto_begin = False) %} - {{ q }} + {{ exit_txn }} {{ q }} {% endcall %} {% set status = load_result('runner')['status'] %} diff --git a/macros/helpers/common.sql b/macros/helpers/common.sql index 99d2e2eb..3df303fe 100644 --- a/macros/helpers/common.sql +++ b/macros/helpers/common.sql @@ -12,7 +12,7 @@ {% set package_context = context[package_name] %} {% else %} {% set error_msg %} - In adapter_macro: could not find package '{{package_name}}', called with '{{original_name}}' + Could not find package '{{package_name}}', called by macro '{{original_name}}' {% endset %} {{ exceptions.raise_compiler_error(error_msg | trim) }} {% endif %} diff --git a/macros/helpers/redshift/transaction.sql b/macros/helpers/redshift/transaction.sql index 5e7bb472..b41550a9 100644 --- a/macros/helpers/redshift/transaction.sql +++ b/macros/helpers/redshift/transaction.sql @@ -1,11 +1,11 @@ {% macro exit_transaction() %} - {{ return(adapter_macro('dbt_external_tables.exit_transaction')) }} + {{ return(adapter.dispatch('exit_transaction', dbt_external_tables._get_dbt_external_tables_namespaces())()) }} {% endmacro %} {% macro default__exit_transaction() %} - {# noop #} + {{ return('') }} {% endmacro %} {% macro redshift__exit_transaction() %} - {% do run_query('begin; commit;') %} + {{ return('begin; commit;') }} {% endmacro %}