From 90a58ea8aa2630f47a19bd24a91f4b656b996b72 Mon Sep 17 00:00:00 2001 From: Jeremy Cohen Date: Thu, 3 Sep 2020 18:18:43 -0400 Subject: [PATCH] Updates from local testing --- macros/external/create_external_table.sql | 4 +++- macros/external/refresh_external_table.sql | 4 +++- macros/external/stage_external_sources.sql | 9 +++++---- macros/helpers/common.sql | 2 +- macros/helpers/redshift/transaction.sql | 6 +++--- 5 files changed, 15 insertions(+), 10 deletions(-) diff --git a/macros/external/create_external_table.sql b/macros/external/create_external_table.sql index df4e5018..aa08e4cb 100644 --- a/macros/external/create_external_table.sql +++ b/macros/external/create_external_table.sql @@ -1,5 +1,7 @@ {% macro create_external_table(source_node) %} - {{ adapter.dispatch('create_external_table', dbt_external_tables._get_dbt_external_tables_namespaces()) (source_node) }} + {{ adapter.dispatch('create_external_table', + packages = dbt_external_tables._get_dbt_external_tables_namespaces()) + (source_node) }} {% endmacro %} {% macro default__create_external_table(source_node) %} diff --git a/macros/external/refresh_external_table.sql b/macros/external/refresh_external_table.sql index cdb0cb57..7d8c214d 100644 --- a/macros/external/refresh_external_table.sql +++ b/macros/external/refresh_external_table.sql @@ -1,5 +1,7 @@ {% macro refresh_external_table(source_node) %} - {{ return(adapter_macro('dbt_external_tables.refresh_external_table', source_node)) }} + {{ return(adapter.dispatch('refresh_external_table', + packages = dbt_external_tables._get_dbt_external_tables_namespaces()) + (source_node)) }} {% endmacro %} {% macro default__refresh_external_table(source_node) %} diff --git a/macros/external/stage_external_sources.sql b/macros/external/stage_external_sources.sql index 9ba215b2..1f24eb16 100644 --- a/macros/external/stage_external_sources.sql +++ b/macros/external/stage_external_sources.sql @@ -1,5 +1,7 @@ {% macro get_external_build_plan(source_node) %} - {{ return(adapter_macro('dbt_external_tables.get_external_build_plan', source_node)) }} + {{ return(adapter.dispatch('get_external_build_plan', + packages = dbt_external_tables._get_dbt_external_tables_namespaces()) + (source_node)) }} {% endmacro %} {% macro default__get_external_build_plan(source_node) %} @@ -116,17 +118,16 @@ {% do dbt_utils.log_info(loop_label ~ ' SKIP') if run_queue == [] %} - {% do dbt_external_tables.exit_transaction() %} - {% for q in run_queue %} {% set q_msg = q|trim %} {% set q_log = q_msg[:50] ~ '... ' if q_msg|length > 50 else q_msg %} {% do dbt_utils.log_info(loop_label ~ ' (' ~ loop.index ~ ') ' ~ q_log) %} + {% set exit_txn = dbt_external_tables.exit_transaction() %} {% call statement('runner', fetch_result = True, auto_begin = False) %} - {{ q }} + {{ exit_txn }} {{ q }} {% endcall %} {% set status = load_result('runner')['status'] %} diff --git a/macros/helpers/common.sql b/macros/helpers/common.sql index 99d2e2eb..3df303fe 100644 --- a/macros/helpers/common.sql +++ b/macros/helpers/common.sql @@ -12,7 +12,7 @@ {% set package_context = context[package_name] %} {% else %} {% set error_msg %} - In adapter_macro: could not find package '{{package_name}}', called with '{{original_name}}' + Could not find package '{{package_name}}', called by macro '{{original_name}}' {% endset %} {{ exceptions.raise_compiler_error(error_msg | trim) }} {% endif %} diff --git a/macros/helpers/redshift/transaction.sql b/macros/helpers/redshift/transaction.sql index 5e7bb472..b41550a9 100644 --- a/macros/helpers/redshift/transaction.sql +++ b/macros/helpers/redshift/transaction.sql @@ -1,11 +1,11 @@ {% macro exit_transaction() %} - {{ return(adapter_macro('dbt_external_tables.exit_transaction')) }} + {{ return(adapter.dispatch('exit_transaction', dbt_external_tables._get_dbt_external_tables_namespaces())()) }} {% endmacro %} {% macro default__exit_transaction() %} - {# noop #} + {{ return('') }} {% endmacro %} {% macro redshift__exit_transaction() %} - {% do run_query('begin; commit;') %} + {{ return('begin; commit;') }} {% endmacro %}