diff --git a/.changes/unreleased/Dependencies-20230817-121652.yaml b/.changes/unreleased/Dependencies-20230817-121652.yaml new file mode 100644 index 000000000..9f7440852 --- /dev/null +++ b/.changes/unreleased/Dependencies-20230817-121652.yaml @@ -0,0 +1,6 @@ +kind: "Dependencies" +body: "Bump mypy from 1.5.0 to 1.5.1" +time: 2023-08-17T12:16:52.00000Z +custom: + Author: dependabot[bot] + PR: 749 diff --git a/.changes/unreleased/Dependencies-20230830-125448.yaml b/.changes/unreleased/Dependencies-20230830-125448.yaml new file mode 100644 index 000000000..a0ca197ba --- /dev/null +++ b/.changes/unreleased/Dependencies-20230830-125448.yaml @@ -0,0 +1,6 @@ +kind: "Dependencies" +body: "Update tox requirement from ~=4.10 to ~=4.11" +time: 2023-08-30T12:54:48.00000Z +custom: + Author: dependabot[bot] + PR: 759 diff --git a/.changes/unreleased/Fixes-20230828-143834.yaml b/.changes/unreleased/Fixes-20230828-143834.yaml new file mode 100644 index 000000000..793b2b65c --- /dev/null +++ b/.changes/unreleased/Fixes-20230828-143834.yaml @@ -0,0 +1,6 @@ +kind: Fixes +body: remove senesitive creds from dbt debug stdout +time: 2023-08-28T14:38:34.380646-05:00 +custom: + Author: McKnight-42 + Issue: "754" diff --git a/.changes/unreleased/Under the Hood-20230821-230921.yaml b/.changes/unreleased/Under the Hood-20230821-230921.yaml new file mode 100644 index 000000000..b19517caf --- /dev/null +++ b/.changes/unreleased/Under the Hood-20230821-230921.yaml @@ -0,0 +1,7 @@ +kind: Under the Hood +body: Restructure macros files - move relation ddl statements into separate files + and directories +time: 2023-08-21T23:09:21.317975-04:00 +custom: + Author: mikealfare + Issue: "750" diff --git a/dbt/adapters/snowflake/connections.py b/dbt/adapters/snowflake/connections.py index 3272eee55..be3477b69 100644 --- a/dbt/adapters/snowflake/connections.py +++ b/dbt/adapters/snowflake/connections.py @@ -103,6 +103,8 @@ def type(self): def unique_field(self): return self.account + # the results show up in the output of dbt debug runs, for more see.. + # https://docs.getdbt.com/guides/dbt-ecosystem/adapter-development/3-building-a-new-adapter#editing-the-connection-manager def _connection_keys(self): return ( "account", @@ -112,9 +114,7 @@ def _connection_keys(self): "role", "schema", "authenticator", - "private_key", "private_key_path", - "private_key_passphrase", "token", "oauth_client_id", "query_tag", diff --git a/dbt/include/snowflake/macros/adapters.sql b/dbt/include/snowflake/macros/adapters.sql index 6310f2fbf..c4afd525b 100644 --- a/dbt/include/snowflake/macros/adapters.sql +++ b/dbt/include/snowflake/macros/adapters.sql @@ -1,58 +1,3 @@ -{% macro snowflake__create_table_as(temporary, relation, compiled_code, language='sql') -%} - {%- if language == 'sql' -%} - {%- set transient = config.get('transient', default=true) -%} - {%- set cluster_by_keys = config.get('cluster_by', default=none) -%} - {%- set enable_automatic_clustering = config.get('automatic_clustering', default=false) -%} - {%- set copy_grants = config.get('copy_grants', default=false) -%} - - {%- if cluster_by_keys is not none and cluster_by_keys is string -%} - {%- set cluster_by_keys = [cluster_by_keys] -%} - {%- endif -%} - {%- if cluster_by_keys is not none -%} - {%- set cluster_by_string = cluster_by_keys|join(", ")-%} - {% else %} - {%- set cluster_by_string = none -%} - {%- endif -%} - {%- set sql_header = config.get('sql_header', none) -%} - - {{ sql_header if sql_header is not none }} - - create or replace {% if temporary -%} - temporary - {%- elif transient -%} - transient - {%- endif %} table {{ relation }} - {%- set contract_config = config.get('contract') -%} - {%- if contract_config.enforced -%} - {{ get_assert_columns_equivalent(sql) }} - {{ get_table_columns_and_constraints() }} - {% set compiled_code = get_select_subquery(compiled_code) %} - {% endif %} - {% if copy_grants and not temporary -%} copy grants {%- endif %} as - ( - {%- if cluster_by_string is not none -%} - select * from ( - {{ compiled_code }} - ) order by ({{ cluster_by_string }}) - {%- else -%} - {{ compiled_code }} - {%- endif %} - ); - {% if cluster_by_string is not none and not temporary -%} - alter table {{relation}} cluster by ({{cluster_by_string}}); - {%- endif -%} - {% if enable_automatic_clustering and cluster_by_string is not none and not temporary -%} - alter table {{relation}} resume recluster; - {%- endif -%} - - {%- elif language == 'python' -%} - {{ py_write_table(compiled_code=compiled_code, target_relation=relation, temporary=temporary) }} - {%- else -%} - {% do exceptions.raise_compiler_error("snowflake__create_table_as macro didn't get supported language, it got %s" % language) %} - {%- endif -%} - -{% endmacro %} - {% macro get_column_comment_sql(column_name, column_dict) -%} {% if (column_name|upper in column_dict) -%} {% set matched_column = column_name|upper -%} @@ -79,35 +24,6 @@ ) {% endmacro %} -{% macro snowflake__create_view_as_with_temp_flag(relation, sql, is_temporary=False) -%} - {%- set secure = config.get('secure', default=false) -%} - {%- set copy_grants = config.get('copy_grants', default=false) -%} - {%- set sql_header = config.get('sql_header', none) -%} - - {{ sql_header if sql_header is not none }} - create or replace {% if secure -%} - secure - {%- endif %} {% if is_temporary -%} - temporary - {%- endif %} view {{ relation }} - {% if config.persist_column_docs() -%} - {% set model_columns = model.columns %} - {% set query_columns = get_columns_in_query(sql) %} - {{ get_persist_docs_column_list(model_columns, query_columns) }} - - {%- endif %} - {%- set contract_config = config.get('contract') -%} - {%- if contract_config.enforced -%} - {{ get_assert_columns_equivalent(sql) }} - {%- endif %} - {% if copy_grants -%} copy grants {%- endif %} as ( - {{ sql }} - ); -{% endmacro %} - -{% macro snowflake__create_view_as(relation, sql) -%} - {{ snowflake__create_view_as_with_temp_flag(relation, sql) }} -{% endmacro %} {% macro snowflake__get_columns_in_relation(relation) -%} {%- set sql -%} @@ -248,13 +164,6 @@ {%- endmacro %} -{% macro snowflake__rename_relation(from_relation, to_relation) -%} - {% call statement('rename_relation') -%} - alter table {{ from_relation }} rename to {{ to_relation }} - {%- endcall %} -{% endmacro %} - - {% macro snowflake__alter_column_type(relation, column_name, new_column_type) -%} {% call statement('alter_column_type') %} alter table {{ relation }} alter {{ adapter.quote(column_name) }} set data type {{ new_column_type }}; @@ -372,14 +281,3 @@ {{ snowflake_dml_explicit_transaction(truncate_dml) }} {%- endcall %} {% endmacro %} - - -{% macro snowflake__drop_relation(relation) -%} - {%- if relation.is_dynamic_table -%} - {% call statement('drop_relation', auto_begin=False) -%} - drop dynamic table if exists {{ relation }} - {%- endcall %} - {%- else -%} - {{- default__drop_relation(relation) -}} - {%- endif -%} -{% endmacro %} diff --git a/dbt/include/snowflake/macros/materializations/dynamic_table/materialization.sql b/dbt/include/snowflake/macros/materializations/dynamic_table.sql similarity index 93% rename from dbt/include/snowflake/macros/materializations/dynamic_table/materialization.sql rename to dbt/include/snowflake/macros/materializations/dynamic_table.sql index dc1ed8599..54ecafae8 100644 --- a/dbt/include/snowflake/macros/materializations/dynamic_table/materialization.sql +++ b/dbt/include/snowflake/macros/materializations/dynamic_table.sql @@ -118,3 +118,10 @@ {% do persist_docs(target_relation, model) %} {% endmacro %} + + +{% macro snowflake__get_dynamic_table_configuration_changes(existing_relation, new_config) -%} + {% set _existing_dynamic_table = snowflake__describe_dynamic_table(existing_relation) %} + {% set _configuration_changes = existing_relation.dynamic_table_config_changeset(_existing_dynamic_table, new_config) %} + {% do return(_configuration_changes) %} +{%- endmacro %} diff --git a/dbt/include/snowflake/macros/materializations/dynamic_table/ddl.sql b/dbt/include/snowflake/macros/materializations/dynamic_table/ddl.sql deleted file mode 100644 index cc7228033..000000000 --- a/dbt/include/snowflake/macros/materializations/dynamic_table/ddl.sql +++ /dev/null @@ -1,89 +0,0 @@ -{% macro snowflake__get_alter_dynamic_table_as_sql( - target_relation, - configuration_changes, - sql, - existing_relation, - backup_relation, - intermediate_relation -) -%} - {{- log('Applying ALTER to: ' ~ target_relation) -}} - - {% if configuration_changes.requires_full_refresh %} - {{- snowflake__get_replace_dynamic_table_as_sql(target_relation, sql, existing_relation, backup_relation, intermediate_relation) -}} - - {% else %} - - {%- set target_lag = configuration_changes.target_lag -%} - {%- if target_lag -%}{{- log('Applying UPDATE TARGET_LAG to: ' ~ existing_relation) -}}{%- endif -%} - {%- set snowflake_warehouse = configuration_changes.snowflake_warehouse -%} - {%- if snowflake_warehouse -%}{{- log('Applying UPDATE WAREHOUSE to: ' ~ existing_relation) -}}{%- endif -%} - - alter dynamic table {{ existing_relation }} set - {% if target_lag %}target_lag = '{{ target_lag.context }}'{% endif %} - {% if snowflake_warehouse %}warehouse = {{ snowflake_warehouse.context }}{% endif %} - - {%- endif -%} - -{%- endmacro %} - - -{% macro snowflake__get_create_dynamic_table_as_sql(relation, sql) -%} - {{- log('Applying CREATE to: ' ~ relation) -}} - - create or replace dynamic table {{ relation }} - target_lag = '{{ config.get("target_lag") }}' - warehouse = {{ config.get("snowflake_warehouse") }} - as ( - {{ sql }} - ) - ; - {{ snowflake__refresh_dynamic_table(relation) }} - -{%- endmacro %} - - -{% macro snowflake__describe_dynamic_table(relation) %} - {%- set _dynamic_table_sql -%} - show dynamic tables - like '{{ relation.identifier }}' - in schema {{ relation.database }}.{{ relation.schema }} - ; - select - "name", - "schema_name", - "database_name", - "text", - "target_lag", - "warehouse" - from table(result_scan(last_query_id())) - {%- endset %} - {% set _dynamic_table = run_query(_dynamic_table_sql) %} - - {% do return({'dynamic_table': _dynamic_table}) %} -{% endmacro %} - - -{% macro snowflake__get_replace_dynamic_table_as_sql(target_relation, sql, existing_relation, backup_relation, intermediate_relation) -%} - {{- log('Applying REPLACE to: ' ~ target_relation) -}} - {{ snowflake__get_drop_dynamic_table_sql(existing_relation) }}; - {{ snowflake__get_create_dynamic_table_as_sql(target_relation, sql) }} -{%- endmacro %} - - -{% macro snowflake__refresh_dynamic_table(relation) -%} - {{- log('Applying REFRESH to: ' ~ relation) -}} - - alter dynamic table {{ relation }} refresh -{%- endmacro %} - - -{% macro snowflake__get_dynamic_table_configuration_changes(existing_relation, new_config) -%} - {% set _existing_dynamic_table = snowflake__describe_dynamic_table(existing_relation) %} - {% set _configuration_changes = existing_relation.dynamic_table_config_changeset(_existing_dynamic_table, new_config) %} - {% do return(_configuration_changes) %} -{%- endmacro %} - - -{% macro snowflake__get_drop_dynamic_table_sql(relation) %} - drop dynamic table if exists {{ relation }} -{% endmacro %} diff --git a/dbt/include/snowflake/macros/relations/drop.sql b/dbt/include/snowflake/macros/relations/drop.sql new file mode 100644 index 000000000..1d1de9325 --- /dev/null +++ b/dbt/include/snowflake/macros/relations/drop.sql @@ -0,0 +1,9 @@ +{% macro snowflake__drop_relation(relation) -%} + {%- if relation.is_dynamic_table -%} + {% call statement('drop_relation', auto_begin=False) -%} + drop dynamic table if exists {{ relation }} + {%- endcall %} + {%- else -%} + {{- default__drop_relation(relation) -}} + {%- endif -%} +{% endmacro %} diff --git a/dbt/include/snowflake/macros/relations/dynamic_table/_replace.sql b/dbt/include/snowflake/macros/relations/dynamic_table/_replace.sql new file mode 100644 index 000000000..939322316 --- /dev/null +++ b/dbt/include/snowflake/macros/relations/dynamic_table/_replace.sql @@ -0,0 +1,5 @@ +{% macro snowflake__get_replace_dynamic_table_as_sql(target_relation, sql, existing_relation, backup_relation, intermediate_relation) -%} + {{- log('Applying REPLACE to: ' ~ target_relation) -}} + {{ snowflake__get_drop_dynamic_table_sql(existing_relation) }}; + {{ snowflake__get_create_dynamic_table_as_sql(target_relation, sql) }} +{%- endmacro %} diff --git a/dbt/include/snowflake/macros/relations/dynamic_table/alter.sql b/dbt/include/snowflake/macros/relations/dynamic_table/alter.sql new file mode 100644 index 000000000..2d2dd978e --- /dev/null +++ b/dbt/include/snowflake/macros/relations/dynamic_table/alter.sql @@ -0,0 +1,27 @@ +{% macro snowflake__get_alter_dynamic_table_as_sql( + target_relation, + configuration_changes, + sql, + existing_relation, + backup_relation, + intermediate_relation +) -%} + {{- log('Applying ALTER to: ' ~ target_relation) -}} + + {% if configuration_changes.requires_full_refresh %} + {{- snowflake__get_replace_dynamic_table_as_sql(target_relation, sql, existing_relation, backup_relation, intermediate_relation) -}} + + {% else %} + + {%- set target_lag = configuration_changes.target_lag -%} + {%- if target_lag -%}{{- log('Applying UPDATE TARGET_LAG to: ' ~ existing_relation) -}}{%- endif -%} + {%- set snowflake_warehouse = configuration_changes.snowflake_warehouse -%} + {%- if snowflake_warehouse -%}{{- log('Applying UPDATE WAREHOUSE to: ' ~ existing_relation) -}}{%- endif -%} + + alter dynamic table {{ existing_relation }} set + {% if target_lag %}target_lag = '{{ target_lag.context }}'{% endif %} + {% if snowflake_warehouse %}warehouse = {{ snowflake_warehouse.context }}{% endif %} + + {%- endif -%} + +{%- endmacro %} diff --git a/dbt/include/snowflake/macros/relations/dynamic_table/create.sql b/dbt/include/snowflake/macros/relations/dynamic_table/create.sql new file mode 100644 index 000000000..45271be21 --- /dev/null +++ b/dbt/include/snowflake/macros/relations/dynamic_table/create.sql @@ -0,0 +1,13 @@ +{% macro snowflake__get_create_dynamic_table_as_sql(relation, sql) -%} + {{- log('Applying CREATE to: ' ~ relation) -}} + + create or replace dynamic table {{ relation }} + target_lag = '{{ config.get("target_lag") }}' + warehouse = {{ config.get("snowflake_warehouse") }} + as ( + {{ sql }} + ) + ; + {{ snowflake__refresh_dynamic_table(relation) }} + +{%- endmacro %} diff --git a/dbt/include/snowflake/macros/relations/dynamic_table/describe.sql b/dbt/include/snowflake/macros/relations/dynamic_table/describe.sql new file mode 100644 index 000000000..a5f612039 --- /dev/null +++ b/dbt/include/snowflake/macros/relations/dynamic_table/describe.sql @@ -0,0 +1,19 @@ +{% macro snowflake__describe_dynamic_table(relation) %} + {%- set _dynamic_table_sql -%} + show dynamic tables + like '{{ relation.identifier }}' + in schema {{ relation.database }}.{{ relation.schema }} + ; + select + "name", + "schema_name", + "database_name", + "text", + "target_lag", + "warehouse" + from table(result_scan(last_query_id())) + {%- endset %} + {% set _dynamic_table = run_query(_dynamic_table_sql) %} + + {% do return({'dynamic_table': _dynamic_table}) %} +{% endmacro %} diff --git a/dbt/include/snowflake/macros/relations/dynamic_table/drop.sql b/dbt/include/snowflake/macros/relations/dynamic_table/drop.sql new file mode 100644 index 000000000..577bd06a0 --- /dev/null +++ b/dbt/include/snowflake/macros/relations/dynamic_table/drop.sql @@ -0,0 +1,3 @@ +{% macro snowflake__get_drop_dynamic_table_sql(relation) %} + drop dynamic table if exists {{ relation }} +{% endmacro %} diff --git a/dbt/include/snowflake/macros/relations/dynamic_table/refresh.sql b/dbt/include/snowflake/macros/relations/dynamic_table/refresh.sql new file mode 100644 index 000000000..5c6af1bda --- /dev/null +++ b/dbt/include/snowflake/macros/relations/dynamic_table/refresh.sql @@ -0,0 +1,5 @@ +{% macro snowflake__refresh_dynamic_table(relation) -%} + {{- log('Applying REFRESH to: ' ~ relation) -}} + + alter dynamic table {{ relation }} refresh +{%- endmacro %} diff --git a/dbt/include/snowflake/macros/relations/rename.sql b/dbt/include/snowflake/macros/relations/rename.sql new file mode 100644 index 000000000..12f0fae98 --- /dev/null +++ b/dbt/include/snowflake/macros/relations/rename.sql @@ -0,0 +1,5 @@ +{% macro snowflake__rename_relation(from_relation, to_relation) -%} + {% call statement('rename_relation') -%} + alter table {{ from_relation }} rename to {{ to_relation }} + {%- endcall %} +{% endmacro %} diff --git a/dbt/include/snowflake/macros/relations/table/create.sql b/dbt/include/snowflake/macros/relations/table/create.sql new file mode 100644 index 000000000..8924af00a --- /dev/null +++ b/dbt/include/snowflake/macros/relations/table/create.sql @@ -0,0 +1,54 @@ +{% macro snowflake__create_table_as(temporary, relation, compiled_code, language='sql') -%} + {%- if language == 'sql' -%} + {%- set transient = config.get('transient', default=true) -%} + {%- set cluster_by_keys = config.get('cluster_by', default=none) -%} + {%- set enable_automatic_clustering = config.get('automatic_clustering', default=false) -%} + {%- set copy_grants = config.get('copy_grants', default=false) -%} + + {%- if cluster_by_keys is not none and cluster_by_keys is string -%} + {%- set cluster_by_keys = [cluster_by_keys] -%} + {%- endif -%} + {%- if cluster_by_keys is not none -%} + {%- set cluster_by_string = cluster_by_keys|join(", ")-%} + {% else %} + {%- set cluster_by_string = none -%} + {%- endif -%} + {%- set sql_header = config.get('sql_header', none) -%} + + {{ sql_header if sql_header is not none }} + + create or replace {% if temporary -%} + temporary + {%- elif transient -%} + transient + {%- endif %} table {{ relation }} + {%- set contract_config = config.get('contract') -%} + {%- if contract_config.enforced -%} + {{ get_assert_columns_equivalent(sql) }} + {{ get_table_columns_and_constraints() }} + {% set compiled_code = get_select_subquery(compiled_code) %} + {% endif %} + {% if copy_grants and not temporary -%} copy grants {%- endif %} as + ( + {%- if cluster_by_string is not none -%} + select * from ( + {{ compiled_code }} + ) order by ({{ cluster_by_string }}) + {%- else -%} + {{ compiled_code }} + {%- endif %} + ); + {% if cluster_by_string is not none and not temporary -%} + alter table {{relation}} cluster by ({{cluster_by_string}}); + {%- endif -%} + {% if enable_automatic_clustering and cluster_by_string is not none and not temporary -%} + alter table {{relation}} resume recluster; + {%- endif -%} + + {%- elif language == 'python' -%} + {{ py_write_table(compiled_code=compiled_code, target_relation=relation, temporary=temporary) }} + {%- else -%} + {% do exceptions.raise_compiler_error("snowflake__create_table_as macro didn't get supported language, it got %s" % language) %} + {%- endif -%} + +{% endmacro %} diff --git a/dbt/include/snowflake/macros/relations/view/create.sql b/dbt/include/snowflake/macros/relations/view/create.sql new file mode 100644 index 000000000..3212e15c6 --- /dev/null +++ b/dbt/include/snowflake/macros/relations/view/create.sql @@ -0,0 +1,30 @@ +{% macro snowflake__create_view_as_with_temp_flag(relation, sql, is_temporary=False) -%} + {%- set secure = config.get('secure', default=false) -%} + {%- set copy_grants = config.get('copy_grants', default=false) -%} + {%- set sql_header = config.get('sql_header', none) -%} + + {{ sql_header if sql_header is not none }} + create or replace {% if secure -%} + secure + {%- endif %} {% if is_temporary -%} + temporary + {%- endif %} view {{ relation }} + {% if config.persist_column_docs() -%} + {% set model_columns = model.columns %} + {% set query_columns = get_columns_in_query(sql) %} + {{ get_persist_docs_column_list(model_columns, query_columns) }} + + {%- endif %} + {%- set contract_config = config.get('contract') -%} + {%- if contract_config.enforced -%} + {{ get_assert_columns_equivalent(sql) }} + {%- endif %} + {% if copy_grants -%} copy grants {%- endif %} as ( + {{ sql }} + ); +{% endmacro %} + + +{% macro snowflake__create_view_as(relation, sql) -%} + {{ snowflake__create_view_as_with_temp_flag(relation, sql) }} +{% endmacro %} diff --git a/dev-requirements.txt b/dev-requirements.txt index 2ccb4e980..5a25955fa 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -13,7 +13,7 @@ flake8~=6.1 flaky~=3.7 freezegun~=1.2 ipdb~=0.13.13 -mypy==1.5.0 # patch updates have historically introduced breaking changes +mypy==1.5.1 # patch updates have historically introduced breaking changes pip-tools~=7.3 pre-commit~=3.3 pre-commit-hooks~=4.4 @@ -23,7 +23,7 @@ pytest-dotenv~=0.5.2 pytest-logbook~=1.2 pytest-xdist~=3.3 pytz~=2023.3 -tox~=4.10 +tox~=4.11 types-pytz~=2023.3 types-requests~=2.31 twine~=4.0