From 7f6095be42b2aff23aba57acdcd7ede3bc2a6caf Mon Sep 17 00:00:00 2001 From: Mila Page Date: Tue, 29 Nov 2022 01:48:26 -0800 Subject: [PATCH 1/4] Convert incremental schema tests. --- .../incremental_schema_tests/fixtures.py | 395 ++++++++++++++++++ .../models/incremental_append_new_columns.sql | 29 ++ ...remental_append_new_columns_remove_one.sql | 28 ++ ...l_append_new_columns_remove_one_target.sql | 19 + .../incremental_append_new_columns_target.sql | 19 + .../models/incremental_fail.sql | 19 + .../models/incremental_ignore.sql | 19 + .../models/incremental_ignore_target.sql | 15 + .../models/incremental_sync_all_columns.sql | 31 ++ .../incremental_sync_all_columns_target.sql | 20 + .../models/incremental_sync_remove_only.sql | 29 ++ .../incremental_sync_remove_only_target.sql | 17 + .../models/model_a.sql | 22 + .../models/schema.yml | 54 +++ .../test_incremental_schema.py | 136 ++++++ .../tests/select_from_a.sql | 1 + ...ct_from_incremental_append_new_columns.sql | 1 + ..._incremental_append_new_columns_target.sql | 1 + .../tests/select_from_incremental_ignore.sql | 1 + .../select_from_incremental_ignore_target.sql | 1 + ...lect_from_incremental_sync_all_columns.sql | 1 + ...om_incremental_sync_all_columns_target.sql | 1 + 22 files changed, 859 insertions(+) create mode 100644 tests/functional/incremental_schema_tests/fixtures.py create mode 100644 tests/functional/incremental_schema_tests/models/incremental_append_new_columns.sql create mode 100644 tests/functional/incremental_schema_tests/models/incremental_append_new_columns_remove_one.sql create mode 100644 tests/functional/incremental_schema_tests/models/incremental_append_new_columns_remove_one_target.sql create mode 100644 tests/functional/incremental_schema_tests/models/incremental_append_new_columns_target.sql create mode 100644 tests/functional/incremental_schema_tests/models/incremental_fail.sql create mode 100644 tests/functional/incremental_schema_tests/models/incremental_ignore.sql create mode 100644 tests/functional/incremental_schema_tests/models/incremental_ignore_target.sql create mode 100644 tests/functional/incremental_schema_tests/models/incremental_sync_all_columns.sql create mode 100644 tests/functional/incremental_schema_tests/models/incremental_sync_all_columns_target.sql create mode 100644 tests/functional/incremental_schema_tests/models/incremental_sync_remove_only.sql create mode 100644 tests/functional/incremental_schema_tests/models/incremental_sync_remove_only_target.sql create mode 100644 tests/functional/incremental_schema_tests/models/model_a.sql create mode 100644 tests/functional/incremental_schema_tests/models/schema.yml create mode 100644 tests/functional/incremental_schema_tests/test_incremental_schema.py create mode 100644 tests/functional/incremental_schema_tests/tests/select_from_a.sql create mode 100644 tests/functional/incremental_schema_tests/tests/select_from_incremental_append_new_columns.sql create mode 100644 tests/functional/incremental_schema_tests/tests/select_from_incremental_append_new_columns_target.sql create mode 100644 tests/functional/incremental_schema_tests/tests/select_from_incremental_ignore.sql create mode 100644 tests/functional/incremental_schema_tests/tests/select_from_incremental_ignore_target.sql create mode 100644 tests/functional/incremental_schema_tests/tests/select_from_incremental_sync_all_columns.sql create mode 100644 tests/functional/incremental_schema_tests/tests/select_from_incremental_sync_all_columns_target.sql diff --git a/tests/functional/incremental_schema_tests/fixtures.py b/tests/functional/incremental_schema_tests/fixtures.py new file mode 100644 index 00000000000..e6dbd5f27cf --- /dev/null +++ b/tests/functional/incremental_schema_tests/fixtures.py @@ -0,0 +1,395 @@ + +# +# Properties +# +_PROPERTIES__SCHEMA = """ +version: 2 + +models: + - name: model_a + columns: + - name: id + tags: [column_level_tag] + tests: + - unique + + - name: incremental_ignore + columns: + - name: id + tags: [column_level_tag] + tests: + - unique + + - name: incremental_ignore_target + columns: + - name: id + tags: [column_level_tag] + tests: + - unique + + - name: incremental_append_new_columns + columns: + - name: id + tags: [column_level_tag] + tests: + - unique + + - name: incremental_append_new_columns_target + columns: + - name: id + tags: [column_level_tag] + tests: + - unique + + - name: incremental_sync_all_columns + columns: + - name: id + tags: [column_level_tag] + tests: + - unique + + - name: incremental_sync_all_columns_target + columns: + - name: id + tags: [column_leveL_tag] + tests: + - unique +""" + +# +# Models +# +_MODELS__INCREMENTAL_SYNC_REMOVE_ONLY = """ +{{ + config( + materialized='incremental', + unique_key='id', + on_schema_change='sync_all_columns' + + ) +}} + +WITH source_data AS (SELECT * FROM {{ ref('model_a') }} ) + +{% set string_type = 'varchar(10)' %} + +{% if is_incremental() %} + +SELECT id, + cast(field1 as {{string_type}}) as field1 + +FROM source_data WHERE id NOT IN (SELECT id from {{ this }} ) + +{% else %} + +select id, + cast(field1 as {{string_type}}) as field1, + cast(field2 as {{string_type}}) as field2 + +from source_data where id <= 3 + +{% endif %} +""" + +_MODELS__INCREMENTAL_IGNORE = """ +{{ + config( + materialized='incremental', + unique_key='id', + on_schema_change='ignore' + ) +}} + +WITH source_data AS (SELECT * FROM {{ ref('model_a') }} ) + +{% if is_incremental() %} + +SELECT id, field1, field2, field3, field4 FROM source_data WHERE id NOT IN (SELECT id from {{ this }} ) + +{% else %} + +SELECT id, field1, field2 FROM source_data LIMIT 3 + +{% endif %} +""" + +_MODELS__INCREMENTAL_SYNC_REMOVE_ONLY_TARGET = """ +{{ + config(materialized='table') +}} + +with source_data as ( + + select * from {{ ref('model_a') }} + +) + +{% set string_type = 'varchar(10)' %} + +select id + ,cast(field1 as {{string_type}}) as field1 + +from source_data +order by id +""" + +_MODELS__INCREMENTAL_IGNORE_TARGET = """ +{{ + config(materialized='table') +}} + +with source_data as ( + + select * from {{ ref('model_a') }} + +) + +select id + ,field1 + ,field2 + +from source_data +""" + +_MODELS__INCREMENTAL_FAIL = """ +{{ + config( + materialized='incremental', + unique_key='id', + on_schema_change='fail' + ) +}} + +WITH source_data AS (SELECT * FROM {{ ref('model_a') }} ) + +{% if is_incremental() %} + +SELECT id, field1, field2 FROM source_data + +{% else %} + +SELECT id, field1, field3 FROm source_data + +{% endif %} +""" + +_MODELS__INCREMENTAL_SYNC_ALL_COLUMNS = """ +{{ + config( + materialized='incremental', + unique_key='id', + on_schema_change='sync_all_columns' + + ) +}} + +WITH source_data AS (SELECT * FROM {{ ref('model_a') }} ) + +{% set string_type = 'varchar(10)' %} + +{% if is_incremental() %} + +SELECT id, + cast(field1 as {{string_type}}) as field1, + cast(field3 as {{string_type}}) as field3, -- to validate new fields + cast(field4 as {{string_type}}) AS field4 -- to validate new fields + +FROM source_data WHERE id NOT IN (SELECT id from {{ this }} ) + +{% else %} + +select id, + cast(field1 as {{string_type}}) as field1, + cast(field2 as {{string_type}}) as field2 + +from source_data where id <= 3 + +{% endif %} +""" + +_MODELS__INCREMENTAL_APPEND_NEW_COLUMNS_REMOVE_ONE = """ +{{ + config( + materialized='incremental', + unique_key='id', + on_schema_change='append_new_columns' + ) +}} + +{% set string_type = 'varchar(10)' %} + +WITH source_data AS (SELECT * FROM {{ ref('model_a') }} ) + +{% if is_incremental() %} + +SELECT id, + cast(field1 as {{string_type}}) as field1, + cast(field3 as {{string_type}}) as field3, + cast(field4 as {{string_type}}) as field4 +FROM source_data WHERE id NOT IN (SELECT id from {{ this }} ) + +{% else %} + +SELECT id, + cast(field1 as {{string_type}}) as field1, + cast(field2 as {{string_type}}) as field2 +FROM source_data where id <= 3 + +{% endif %} +""" + +_MODELS__A = """ +{{ + config(materialized='table') +}} + +with source_data as ( + + select 1 as id, 'aaa' as field1, 'bbb' as field2, 111 as field3, 'TTT' as field4 + union all select 2 as id, 'ccc' as field1, 'ddd' as field2, 222 as field3, 'UUU' as field4 + union all select 3 as id, 'eee' as field1, 'fff' as field2, 333 as field3, 'VVV' as field4 + union all select 4 as id, 'ggg' as field1, 'hhh' as field2, 444 as field3, 'WWW' as field4 + union all select 5 as id, 'iii' as field1, 'jjj' as field2, 555 as field3, 'XXX' as field4 + union all select 6 as id, 'kkk' as field1, 'lll' as field2, 666 as field3, 'YYY' as field4 + +) + +select id + ,field1 + ,field2 + ,field3 + ,field4 + +from source_data +""" + +_MODELS__INCREMENTAL_APPEND_NEW_COLUMNS_TARGET = """ +{{ + config(materialized='table') +}} + +{% set string_type = 'varchar(10)' %} + +with source_data as ( + + select * from {{ ref('model_a') }} + +) + +select id + ,cast(field1 as {{string_type}}) as field1 + ,cast(field2 as {{string_type}}) as field2 + ,cast(CASE WHEN id <= 3 THEN NULL ELSE field3 END as {{string_type}}) AS field3 + ,cast(CASE WHEN id <= 3 THEN NULL ELSE field4 END as {{string_type}}) AS field4 + +from source_data +""" + +_MODELS__INCREMENTAL_APPEND_NEW_COLUMNS = """ +{{ + config( + materialized='incremental', + unique_key='id', + on_schema_change='append_new_columns' + ) +}} + +{% set string_type = 'varchar(10)' %} + +WITH source_data AS (SELECT * FROM {{ ref('model_a') }} ) + +{% if is_incremental() %} + +SELECT id, + cast(field1 as {{string_type}}) as field1, + cast(field2 as {{string_type}}) as field2, + cast(field3 as {{string_type}}) as field3, + cast(field4 as {{string_type}}) as field4 +FROM source_data WHERE id NOT IN (SELECT id from {{ this }} ) + +{% else %} + +SELECT id, + cast(field1 as {{string_type}}) as field1, + cast(field2 as {{string_type}}) as field2 +FROM source_data where id <= 3 + +{% endif %} +""" + +_MODELS__INCREMENTAL_SYNC_ALL_COLUMNS_TARGET = """ +{{ + config(materialized='table') +}} + +with source_data as ( + + select * from {{ ref('model_a') }} + +) + +{% set string_type = 'varchar(10)' %} + +select id + ,cast(field1 as {{string_type}}) as field1 + --,field2 + ,cast(case when id <= 3 then null else field3 end as {{string_type}}) as field3 + ,cast(case when id <= 3 then null else field4 end as {{string_type}}) as field4 + +from source_data +order by id +""" + +_MODELS__INCREMENTAL_APPEND_NEW_COLUMNS_REMOVE_ONE_TARGET = """ +{{ + config(materialized='table') +}} + +{% set string_type = 'varchar(10)' %} + +with source_data as ( + + select * from {{ ref('model_a') }} + +) + +select id, + cast(field1 as {{string_type}}) as field1, + cast(CASE WHEN id > 3 THEN NULL ELSE field2 END as {{string_type}}) AS field2, + cast(CASE WHEN id <= 3 THEN NULL ELSE field3 END as {{string_type}}) AS field3, + cast(CASE WHEN id <= 3 THEN NULL ELSE field4 END as {{string_type}}) AS field4 + +from source_data +""" + +# +# Tests +# + +_TESTS__SELECT_FROM_INCREMENTAL_IGNORE = """ +select * from {{ ref('incremental_ignore') }} where false +""" + +_TESTS__SELECT_FROM_A = """ +select * from {{ ref('model_a') }} where false +""" + +_TESTS__SELECT_FROM_INCREMENTAL_APPEND_NEW_COLUMNS_TARGET = """ +select * from {{ ref('incremental_append_new_columns_target') }} where false +""" + +_TESTS__SELECT_FROM_INCREMENTAL_SYNC_ALL_COLUMNS = """ +select * from {{ ref('incremental_sync_all_columns') }} where false +""" + +_TESTS__SELECT_FROM_INCREMENTAL_SYNC_ALL_COLUMNS_TARGET = """ +select * from {{ ref('incremental_sync_all_columns_target') }} where false +""" + +_TESTS__SELECT_FROM_INCREMENTAL_IGNORE_TARGET = """ +select * from {{ ref('incremental_ignore_target') }} where false +""" + +_TESTS__SELECT_FROM_INCREMENTAL_APPEND_NEW_COLUMNS = """ +select * from {{ ref('incremental_append_new_columns') }} where false +""" diff --git a/tests/functional/incremental_schema_tests/models/incremental_append_new_columns.sql b/tests/functional/incremental_schema_tests/models/incremental_append_new_columns.sql new file mode 100644 index 00000000000..f9eebdcb852 --- /dev/null +++ b/tests/functional/incremental_schema_tests/models/incremental_append_new_columns.sql @@ -0,0 +1,29 @@ +{{ + config( + materialized='incremental', + unique_key='id', + on_schema_change='append_new_columns' + ) +}} + +{% set string_type = 'varchar(10)' %} + +WITH source_data AS (SELECT * FROM {{ ref('model_a') }} ) + +{% if is_incremental() %} + +SELECT id, + cast(field1 as {{string_type}}) as field1, + cast(field2 as {{string_type}}) as field2, + cast(field3 as {{string_type}}) as field3, + cast(field4 as {{string_type}}) as field4 +FROM source_data WHERE id NOT IN (SELECT id from {{ this }} ) + +{% else %} + +SELECT id, + cast(field1 as {{string_type}}) as field1, + cast(field2 as {{string_type}}) as field2 +FROM source_data where id <= 3 + +{% endif %} \ No newline at end of file diff --git a/tests/functional/incremental_schema_tests/models/incremental_append_new_columns_remove_one.sql b/tests/functional/incremental_schema_tests/models/incremental_append_new_columns_remove_one.sql new file mode 100644 index 00000000000..dbb4962a7e5 --- /dev/null +++ b/tests/functional/incremental_schema_tests/models/incremental_append_new_columns_remove_one.sql @@ -0,0 +1,28 @@ +{{ + config( + materialized='incremental', + unique_key='id', + on_schema_change='append_new_columns' + ) +}} + +{% set string_type = 'varchar(10)' %} + +WITH source_data AS (SELECT * FROM {{ ref('model_a') }} ) + +{% if is_incremental() %} + +SELECT id, + cast(field1 as {{string_type}}) as field1, + cast(field3 as {{string_type}}) as field3, + cast(field4 as {{string_type}}) as field4 +FROM source_data WHERE id NOT IN (SELECT id from {{ this }} ) + +{% else %} + +SELECT id, + cast(field1 as {{string_type}}) as field1, + cast(field2 as {{string_type}}) as field2 +FROM source_data where id <= 3 + +{% endif %} \ No newline at end of file diff --git a/tests/functional/incremental_schema_tests/models/incremental_append_new_columns_remove_one_target.sql b/tests/functional/incremental_schema_tests/models/incremental_append_new_columns_remove_one_target.sql new file mode 100644 index 00000000000..f3a279f0285 --- /dev/null +++ b/tests/functional/incremental_schema_tests/models/incremental_append_new_columns_remove_one_target.sql @@ -0,0 +1,19 @@ +{{ + config(materialized='table') +}} + +{% set string_type = 'varchar(10)' %} + +with source_data as ( + + select * from {{ ref('model_a') }} + +) + +select id, + cast(field1 as {{string_type}}) as field1, + cast(CASE WHEN id > 3 THEN NULL ELSE field2 END as {{string_type}}) AS field2, + cast(CASE WHEN id <= 3 THEN NULL ELSE field3 END as {{string_type}}) AS field3, + cast(CASE WHEN id <= 3 THEN NULL ELSE field4 END as {{string_type}}) AS field4 + +from source_data \ No newline at end of file diff --git a/tests/functional/incremental_schema_tests/models/incremental_append_new_columns_target.sql b/tests/functional/incremental_schema_tests/models/incremental_append_new_columns_target.sql new file mode 100644 index 00000000000..5ff759d7dab --- /dev/null +++ b/tests/functional/incremental_schema_tests/models/incremental_append_new_columns_target.sql @@ -0,0 +1,19 @@ +{{ + config(materialized='table') +}} + +{% set string_type = 'varchar(10)' %} + +with source_data as ( + + select * from {{ ref('model_a') }} + +) + +select id + ,cast(field1 as {{string_type}}) as field1 + ,cast(field2 as {{string_type}}) as field2 + ,cast(CASE WHEN id <= 3 THEN NULL ELSE field3 END as {{string_type}}) AS field3 + ,cast(CASE WHEN id <= 3 THEN NULL ELSE field4 END as {{string_type}}) AS field4 + +from source_data \ No newline at end of file diff --git a/tests/functional/incremental_schema_tests/models/incremental_fail.sql b/tests/functional/incremental_schema_tests/models/incremental_fail.sql new file mode 100644 index 00000000000..590f5b56d97 --- /dev/null +++ b/tests/functional/incremental_schema_tests/models/incremental_fail.sql @@ -0,0 +1,19 @@ +{{ + config( + materialized='incremental', + unique_key='id', + on_schema_change='fail' + ) +}} + +WITH source_data AS (SELECT * FROM {{ ref('model_a') }} ) + +{% if is_incremental() %} + +SELECT id, field1, field2 FROM source_data + +{% else %} + +SELECT id, field1, field3 FROm source_data + +{% endif %} \ No newline at end of file diff --git a/tests/functional/incremental_schema_tests/models/incremental_ignore.sql b/tests/functional/incremental_schema_tests/models/incremental_ignore.sql new file mode 100644 index 00000000000..51dee6022fb --- /dev/null +++ b/tests/functional/incremental_schema_tests/models/incremental_ignore.sql @@ -0,0 +1,19 @@ +{{ + config( + materialized='incremental', + unique_key='id', + on_schema_change='ignore' + ) +}} + +WITH source_data AS (SELECT * FROM {{ ref('model_a') }} ) + +{% if is_incremental() %} + +SELECT id, field1, field2, field3, field4 FROM source_data WHERE id NOT IN (SELECT id from {{ this }} ) + +{% else %} + +SELECT id, field1, field2 FROM source_data LIMIT 3 + +{% endif %} \ No newline at end of file diff --git a/tests/functional/incremental_schema_tests/models/incremental_ignore_target.sql b/tests/functional/incremental_schema_tests/models/incremental_ignore_target.sql new file mode 100644 index 00000000000..92d4564e0e8 --- /dev/null +++ b/tests/functional/incremental_schema_tests/models/incremental_ignore_target.sql @@ -0,0 +1,15 @@ +{{ + config(materialized='table') +}} + +with source_data as ( + + select * from {{ ref('model_a') }} + +) + +select id + ,field1 + ,field2 + +from source_data \ No newline at end of file diff --git a/tests/functional/incremental_schema_tests/models/incremental_sync_all_columns.sql b/tests/functional/incremental_schema_tests/models/incremental_sync_all_columns.sql new file mode 100644 index 00000000000..b742c970419 --- /dev/null +++ b/tests/functional/incremental_schema_tests/models/incremental_sync_all_columns.sql @@ -0,0 +1,31 @@ +{{ + config( + materialized='incremental', + unique_key='id', + on_schema_change='sync_all_columns' + + ) +}} + +WITH source_data AS (SELECT * FROM {{ ref('model_a') }} ) + +{% set string_type = 'varchar(10)' %} + +{% if is_incremental() %} + +SELECT id, + cast(field1 as {{string_type}}) as field1, + cast(field3 as {{string_type}}) as field3, -- to validate new fields + cast(field4 as {{string_type}}) AS field4 -- to validate new fields + +FROM source_data WHERE id NOT IN (SELECT id from {{ this }} ) + +{% else %} + +select id, + cast(field1 as {{string_type}}) as field1, + cast(field2 as {{string_type}}) as field2 + +from source_data where id <= 3 + +{% endif %} \ No newline at end of file diff --git a/tests/functional/incremental_schema_tests/models/incremental_sync_all_columns_target.sql b/tests/functional/incremental_schema_tests/models/incremental_sync_all_columns_target.sql new file mode 100644 index 00000000000..6cdbaba5c0d --- /dev/null +++ b/tests/functional/incremental_schema_tests/models/incremental_sync_all_columns_target.sql @@ -0,0 +1,20 @@ +{{ + config(materialized='table') +}} + +with source_data as ( + + select * from {{ ref('model_a') }} + +) + +{% set string_type = 'varchar(10)' %} + +select id + ,cast(field1 as {{string_type}}) as field1 + --,field2 + ,cast(case when id <= 3 then null else field3 end as {{string_type}}) as field3 + ,cast(case when id <= 3 then null else field4 end as {{string_type}}) as field4 + +from source_data +order by id \ No newline at end of file diff --git a/tests/functional/incremental_schema_tests/models/incremental_sync_remove_only.sql b/tests/functional/incremental_schema_tests/models/incremental_sync_remove_only.sql new file mode 100644 index 00000000000..55bae0ad17e --- /dev/null +++ b/tests/functional/incremental_schema_tests/models/incremental_sync_remove_only.sql @@ -0,0 +1,29 @@ +{{ + config( + materialized='incremental', + unique_key='id', + on_schema_change='sync_all_columns' + + ) +}} + +WITH source_data AS (SELECT * FROM {{ ref('model_a') }} ) + +{% set string_type = 'varchar(10)' %} + +{% if is_incremental() %} + +SELECT id, + cast(field1 as {{string_type}}) as field1 + +FROM source_data WHERE id NOT IN (SELECT id from {{ this }} ) + +{% else %} + +select id, + cast(field1 as {{string_type}}) as field1, + cast(field2 as {{string_type}}) as field2 + +from source_data where id <= 3 + +{% endif %} \ No newline at end of file diff --git a/tests/functional/incremental_schema_tests/models/incremental_sync_remove_only_target.sql b/tests/functional/incremental_schema_tests/models/incremental_sync_remove_only_target.sql new file mode 100644 index 00000000000..ff88512c6f5 --- /dev/null +++ b/tests/functional/incremental_schema_tests/models/incremental_sync_remove_only_target.sql @@ -0,0 +1,17 @@ +{{ + config(materialized='table') +}} + +with source_data as ( + + select * from {{ ref('model_a') }} + +) + +{% set string_type = 'varchar(10)' %} + +select id + ,cast(field1 as {{string_type}}) as field1 + +from source_data +order by id \ No newline at end of file diff --git a/tests/functional/incremental_schema_tests/models/model_a.sql b/tests/functional/incremental_schema_tests/models/model_a.sql new file mode 100644 index 00000000000..2a0b2ddaff2 --- /dev/null +++ b/tests/functional/incremental_schema_tests/models/model_a.sql @@ -0,0 +1,22 @@ +{{ + config(materialized='table') +}} + +with source_data as ( + + select 1 as id, 'aaa' as field1, 'bbb' as field2, 111 as field3, 'TTT' as field4 + union all select 2 as id, 'ccc' as field1, 'ddd' as field2, 222 as field3, 'UUU' as field4 + union all select 3 as id, 'eee' as field1, 'fff' as field2, 333 as field3, 'VVV' as field4 + union all select 4 as id, 'ggg' as field1, 'hhh' as field2, 444 as field3, 'WWW' as field4 + union all select 5 as id, 'iii' as field1, 'jjj' as field2, 555 as field3, 'XXX' as field4 + union all select 6 as id, 'kkk' as field1, 'lll' as field2, 666 as field3, 'YYY' as field4 + +) + +select id + ,field1 + ,field2 + ,field3 + ,field4 + +from source_data \ No newline at end of file diff --git a/tests/functional/incremental_schema_tests/models/schema.yml b/tests/functional/incremental_schema_tests/models/schema.yml new file mode 100644 index 00000000000..5546314e413 --- /dev/null +++ b/tests/functional/incremental_schema_tests/models/schema.yml @@ -0,0 +1,54 @@ +version: 2 + +models: + - name: model_a + columns: + - name: id + tags: [column_level_tag] + tests: + - unique + + - name: incremental_ignore + columns: + - name: id + tags: [column_level_tag] + tests: + - unique + + - name: incremental_ignore_target + columns: + - name: id + tags: [column_level_tag] + tests: + - unique + + - name: incremental_append_new_columns + columns: + - name: id + tags: [column_level_tag] + tests: + - unique + + - name: incremental_append_new_columns_target + columns: + - name: id + tags: [column_level_tag] + tests: + - unique + + - name: incremental_sync_all_columns + columns: + - name: id + tags: [column_level_tag] + tests: + - unique + + - name: incremental_sync_all_columns_target + columns: + - name: id + tags: [column_leveL_tag] + tests: + - unique + + + \ No newline at end of file diff --git a/tests/functional/incremental_schema_tests/test_incremental_schema.py b/tests/functional/incremental_schema_tests/test_incremental_schema.py new file mode 100644 index 00000000000..56e250d0f3b --- /dev/null +++ b/tests/functional/incremental_schema_tests/test_incremental_schema.py @@ -0,0 +1,136 @@ +import pytest + +from dbt.tests.util import ( + check_relations_equal, + run_dbt, +) + +from tests.functional.incremental_schema_tests.fixtures import ( + _PROPERTIES__SCHEMA, + _MODELS__INCREMENTAL_SYNC_REMOVE_ONLY, + _MODELS__INCREMENTAL_IGNORE, + _MODELS__INCREMENTAL_SYNC_REMOVE_ONLY_TARGET, + _MODELS__INCREMENTAL_IGNORE_TARGET, + _MODELS__INCREMENTAL_FAIL, + _MODELS__INCREMENTAL_SYNC_ALL_COLUMNS, + _MODELS__INCREMENTAL_APPEND_NEW_COLUMNS_REMOVE_ONE, + _MODELS__A, + _MODELS__INCREMENTAL_APPEND_NEW_COLUMNS_TARGET, + _MODELS__INCREMENTAL_APPEND_NEW_COLUMNS, + _MODELS__INCREMENTAL_SYNC_ALL_COLUMNS_TARGET, + _MODELS__INCREMENTAL_APPEND_NEW_COLUMNS_REMOVE_ONE_TARGET, + _TESTS__SELECT_FROM_INCREMENTAL_IGNORE, + _TESTS__SELECT_FROM_A, + _TESTS__SELECT_FROM_INCREMENTAL_APPEND_NEW_COLUMNS_TARGET, + _TESTS__SELECT_FROM_INCREMENTAL_SYNC_ALL_COLUMNS, + _TESTS__SELECT_FROM_INCREMENTAL_SYNC_ALL_COLUMNS_TARGET, + _TESTS__SELECT_FROM_INCREMENTAL_IGNORE_TARGET, + _TESTS__SELECT_FROM_INCREMENTAL_APPEND_NEW_COLUMNS, +) + + +class TestIncrementalSchemaChange: + @pytest.fixture(scope="class") + def properties(self): + return { + "schema.yml": _PROPERTIES__SCHEMA, + } + + @pytest.fixture(scope="class") + def models(self): + return { + "incremental_sync_remove_only.sql": _MODELS__INCREMENTAL_SYNC_REMOVE_ONLY, + "incremental_ignore.sql": _MODELS__INCREMENTAL_IGNORE, + "incremental_sync_remove_only_target.sql": + _MODELS__INCREMENTAL_SYNC_REMOVE_ONLY_TARGET, + "incremental_ignore_target.sql": _MODELS__INCREMENTAL_IGNORE_TARGET, + "incremental_fail.sql": _MODELS__INCREMENTAL_FAIL, + "incremental_sync_all_columns.sql": _MODELS__INCREMENTAL_SYNC_ALL_COLUMNS, + "incremental_append_new_columns_remove_one.sql": + _MODELS__INCREMENTAL_APPEND_NEW_COLUMNS_REMOVE_ONE, + "model_a.sql": _MODELS__A, + "incremental_append_new_columns_target.sql": + _MODELS__INCREMENTAL_APPEND_NEW_COLUMNS_TARGET, + "incremental_append_new_columns.sql": _MODELS__INCREMENTAL_APPEND_NEW_COLUMNS, + "incremental_sync_all_columns_target.sql": + _MODELS__INCREMENTAL_SYNC_ALL_COLUMNS_TARGET, + "incremental_append_new_columns_remove_one_target.sql": + _MODELS__INCREMENTAL_APPEND_NEW_COLUMNS_REMOVE_ONE_TARGET, + } + + @pytest.fixture(scope="class") + def tests(self): + return { + "select_from_incremental.sql": _TESTS__SELECT_FROM_INCREMENTAL_IGNORE, + "select_from_a.sql": _TESTS__SELECT_FROM_A, + "select_from_incremental_append_new_columns_target.sql": + _TESTS__SELECT_FROM_INCREMENTAL_APPEND_NEW_COLUMNS_TARGET, + "select_from_incremental_sync_all_columns.sql": + _TESTS__SELECT_FROM_INCREMENTAL_SYNC_ALL_COLUMNS, + "select_from_incremental_sync_all_columns_target.sql": + _TESTS__SELECT_FROM_INCREMENTAL_SYNC_ALL_COLUMNS_TARGET, + "select_from_incremental_ignore_target.sql": + _TESTS__SELECT_FROM_INCREMENTAL_IGNORE_TARGET, + "select_from_incremental_append_new_columns.sql": + _TESTS__SELECT_FROM_INCREMENTAL_APPEND_NEW_COLUMNS, + } + + def run_twice_and_assert( + self, include, compare_source, compare_target, project + ): + + # dbt run (twice) + run_args = ['run'] + if include: + run_args.extend(('--select', include)) + results_one = run_dbt(run_args) + assert len(results_one) == 3 + + results_two = run_dbt(run_args) + assert len(results_two) == 3 + + check_relations_equal(project.adapter, [compare_source, compare_target]) + + def run_incremental_append_new_columns(self, project): + select = 'model_a incremental_append_new_columns incremental_append_new_columns_target' + compare_source = 'incremental_append_new_columns' + compare_target = 'incremental_append_new_columns_target' + self.run_twice_and_assert(select, compare_source, compare_target, project) + + def run_incremental_append_new_columns_remove_one(self, project): + select = 'model_a incremental_append_new_columns_remove_one incremental_append_new_columns_remove_one_target' + compare_source = 'incremental_append_new_columns_remove_one' + compare_target = 'incremental_append_new_columns_remove_one_target' + self.run_twice_and_assert(select, compare_source, compare_target, project) + + def run_incremental_sync_all_columns(self, project): + select = 'model_a incremental_sync_all_columns incremental_sync_all_columns_target' + compare_source = 'incremental_sync_all_columns' + compare_target = 'incremental_sync_all_columns_target' + self.run_twice_and_assert(select, compare_source, compare_target, project) + + def run_incremental_sync_remove_only(self, project): + select = 'model_a incremental_sync_remove_only incremental_sync_remove_only_target' + compare_source = 'incremental_sync_remove_only' + compare_target = 'incremental_sync_remove_only_target' + self.run_twice_and_assert(select, compare_source, compare_target, project) + + def test_run_incremental_ignore(self, project): + select = 'model_a incremental_ignore incremental_ignore_target' + compare_source = 'incremental_ignore' + compare_target = 'incremental_ignore_target' + self.run_twice_and_assert(select, compare_source, compare_target, project) + + def test_run_incremental_append_new_columns(self, project): + self.run_incremental_append_new_columns(project) + self.run_incremental_append_new_columns_remove_one(project) + + def test_run_incremental_sync_all_columns(self, project): + self.run_incremental_sync_all_columns(project) + self.run_incremental_sync_remove_only(project) + + def test_run_incremental_fail_on_schema_change(self, project): + select = 'model_a incremental_fail' + results_one = run_dbt(['run', '--models', select, '--full-refresh']) + results_two = run_dbt(['run', '--models', select], expect_pass=False) + assert 'Compilation Error' in results_two[1].message diff --git a/tests/functional/incremental_schema_tests/tests/select_from_a.sql b/tests/functional/incremental_schema_tests/tests/select_from_a.sql new file mode 100644 index 00000000000..3dc8f2857bd --- /dev/null +++ b/tests/functional/incremental_schema_tests/tests/select_from_a.sql @@ -0,0 +1 @@ +select * from {{ ref('model_a') }} where false diff --git a/tests/functional/incremental_schema_tests/tests/select_from_incremental_append_new_columns.sql b/tests/functional/incremental_schema_tests/tests/select_from_incremental_append_new_columns.sql new file mode 100644 index 00000000000..947e8458854 --- /dev/null +++ b/tests/functional/incremental_schema_tests/tests/select_from_incremental_append_new_columns.sql @@ -0,0 +1 @@ +select * from {{ ref('incremental_append_new_columns') }} where false \ No newline at end of file diff --git a/tests/functional/incremental_schema_tests/tests/select_from_incremental_append_new_columns_target.sql b/tests/functional/incremental_schema_tests/tests/select_from_incremental_append_new_columns_target.sql new file mode 100644 index 00000000000..8b86eddd71d --- /dev/null +++ b/tests/functional/incremental_schema_tests/tests/select_from_incremental_append_new_columns_target.sql @@ -0,0 +1 @@ +select * from {{ ref('incremental_append_new_columns_target') }} where false \ No newline at end of file diff --git a/tests/functional/incremental_schema_tests/tests/select_from_incremental_ignore.sql b/tests/functional/incremental_schema_tests/tests/select_from_incremental_ignore.sql new file mode 100644 index 00000000000..d565c846465 --- /dev/null +++ b/tests/functional/incremental_schema_tests/tests/select_from_incremental_ignore.sql @@ -0,0 +1 @@ +select * from {{ ref('incremental_ignore') }} where false diff --git a/tests/functional/incremental_schema_tests/tests/select_from_incremental_ignore_target.sql b/tests/functional/incremental_schema_tests/tests/select_from_incremental_ignore_target.sql new file mode 100644 index 00000000000..35d535c5ca5 --- /dev/null +++ b/tests/functional/incremental_schema_tests/tests/select_from_incremental_ignore_target.sql @@ -0,0 +1 @@ +select * from {{ ref('incremental_ignore_target') }} where false \ No newline at end of file diff --git a/tests/functional/incremental_schema_tests/tests/select_from_incremental_sync_all_columns.sql b/tests/functional/incremental_schema_tests/tests/select_from_incremental_sync_all_columns.sql new file mode 100644 index 00000000000..aedc9f80396 --- /dev/null +++ b/tests/functional/incremental_schema_tests/tests/select_from_incremental_sync_all_columns.sql @@ -0,0 +1 @@ +select * from {{ ref('incremental_sync_all_columns') }} where false \ No newline at end of file diff --git a/tests/functional/incremental_schema_tests/tests/select_from_incremental_sync_all_columns_target.sql b/tests/functional/incremental_schema_tests/tests/select_from_incremental_sync_all_columns_target.sql new file mode 100644 index 00000000000..4b703c988bf --- /dev/null +++ b/tests/functional/incremental_schema_tests/tests/select_from_incremental_sync_all_columns_target.sql @@ -0,0 +1 @@ +select * from {{ ref('incremental_sync_all_columns_target') }} where false \ No newline at end of file From 6c348e63c31eabd49017a57c327909de35865438 Mon Sep 17 00:00:00 2001 From: Mila Page Date: Tue, 29 Nov 2022 01:48:54 -0800 Subject: [PATCH 2/4] Drop the old test. --- .../models/incremental_append_new_columns.sql | 29 ------ ...remental_append_new_columns_remove_one.sql | 28 ------ ...l_append_new_columns_remove_one_target.sql | 19 ---- .../incremental_append_new_columns_target.sql | 19 ---- .../models/incremental_fail.sql | 19 ---- .../models/incremental_ignore.sql | 19 ---- .../models/incremental_ignore_target.sql | 15 ---- .../models/incremental_sync_all_columns.sql | 31 ------- .../incremental_sync_all_columns_target.sql | 20 ----- .../models/incremental_sync_remove_only.sql | 29 ------ .../incremental_sync_remove_only_target.sql | 17 ---- .../models/model_a.sql | 22 ----- .../models/schema.yml | 54 ------------ .../test_incremental_schema.py | 88 ------------------- .../tests/select_from_a.sql | 1 - ...ct_from_incremental_append_new_columns.sql | 1 - ..._incremental_append_new_columns_target.sql | 1 - .../tests/select_from_incremental_ignore.sql | 1 - .../select_from_incremental_ignore_target.sql | 1 - ...lect_from_incremental_sync_all_columns.sql | 1 - ...om_incremental_sync_all_columns_target.sql | 1 - 21 files changed, 416 deletions(-) delete mode 100644 test/integration/070_incremental_schema_tests/models/incremental_append_new_columns.sql delete mode 100644 test/integration/070_incremental_schema_tests/models/incremental_append_new_columns_remove_one.sql delete mode 100644 test/integration/070_incremental_schema_tests/models/incremental_append_new_columns_remove_one_target.sql delete mode 100644 test/integration/070_incremental_schema_tests/models/incremental_append_new_columns_target.sql delete mode 100644 test/integration/070_incremental_schema_tests/models/incremental_fail.sql delete mode 100644 test/integration/070_incremental_schema_tests/models/incremental_ignore.sql delete mode 100644 test/integration/070_incremental_schema_tests/models/incremental_ignore_target.sql delete mode 100644 test/integration/070_incremental_schema_tests/models/incremental_sync_all_columns.sql delete mode 100644 test/integration/070_incremental_schema_tests/models/incremental_sync_all_columns_target.sql delete mode 100644 test/integration/070_incremental_schema_tests/models/incremental_sync_remove_only.sql delete mode 100644 test/integration/070_incremental_schema_tests/models/incremental_sync_remove_only_target.sql delete mode 100644 test/integration/070_incremental_schema_tests/models/model_a.sql delete mode 100644 test/integration/070_incremental_schema_tests/models/schema.yml delete mode 100644 test/integration/070_incremental_schema_tests/test_incremental_schema.py delete mode 100644 test/integration/070_incremental_schema_tests/tests/select_from_a.sql delete mode 100644 test/integration/070_incremental_schema_tests/tests/select_from_incremental_append_new_columns.sql delete mode 100644 test/integration/070_incremental_schema_tests/tests/select_from_incremental_append_new_columns_target.sql delete mode 100644 test/integration/070_incremental_schema_tests/tests/select_from_incremental_ignore.sql delete mode 100644 test/integration/070_incremental_schema_tests/tests/select_from_incremental_ignore_target.sql delete mode 100644 test/integration/070_incremental_schema_tests/tests/select_from_incremental_sync_all_columns.sql delete mode 100644 test/integration/070_incremental_schema_tests/tests/select_from_incremental_sync_all_columns_target.sql diff --git a/test/integration/070_incremental_schema_tests/models/incremental_append_new_columns.sql b/test/integration/070_incremental_schema_tests/models/incremental_append_new_columns.sql deleted file mode 100644 index f9eebdcb852..00000000000 --- a/test/integration/070_incremental_schema_tests/models/incremental_append_new_columns.sql +++ /dev/null @@ -1,29 +0,0 @@ -{{ - config( - materialized='incremental', - unique_key='id', - on_schema_change='append_new_columns' - ) -}} - -{% set string_type = 'varchar(10)' %} - -WITH source_data AS (SELECT * FROM {{ ref('model_a') }} ) - -{% if is_incremental() %} - -SELECT id, - cast(field1 as {{string_type}}) as field1, - cast(field2 as {{string_type}}) as field2, - cast(field3 as {{string_type}}) as field3, - cast(field4 as {{string_type}}) as field4 -FROM source_data WHERE id NOT IN (SELECT id from {{ this }} ) - -{% else %} - -SELECT id, - cast(field1 as {{string_type}}) as field1, - cast(field2 as {{string_type}}) as field2 -FROM source_data where id <= 3 - -{% endif %} \ No newline at end of file diff --git a/test/integration/070_incremental_schema_tests/models/incremental_append_new_columns_remove_one.sql b/test/integration/070_incremental_schema_tests/models/incremental_append_new_columns_remove_one.sql deleted file mode 100644 index dbb4962a7e5..00000000000 --- a/test/integration/070_incremental_schema_tests/models/incremental_append_new_columns_remove_one.sql +++ /dev/null @@ -1,28 +0,0 @@ -{{ - config( - materialized='incremental', - unique_key='id', - on_schema_change='append_new_columns' - ) -}} - -{% set string_type = 'varchar(10)' %} - -WITH source_data AS (SELECT * FROM {{ ref('model_a') }} ) - -{% if is_incremental() %} - -SELECT id, - cast(field1 as {{string_type}}) as field1, - cast(field3 as {{string_type}}) as field3, - cast(field4 as {{string_type}}) as field4 -FROM source_data WHERE id NOT IN (SELECT id from {{ this }} ) - -{% else %} - -SELECT id, - cast(field1 as {{string_type}}) as field1, - cast(field2 as {{string_type}}) as field2 -FROM source_data where id <= 3 - -{% endif %} \ No newline at end of file diff --git a/test/integration/070_incremental_schema_tests/models/incremental_append_new_columns_remove_one_target.sql b/test/integration/070_incremental_schema_tests/models/incremental_append_new_columns_remove_one_target.sql deleted file mode 100644 index f3a279f0285..00000000000 --- a/test/integration/070_incremental_schema_tests/models/incremental_append_new_columns_remove_one_target.sql +++ /dev/null @@ -1,19 +0,0 @@ -{{ - config(materialized='table') -}} - -{% set string_type = 'varchar(10)' %} - -with source_data as ( - - select * from {{ ref('model_a') }} - -) - -select id, - cast(field1 as {{string_type}}) as field1, - cast(CASE WHEN id > 3 THEN NULL ELSE field2 END as {{string_type}}) AS field2, - cast(CASE WHEN id <= 3 THEN NULL ELSE field3 END as {{string_type}}) AS field3, - cast(CASE WHEN id <= 3 THEN NULL ELSE field4 END as {{string_type}}) AS field4 - -from source_data \ No newline at end of file diff --git a/test/integration/070_incremental_schema_tests/models/incremental_append_new_columns_target.sql b/test/integration/070_incremental_schema_tests/models/incremental_append_new_columns_target.sql deleted file mode 100644 index 5ff759d7dab..00000000000 --- a/test/integration/070_incremental_schema_tests/models/incremental_append_new_columns_target.sql +++ /dev/null @@ -1,19 +0,0 @@ -{{ - config(materialized='table') -}} - -{% set string_type = 'varchar(10)' %} - -with source_data as ( - - select * from {{ ref('model_a') }} - -) - -select id - ,cast(field1 as {{string_type}}) as field1 - ,cast(field2 as {{string_type}}) as field2 - ,cast(CASE WHEN id <= 3 THEN NULL ELSE field3 END as {{string_type}}) AS field3 - ,cast(CASE WHEN id <= 3 THEN NULL ELSE field4 END as {{string_type}}) AS field4 - -from source_data \ No newline at end of file diff --git a/test/integration/070_incremental_schema_tests/models/incremental_fail.sql b/test/integration/070_incremental_schema_tests/models/incremental_fail.sql deleted file mode 100644 index 590f5b56d97..00000000000 --- a/test/integration/070_incremental_schema_tests/models/incremental_fail.sql +++ /dev/null @@ -1,19 +0,0 @@ -{{ - config( - materialized='incremental', - unique_key='id', - on_schema_change='fail' - ) -}} - -WITH source_data AS (SELECT * FROM {{ ref('model_a') }} ) - -{% if is_incremental() %} - -SELECT id, field1, field2 FROM source_data - -{% else %} - -SELECT id, field1, field3 FROm source_data - -{% endif %} \ No newline at end of file diff --git a/test/integration/070_incremental_schema_tests/models/incremental_ignore.sql b/test/integration/070_incremental_schema_tests/models/incremental_ignore.sql deleted file mode 100644 index 51dee6022fb..00000000000 --- a/test/integration/070_incremental_schema_tests/models/incremental_ignore.sql +++ /dev/null @@ -1,19 +0,0 @@ -{{ - config( - materialized='incremental', - unique_key='id', - on_schema_change='ignore' - ) -}} - -WITH source_data AS (SELECT * FROM {{ ref('model_a') }} ) - -{% if is_incremental() %} - -SELECT id, field1, field2, field3, field4 FROM source_data WHERE id NOT IN (SELECT id from {{ this }} ) - -{% else %} - -SELECT id, field1, field2 FROM source_data LIMIT 3 - -{% endif %} \ No newline at end of file diff --git a/test/integration/070_incremental_schema_tests/models/incremental_ignore_target.sql b/test/integration/070_incremental_schema_tests/models/incremental_ignore_target.sql deleted file mode 100644 index 92d4564e0e8..00000000000 --- a/test/integration/070_incremental_schema_tests/models/incremental_ignore_target.sql +++ /dev/null @@ -1,15 +0,0 @@ -{{ - config(materialized='table') -}} - -with source_data as ( - - select * from {{ ref('model_a') }} - -) - -select id - ,field1 - ,field2 - -from source_data \ No newline at end of file diff --git a/test/integration/070_incremental_schema_tests/models/incremental_sync_all_columns.sql b/test/integration/070_incremental_schema_tests/models/incremental_sync_all_columns.sql deleted file mode 100644 index b742c970419..00000000000 --- a/test/integration/070_incremental_schema_tests/models/incremental_sync_all_columns.sql +++ /dev/null @@ -1,31 +0,0 @@ -{{ - config( - materialized='incremental', - unique_key='id', - on_schema_change='sync_all_columns' - - ) -}} - -WITH source_data AS (SELECT * FROM {{ ref('model_a') }} ) - -{% set string_type = 'varchar(10)' %} - -{% if is_incremental() %} - -SELECT id, - cast(field1 as {{string_type}}) as field1, - cast(field3 as {{string_type}}) as field3, -- to validate new fields - cast(field4 as {{string_type}}) AS field4 -- to validate new fields - -FROM source_data WHERE id NOT IN (SELECT id from {{ this }} ) - -{% else %} - -select id, - cast(field1 as {{string_type}}) as field1, - cast(field2 as {{string_type}}) as field2 - -from source_data where id <= 3 - -{% endif %} \ No newline at end of file diff --git a/test/integration/070_incremental_schema_tests/models/incremental_sync_all_columns_target.sql b/test/integration/070_incremental_schema_tests/models/incremental_sync_all_columns_target.sql deleted file mode 100644 index 6cdbaba5c0d..00000000000 --- a/test/integration/070_incremental_schema_tests/models/incremental_sync_all_columns_target.sql +++ /dev/null @@ -1,20 +0,0 @@ -{{ - config(materialized='table') -}} - -with source_data as ( - - select * from {{ ref('model_a') }} - -) - -{% set string_type = 'varchar(10)' %} - -select id - ,cast(field1 as {{string_type}}) as field1 - --,field2 - ,cast(case when id <= 3 then null else field3 end as {{string_type}}) as field3 - ,cast(case when id <= 3 then null else field4 end as {{string_type}}) as field4 - -from source_data -order by id \ No newline at end of file diff --git a/test/integration/070_incremental_schema_tests/models/incremental_sync_remove_only.sql b/test/integration/070_incremental_schema_tests/models/incremental_sync_remove_only.sql deleted file mode 100644 index 55bae0ad17e..00000000000 --- a/test/integration/070_incremental_schema_tests/models/incremental_sync_remove_only.sql +++ /dev/null @@ -1,29 +0,0 @@ -{{ - config( - materialized='incremental', - unique_key='id', - on_schema_change='sync_all_columns' - - ) -}} - -WITH source_data AS (SELECT * FROM {{ ref('model_a') }} ) - -{% set string_type = 'varchar(10)' %} - -{% if is_incremental() %} - -SELECT id, - cast(field1 as {{string_type}}) as field1 - -FROM source_data WHERE id NOT IN (SELECT id from {{ this }} ) - -{% else %} - -select id, - cast(field1 as {{string_type}}) as field1, - cast(field2 as {{string_type}}) as field2 - -from source_data where id <= 3 - -{% endif %} \ No newline at end of file diff --git a/test/integration/070_incremental_schema_tests/models/incremental_sync_remove_only_target.sql b/test/integration/070_incremental_schema_tests/models/incremental_sync_remove_only_target.sql deleted file mode 100644 index ff88512c6f5..00000000000 --- a/test/integration/070_incremental_schema_tests/models/incremental_sync_remove_only_target.sql +++ /dev/null @@ -1,17 +0,0 @@ -{{ - config(materialized='table') -}} - -with source_data as ( - - select * from {{ ref('model_a') }} - -) - -{% set string_type = 'varchar(10)' %} - -select id - ,cast(field1 as {{string_type}}) as field1 - -from source_data -order by id \ No newline at end of file diff --git a/test/integration/070_incremental_schema_tests/models/model_a.sql b/test/integration/070_incremental_schema_tests/models/model_a.sql deleted file mode 100644 index 2a0b2ddaff2..00000000000 --- a/test/integration/070_incremental_schema_tests/models/model_a.sql +++ /dev/null @@ -1,22 +0,0 @@ -{{ - config(materialized='table') -}} - -with source_data as ( - - select 1 as id, 'aaa' as field1, 'bbb' as field2, 111 as field3, 'TTT' as field4 - union all select 2 as id, 'ccc' as field1, 'ddd' as field2, 222 as field3, 'UUU' as field4 - union all select 3 as id, 'eee' as field1, 'fff' as field2, 333 as field3, 'VVV' as field4 - union all select 4 as id, 'ggg' as field1, 'hhh' as field2, 444 as field3, 'WWW' as field4 - union all select 5 as id, 'iii' as field1, 'jjj' as field2, 555 as field3, 'XXX' as field4 - union all select 6 as id, 'kkk' as field1, 'lll' as field2, 666 as field3, 'YYY' as field4 - -) - -select id - ,field1 - ,field2 - ,field3 - ,field4 - -from source_data \ No newline at end of file diff --git a/test/integration/070_incremental_schema_tests/models/schema.yml b/test/integration/070_incremental_schema_tests/models/schema.yml deleted file mode 100644 index 5546314e413..00000000000 --- a/test/integration/070_incremental_schema_tests/models/schema.yml +++ /dev/null @@ -1,54 +0,0 @@ -version: 2 - -models: - - name: model_a - columns: - - name: id - tags: [column_level_tag] - tests: - - unique - - - name: incremental_ignore - columns: - - name: id - tags: [column_level_tag] - tests: - - unique - - - name: incremental_ignore_target - columns: - - name: id - tags: [column_level_tag] - tests: - - unique - - - name: incremental_append_new_columns - columns: - - name: id - tags: [column_level_tag] - tests: - - unique - - - name: incremental_append_new_columns_target - columns: - - name: id - tags: [column_level_tag] - tests: - - unique - - - name: incremental_sync_all_columns - columns: - - name: id - tags: [column_level_tag] - tests: - - unique - - - name: incremental_sync_all_columns_target - columns: - - name: id - tags: [column_leveL_tag] - tests: - - unique - - - \ No newline at end of file diff --git a/test/integration/070_incremental_schema_tests/test_incremental_schema.py b/test/integration/070_incremental_schema_tests/test_incremental_schema.py deleted file mode 100644 index 09a494b8952..00000000000 --- a/test/integration/070_incremental_schema_tests/test_incremental_schema.py +++ /dev/null @@ -1,88 +0,0 @@ -from test.integration.base import DBTIntegrationTest, use_profile - - -class TestIncrementalSchemaChange(DBTIntegrationTest): - @property - def schema(self): - return "test_incremental_schema_070" - - @property - def models(self): - return "models" - - @property - def project_config(self): - return { - "config-version": 2, - "test-paths": ["tests"] - } - - def run_twice_and_assert( - self, include, compare_source, compare_target - ): - - # dbt run (twice) - run_args = ['run'] - if include: - run_args.extend(('--models', include)) - results_one = self.run_dbt(run_args) - results_two = self.run_dbt(run_args) - - self.assertEqual(len(results_one), 3) - self.assertEqual(len(results_two), 3) - - self.assertTablesEqual(compare_source, compare_target) - - def run_incremental_ignore(self): - select = 'model_a incremental_ignore incremental_ignore_target' - compare_source = 'incremental_ignore' - compare_target = 'incremental_ignore_target' - self.run_twice_and_assert(select, compare_source, compare_target) - - def run_incremental_append_new_columns(self): - select = 'model_a incremental_append_new_columns incremental_append_new_columns_target' - compare_source = 'incremental_append_new_columns' - compare_target = 'incremental_append_new_columns_target' - self.run_twice_and_assert(select, compare_source, compare_target) - - def run_incremental_append_new_columns_remove_one(self): - select = 'model_a incremental_append_new_columns_remove_one incremental_append_new_columns_remove_one_target' - compare_source = 'incremental_append_new_columns_remove_one' - compare_target = 'incremental_append_new_columns_remove_one_target' - self.run_twice_and_assert(select, compare_source, compare_target) - - def run_incremental_sync_all_columns(self): - select = 'model_a incremental_sync_all_columns incremental_sync_all_columns_target' - compare_source = 'incremental_sync_all_columns' - compare_target = 'incremental_sync_all_columns_target' - self.run_twice_and_assert(select, compare_source, compare_target) - - def run_incremental_sync_remove_only(self): - select = 'model_a incremental_sync_remove_only incremental_sync_remove_only_target' - compare_source = 'incremental_sync_remove_only' - compare_target = 'incremental_sync_remove_only_target' - self.run_twice_and_assert(select, compare_source, compare_target) - - def run_incremental_fail_on_schema_change(self): - select = 'model_a incremental_fail' - results_one = self.run_dbt(['run', '--models', select, '--full-refresh']) - results_two = self.run_dbt(['run', '--models', select], expect_pass = False) - self.assertIn('Compilation Error', results_two[1].message) - - @use_profile('postgres') - def test__postgres__run_incremental_ignore(self): - self.run_incremental_ignore() - - @use_profile('postgres') - def test__postgres__run_incremental_append_new_columns(self): - self.run_incremental_append_new_columns() - self.run_incremental_append_new_columns_remove_one() - - @use_profile('postgres') - def test__postgres__run_incremental_sync_all_columns(self): - self.run_incremental_sync_all_columns() - self.run_incremental_sync_remove_only() - - @use_profile('postgres') - def test__postgres__run_incremental_fail_on_schema_change(self): - self.run_incremental_fail_on_schema_change() diff --git a/test/integration/070_incremental_schema_tests/tests/select_from_a.sql b/test/integration/070_incremental_schema_tests/tests/select_from_a.sql deleted file mode 100644 index 3dc8f2857bd..00000000000 --- a/test/integration/070_incremental_schema_tests/tests/select_from_a.sql +++ /dev/null @@ -1 +0,0 @@ -select * from {{ ref('model_a') }} where false diff --git a/test/integration/070_incremental_schema_tests/tests/select_from_incremental_append_new_columns.sql b/test/integration/070_incremental_schema_tests/tests/select_from_incremental_append_new_columns.sql deleted file mode 100644 index 947e8458854..00000000000 --- a/test/integration/070_incremental_schema_tests/tests/select_from_incremental_append_new_columns.sql +++ /dev/null @@ -1 +0,0 @@ -select * from {{ ref('incremental_append_new_columns') }} where false \ No newline at end of file diff --git a/test/integration/070_incremental_schema_tests/tests/select_from_incremental_append_new_columns_target.sql b/test/integration/070_incremental_schema_tests/tests/select_from_incremental_append_new_columns_target.sql deleted file mode 100644 index 8b86eddd71d..00000000000 --- a/test/integration/070_incremental_schema_tests/tests/select_from_incremental_append_new_columns_target.sql +++ /dev/null @@ -1 +0,0 @@ -select * from {{ ref('incremental_append_new_columns_target') }} where false \ No newline at end of file diff --git a/test/integration/070_incremental_schema_tests/tests/select_from_incremental_ignore.sql b/test/integration/070_incremental_schema_tests/tests/select_from_incremental_ignore.sql deleted file mode 100644 index d565c846465..00000000000 --- a/test/integration/070_incremental_schema_tests/tests/select_from_incremental_ignore.sql +++ /dev/null @@ -1 +0,0 @@ -select * from {{ ref('incremental_ignore') }} where false diff --git a/test/integration/070_incremental_schema_tests/tests/select_from_incremental_ignore_target.sql b/test/integration/070_incremental_schema_tests/tests/select_from_incremental_ignore_target.sql deleted file mode 100644 index 35d535c5ca5..00000000000 --- a/test/integration/070_incremental_schema_tests/tests/select_from_incremental_ignore_target.sql +++ /dev/null @@ -1 +0,0 @@ -select * from {{ ref('incremental_ignore_target') }} where false \ No newline at end of file diff --git a/test/integration/070_incremental_schema_tests/tests/select_from_incremental_sync_all_columns.sql b/test/integration/070_incremental_schema_tests/tests/select_from_incremental_sync_all_columns.sql deleted file mode 100644 index aedc9f80396..00000000000 --- a/test/integration/070_incremental_schema_tests/tests/select_from_incremental_sync_all_columns.sql +++ /dev/null @@ -1 +0,0 @@ -select * from {{ ref('incremental_sync_all_columns') }} where false \ No newline at end of file diff --git a/test/integration/070_incremental_schema_tests/tests/select_from_incremental_sync_all_columns_target.sql b/test/integration/070_incremental_schema_tests/tests/select_from_incremental_sync_all_columns_target.sql deleted file mode 100644 index 4b703c988bf..00000000000 --- a/test/integration/070_incremental_schema_tests/tests/select_from_incremental_sync_all_columns_target.sql +++ /dev/null @@ -1 +0,0 @@ -select * from {{ ref('incremental_sync_all_columns_target') }} where false \ No newline at end of file From 38aadb9533d7fa1593f5f7f3d5834cd2d345f234 Mon Sep 17 00:00:00 2001 From: Mila Page Date: Tue, 29 Nov 2022 01:54:10 -0800 Subject: [PATCH 3/4] Bad git add. My disappoint is immeasurable and my day has been ruined. --- .../models/incremental_append_new_columns.sql | 29 ---------- ...remental_append_new_columns_remove_one.sql | 28 ---------- ...l_append_new_columns_remove_one_target.sql | 19 ------- .../incremental_append_new_columns_target.sql | 19 ------- .../models/incremental_fail.sql | 19 ------- .../models/incremental_ignore.sql | 19 ------- .../models/incremental_ignore_target.sql | 15 ------ .../models/incremental_sync_all_columns.sql | 31 ----------- .../incremental_sync_all_columns_target.sql | 20 ------- .../models/incremental_sync_remove_only.sql | 29 ---------- .../incremental_sync_remove_only_target.sql | 17 ------ .../models/model_a.sql | 22 -------- .../models/schema.yml | 54 ------------------- .../tests/select_from_a.sql | 1 - ...ct_from_incremental_append_new_columns.sql | 1 - ..._incremental_append_new_columns_target.sql | 1 - .../tests/select_from_incremental_ignore.sql | 1 - .../select_from_incremental_ignore_target.sql | 1 - ...lect_from_incremental_sync_all_columns.sql | 1 - ...om_incremental_sync_all_columns_target.sql | 1 - 20 files changed, 328 deletions(-) delete mode 100644 tests/functional/incremental_schema_tests/models/incremental_append_new_columns.sql delete mode 100644 tests/functional/incremental_schema_tests/models/incremental_append_new_columns_remove_one.sql delete mode 100644 tests/functional/incremental_schema_tests/models/incremental_append_new_columns_remove_one_target.sql delete mode 100644 tests/functional/incremental_schema_tests/models/incremental_append_new_columns_target.sql delete mode 100644 tests/functional/incremental_schema_tests/models/incremental_fail.sql delete mode 100644 tests/functional/incremental_schema_tests/models/incremental_ignore.sql delete mode 100644 tests/functional/incremental_schema_tests/models/incremental_ignore_target.sql delete mode 100644 tests/functional/incremental_schema_tests/models/incremental_sync_all_columns.sql delete mode 100644 tests/functional/incremental_schema_tests/models/incremental_sync_all_columns_target.sql delete mode 100644 tests/functional/incremental_schema_tests/models/incremental_sync_remove_only.sql delete mode 100644 tests/functional/incremental_schema_tests/models/incremental_sync_remove_only_target.sql delete mode 100644 tests/functional/incremental_schema_tests/models/model_a.sql delete mode 100644 tests/functional/incremental_schema_tests/models/schema.yml delete mode 100644 tests/functional/incremental_schema_tests/tests/select_from_a.sql delete mode 100644 tests/functional/incremental_schema_tests/tests/select_from_incremental_append_new_columns.sql delete mode 100644 tests/functional/incremental_schema_tests/tests/select_from_incremental_append_new_columns_target.sql delete mode 100644 tests/functional/incremental_schema_tests/tests/select_from_incremental_ignore.sql delete mode 100644 tests/functional/incremental_schema_tests/tests/select_from_incremental_ignore_target.sql delete mode 100644 tests/functional/incremental_schema_tests/tests/select_from_incremental_sync_all_columns.sql delete mode 100644 tests/functional/incremental_schema_tests/tests/select_from_incremental_sync_all_columns_target.sql diff --git a/tests/functional/incremental_schema_tests/models/incremental_append_new_columns.sql b/tests/functional/incremental_schema_tests/models/incremental_append_new_columns.sql deleted file mode 100644 index f9eebdcb852..00000000000 --- a/tests/functional/incremental_schema_tests/models/incremental_append_new_columns.sql +++ /dev/null @@ -1,29 +0,0 @@ -{{ - config( - materialized='incremental', - unique_key='id', - on_schema_change='append_new_columns' - ) -}} - -{% set string_type = 'varchar(10)' %} - -WITH source_data AS (SELECT * FROM {{ ref('model_a') }} ) - -{% if is_incremental() %} - -SELECT id, - cast(field1 as {{string_type}}) as field1, - cast(field2 as {{string_type}}) as field2, - cast(field3 as {{string_type}}) as field3, - cast(field4 as {{string_type}}) as field4 -FROM source_data WHERE id NOT IN (SELECT id from {{ this }} ) - -{% else %} - -SELECT id, - cast(field1 as {{string_type}}) as field1, - cast(field2 as {{string_type}}) as field2 -FROM source_data where id <= 3 - -{% endif %} \ No newline at end of file diff --git a/tests/functional/incremental_schema_tests/models/incremental_append_new_columns_remove_one.sql b/tests/functional/incremental_schema_tests/models/incremental_append_new_columns_remove_one.sql deleted file mode 100644 index dbb4962a7e5..00000000000 --- a/tests/functional/incremental_schema_tests/models/incremental_append_new_columns_remove_one.sql +++ /dev/null @@ -1,28 +0,0 @@ -{{ - config( - materialized='incremental', - unique_key='id', - on_schema_change='append_new_columns' - ) -}} - -{% set string_type = 'varchar(10)' %} - -WITH source_data AS (SELECT * FROM {{ ref('model_a') }} ) - -{% if is_incremental() %} - -SELECT id, - cast(field1 as {{string_type}}) as field1, - cast(field3 as {{string_type}}) as field3, - cast(field4 as {{string_type}}) as field4 -FROM source_data WHERE id NOT IN (SELECT id from {{ this }} ) - -{% else %} - -SELECT id, - cast(field1 as {{string_type}}) as field1, - cast(field2 as {{string_type}}) as field2 -FROM source_data where id <= 3 - -{% endif %} \ No newline at end of file diff --git a/tests/functional/incremental_schema_tests/models/incremental_append_new_columns_remove_one_target.sql b/tests/functional/incremental_schema_tests/models/incremental_append_new_columns_remove_one_target.sql deleted file mode 100644 index f3a279f0285..00000000000 --- a/tests/functional/incremental_schema_tests/models/incremental_append_new_columns_remove_one_target.sql +++ /dev/null @@ -1,19 +0,0 @@ -{{ - config(materialized='table') -}} - -{% set string_type = 'varchar(10)' %} - -with source_data as ( - - select * from {{ ref('model_a') }} - -) - -select id, - cast(field1 as {{string_type}}) as field1, - cast(CASE WHEN id > 3 THEN NULL ELSE field2 END as {{string_type}}) AS field2, - cast(CASE WHEN id <= 3 THEN NULL ELSE field3 END as {{string_type}}) AS field3, - cast(CASE WHEN id <= 3 THEN NULL ELSE field4 END as {{string_type}}) AS field4 - -from source_data \ No newline at end of file diff --git a/tests/functional/incremental_schema_tests/models/incremental_append_new_columns_target.sql b/tests/functional/incremental_schema_tests/models/incremental_append_new_columns_target.sql deleted file mode 100644 index 5ff759d7dab..00000000000 --- a/tests/functional/incremental_schema_tests/models/incremental_append_new_columns_target.sql +++ /dev/null @@ -1,19 +0,0 @@ -{{ - config(materialized='table') -}} - -{% set string_type = 'varchar(10)' %} - -with source_data as ( - - select * from {{ ref('model_a') }} - -) - -select id - ,cast(field1 as {{string_type}}) as field1 - ,cast(field2 as {{string_type}}) as field2 - ,cast(CASE WHEN id <= 3 THEN NULL ELSE field3 END as {{string_type}}) AS field3 - ,cast(CASE WHEN id <= 3 THEN NULL ELSE field4 END as {{string_type}}) AS field4 - -from source_data \ No newline at end of file diff --git a/tests/functional/incremental_schema_tests/models/incremental_fail.sql b/tests/functional/incremental_schema_tests/models/incremental_fail.sql deleted file mode 100644 index 590f5b56d97..00000000000 --- a/tests/functional/incremental_schema_tests/models/incremental_fail.sql +++ /dev/null @@ -1,19 +0,0 @@ -{{ - config( - materialized='incremental', - unique_key='id', - on_schema_change='fail' - ) -}} - -WITH source_data AS (SELECT * FROM {{ ref('model_a') }} ) - -{% if is_incremental() %} - -SELECT id, field1, field2 FROM source_data - -{% else %} - -SELECT id, field1, field3 FROm source_data - -{% endif %} \ No newline at end of file diff --git a/tests/functional/incremental_schema_tests/models/incremental_ignore.sql b/tests/functional/incremental_schema_tests/models/incremental_ignore.sql deleted file mode 100644 index 51dee6022fb..00000000000 --- a/tests/functional/incremental_schema_tests/models/incremental_ignore.sql +++ /dev/null @@ -1,19 +0,0 @@ -{{ - config( - materialized='incremental', - unique_key='id', - on_schema_change='ignore' - ) -}} - -WITH source_data AS (SELECT * FROM {{ ref('model_a') }} ) - -{% if is_incremental() %} - -SELECT id, field1, field2, field3, field4 FROM source_data WHERE id NOT IN (SELECT id from {{ this }} ) - -{% else %} - -SELECT id, field1, field2 FROM source_data LIMIT 3 - -{% endif %} \ No newline at end of file diff --git a/tests/functional/incremental_schema_tests/models/incremental_ignore_target.sql b/tests/functional/incremental_schema_tests/models/incremental_ignore_target.sql deleted file mode 100644 index 92d4564e0e8..00000000000 --- a/tests/functional/incremental_schema_tests/models/incremental_ignore_target.sql +++ /dev/null @@ -1,15 +0,0 @@ -{{ - config(materialized='table') -}} - -with source_data as ( - - select * from {{ ref('model_a') }} - -) - -select id - ,field1 - ,field2 - -from source_data \ No newline at end of file diff --git a/tests/functional/incremental_schema_tests/models/incremental_sync_all_columns.sql b/tests/functional/incremental_schema_tests/models/incremental_sync_all_columns.sql deleted file mode 100644 index b742c970419..00000000000 --- a/tests/functional/incremental_schema_tests/models/incremental_sync_all_columns.sql +++ /dev/null @@ -1,31 +0,0 @@ -{{ - config( - materialized='incremental', - unique_key='id', - on_schema_change='sync_all_columns' - - ) -}} - -WITH source_data AS (SELECT * FROM {{ ref('model_a') }} ) - -{% set string_type = 'varchar(10)' %} - -{% if is_incremental() %} - -SELECT id, - cast(field1 as {{string_type}}) as field1, - cast(field3 as {{string_type}}) as field3, -- to validate new fields - cast(field4 as {{string_type}}) AS field4 -- to validate new fields - -FROM source_data WHERE id NOT IN (SELECT id from {{ this }} ) - -{% else %} - -select id, - cast(field1 as {{string_type}}) as field1, - cast(field2 as {{string_type}}) as field2 - -from source_data where id <= 3 - -{% endif %} \ No newline at end of file diff --git a/tests/functional/incremental_schema_tests/models/incremental_sync_all_columns_target.sql b/tests/functional/incremental_schema_tests/models/incremental_sync_all_columns_target.sql deleted file mode 100644 index 6cdbaba5c0d..00000000000 --- a/tests/functional/incremental_schema_tests/models/incremental_sync_all_columns_target.sql +++ /dev/null @@ -1,20 +0,0 @@ -{{ - config(materialized='table') -}} - -with source_data as ( - - select * from {{ ref('model_a') }} - -) - -{% set string_type = 'varchar(10)' %} - -select id - ,cast(field1 as {{string_type}}) as field1 - --,field2 - ,cast(case when id <= 3 then null else field3 end as {{string_type}}) as field3 - ,cast(case when id <= 3 then null else field4 end as {{string_type}}) as field4 - -from source_data -order by id \ No newline at end of file diff --git a/tests/functional/incremental_schema_tests/models/incremental_sync_remove_only.sql b/tests/functional/incremental_schema_tests/models/incremental_sync_remove_only.sql deleted file mode 100644 index 55bae0ad17e..00000000000 --- a/tests/functional/incremental_schema_tests/models/incremental_sync_remove_only.sql +++ /dev/null @@ -1,29 +0,0 @@ -{{ - config( - materialized='incremental', - unique_key='id', - on_schema_change='sync_all_columns' - - ) -}} - -WITH source_data AS (SELECT * FROM {{ ref('model_a') }} ) - -{% set string_type = 'varchar(10)' %} - -{% if is_incremental() %} - -SELECT id, - cast(field1 as {{string_type}}) as field1 - -FROM source_data WHERE id NOT IN (SELECT id from {{ this }} ) - -{% else %} - -select id, - cast(field1 as {{string_type}}) as field1, - cast(field2 as {{string_type}}) as field2 - -from source_data where id <= 3 - -{% endif %} \ No newline at end of file diff --git a/tests/functional/incremental_schema_tests/models/incremental_sync_remove_only_target.sql b/tests/functional/incremental_schema_tests/models/incremental_sync_remove_only_target.sql deleted file mode 100644 index ff88512c6f5..00000000000 --- a/tests/functional/incremental_schema_tests/models/incremental_sync_remove_only_target.sql +++ /dev/null @@ -1,17 +0,0 @@ -{{ - config(materialized='table') -}} - -with source_data as ( - - select * from {{ ref('model_a') }} - -) - -{% set string_type = 'varchar(10)' %} - -select id - ,cast(field1 as {{string_type}}) as field1 - -from source_data -order by id \ No newline at end of file diff --git a/tests/functional/incremental_schema_tests/models/model_a.sql b/tests/functional/incremental_schema_tests/models/model_a.sql deleted file mode 100644 index 2a0b2ddaff2..00000000000 --- a/tests/functional/incremental_schema_tests/models/model_a.sql +++ /dev/null @@ -1,22 +0,0 @@ -{{ - config(materialized='table') -}} - -with source_data as ( - - select 1 as id, 'aaa' as field1, 'bbb' as field2, 111 as field3, 'TTT' as field4 - union all select 2 as id, 'ccc' as field1, 'ddd' as field2, 222 as field3, 'UUU' as field4 - union all select 3 as id, 'eee' as field1, 'fff' as field2, 333 as field3, 'VVV' as field4 - union all select 4 as id, 'ggg' as field1, 'hhh' as field2, 444 as field3, 'WWW' as field4 - union all select 5 as id, 'iii' as field1, 'jjj' as field2, 555 as field3, 'XXX' as field4 - union all select 6 as id, 'kkk' as field1, 'lll' as field2, 666 as field3, 'YYY' as field4 - -) - -select id - ,field1 - ,field2 - ,field3 - ,field4 - -from source_data \ No newline at end of file diff --git a/tests/functional/incremental_schema_tests/models/schema.yml b/tests/functional/incremental_schema_tests/models/schema.yml deleted file mode 100644 index 5546314e413..00000000000 --- a/tests/functional/incremental_schema_tests/models/schema.yml +++ /dev/null @@ -1,54 +0,0 @@ -version: 2 - -models: - - name: model_a - columns: - - name: id - tags: [column_level_tag] - tests: - - unique - - - name: incremental_ignore - columns: - - name: id - tags: [column_level_tag] - tests: - - unique - - - name: incremental_ignore_target - columns: - - name: id - tags: [column_level_tag] - tests: - - unique - - - name: incremental_append_new_columns - columns: - - name: id - tags: [column_level_tag] - tests: - - unique - - - name: incremental_append_new_columns_target - columns: - - name: id - tags: [column_level_tag] - tests: - - unique - - - name: incremental_sync_all_columns - columns: - - name: id - tags: [column_level_tag] - tests: - - unique - - - name: incremental_sync_all_columns_target - columns: - - name: id - tags: [column_leveL_tag] - tests: - - unique - - - \ No newline at end of file diff --git a/tests/functional/incremental_schema_tests/tests/select_from_a.sql b/tests/functional/incremental_schema_tests/tests/select_from_a.sql deleted file mode 100644 index 3dc8f2857bd..00000000000 --- a/tests/functional/incremental_schema_tests/tests/select_from_a.sql +++ /dev/null @@ -1 +0,0 @@ -select * from {{ ref('model_a') }} where false diff --git a/tests/functional/incremental_schema_tests/tests/select_from_incremental_append_new_columns.sql b/tests/functional/incremental_schema_tests/tests/select_from_incremental_append_new_columns.sql deleted file mode 100644 index 947e8458854..00000000000 --- a/tests/functional/incremental_schema_tests/tests/select_from_incremental_append_new_columns.sql +++ /dev/null @@ -1 +0,0 @@ -select * from {{ ref('incremental_append_new_columns') }} where false \ No newline at end of file diff --git a/tests/functional/incremental_schema_tests/tests/select_from_incremental_append_new_columns_target.sql b/tests/functional/incremental_schema_tests/tests/select_from_incremental_append_new_columns_target.sql deleted file mode 100644 index 8b86eddd71d..00000000000 --- a/tests/functional/incremental_schema_tests/tests/select_from_incremental_append_new_columns_target.sql +++ /dev/null @@ -1 +0,0 @@ -select * from {{ ref('incremental_append_new_columns_target') }} where false \ No newline at end of file diff --git a/tests/functional/incremental_schema_tests/tests/select_from_incremental_ignore.sql b/tests/functional/incremental_schema_tests/tests/select_from_incremental_ignore.sql deleted file mode 100644 index d565c846465..00000000000 --- a/tests/functional/incremental_schema_tests/tests/select_from_incremental_ignore.sql +++ /dev/null @@ -1 +0,0 @@ -select * from {{ ref('incremental_ignore') }} where false diff --git a/tests/functional/incremental_schema_tests/tests/select_from_incremental_ignore_target.sql b/tests/functional/incremental_schema_tests/tests/select_from_incremental_ignore_target.sql deleted file mode 100644 index 35d535c5ca5..00000000000 --- a/tests/functional/incremental_schema_tests/tests/select_from_incremental_ignore_target.sql +++ /dev/null @@ -1 +0,0 @@ -select * from {{ ref('incremental_ignore_target') }} where false \ No newline at end of file diff --git a/tests/functional/incremental_schema_tests/tests/select_from_incremental_sync_all_columns.sql b/tests/functional/incremental_schema_tests/tests/select_from_incremental_sync_all_columns.sql deleted file mode 100644 index aedc9f80396..00000000000 --- a/tests/functional/incremental_schema_tests/tests/select_from_incremental_sync_all_columns.sql +++ /dev/null @@ -1 +0,0 @@ -select * from {{ ref('incremental_sync_all_columns') }} where false \ No newline at end of file diff --git a/tests/functional/incremental_schema_tests/tests/select_from_incremental_sync_all_columns_target.sql b/tests/functional/incremental_schema_tests/tests/select_from_incremental_sync_all_columns_target.sql deleted file mode 100644 index 4b703c988bf..00000000000 --- a/tests/functional/incremental_schema_tests/tests/select_from_incremental_sync_all_columns_target.sql +++ /dev/null @@ -1 +0,0 @@ -select * from {{ ref('incremental_sync_all_columns_target') }} where false \ No newline at end of file From cf6a90899d355988a8be54626f73fbbe9cb9b84a Mon Sep 17 00:00:00 2001 From: Mila Page Date: Tue, 29 Nov 2022 12:18:11 -0800 Subject: [PATCH 4/4] Adjustments for flake8. --- tests/functional/incremental_schema_tests/fixtures.py | 2 +- .../incremental_schema_tests/test_incremental_schema.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/functional/incremental_schema_tests/fixtures.py b/tests/functional/incremental_schema_tests/fixtures.py index e6dbd5f27cf..c6eebc5e183 100644 --- a/tests/functional/incremental_schema_tests/fixtures.py +++ b/tests/functional/incremental_schema_tests/fixtures.py @@ -390,6 +390,6 @@ select * from {{ ref('incremental_ignore_target') }} where false """ -_TESTS__SELECT_FROM_INCREMENTAL_APPEND_NEW_COLUMNS = """ +_TESTS__SELECT_FROM_INCREMENTAL_APPEND_NEW_COLUMNS = """ select * from {{ ref('incremental_append_new_columns') }} where false """ diff --git a/tests/functional/incremental_schema_tests/test_incremental_schema.py b/tests/functional/incremental_schema_tests/test_incremental_schema.py index 56e250d0f3b..3ee9e6477e4 100644 --- a/tests/functional/incremental_schema_tests/test_incremental_schema.py +++ b/tests/functional/incremental_schema_tests/test_incremental_schema.py @@ -131,6 +131,6 @@ def test_run_incremental_sync_all_columns(self, project): def test_run_incremental_fail_on_schema_change(self, project): select = 'model_a incremental_fail' - results_one = run_dbt(['run', '--models', select, '--full-refresh']) + run_dbt(['run', '--models', select, '--full-refresh']) results_two = run_dbt(['run', '--models', select], expect_pass=False) assert 'Compilation Error' in results_two[1].message