Skip to content

Commit

Permalink
mv on_schema_change tests -> "adapter zone" (#6618)
Browse files Browse the repository at this point in the history 
* Mv incremental on_schema_change tests to 'adapter zone'

* Use type_string()

* Cleanup
  • Loading branch information
@jtcohen6
jtcohen6 authored Jan 19, 2023
1 parent fa7c4d1 commit b05582d
Show file tree
Hide file tree
Showing 2 changed files with 16 additions and 125 deletions.
105 changes: 8 additions & 97 deletions ...onal/incremental_schema_tests/fixtures.py → ...dbt/tests/adapter/incremental/fixtures.py
View file
Original file line number Diff line number Diff line change
@@ -1,60 +1,3 @@
#
# Properties
#
_PROPERTIES__SCHEMA = """
version: 2
models:
- name: model_a
columns:
- name: id
tags: [column_level_tag]
tests:
- unique
- name: incremental_ignore
columns:
- name: id
tags: [column_level_tag]
tests:
- unique
- name: incremental_ignore_target
columns:
- name: id
tags: [column_level_tag]
tests:
- unique
- name: incremental_append_new_columns
columns:
- name: id
tags: [column_level_tag]
tests:
- unique
- name: incremental_append_new_columns_target
columns:
- name: id
tags: [column_level_tag]
tests:
- unique
- name: incremental_sync_all_columns
columns:
- name: id
tags: [column_level_tag]
tests:
- unique
- name: incremental_sync_all_columns_target
columns:
- name: id
tags: [column_leveL_tag]
tests:
- unique
"""

#
# Models
#
Expand All @@ -70,7 +13,7 @@
WITH source_data AS (SELECT * FROM {{ ref('model_a') }} )
{% set string_type = 'varchar(10)' %}
{% set string_type = dbt.type_string() %}
{% if is_incremental() %}
Expand Down Expand Up @@ -123,7 +66,7 @@
)
{% set string_type = 'varchar(10)' %}
{% set string_type = dbt.type_string() %}
select id
,cast(field1 as {{string_type}}) as field1
Expand Down Expand Up @@ -184,7 +127,7 @@
WITH source_data AS (SELECT * FROM {{ ref('model_a') }} )
{% set string_type = 'varchar(10)' %}
{% set string_type = dbt.type_string() %}
{% if is_incremental() %}
Expand Down Expand Up @@ -215,7 +158,7 @@
)
}}
{% set string_type = 'varchar(10)' %}
{% set string_type = dbt.type_string() %}
WITH source_data AS (SELECT * FROM {{ ref('model_a') }} )
Expand Down Expand Up @@ -267,7 +210,7 @@
config(materialized='table')
}}
{% set string_type = 'varchar(10)' %}
{% set string_type = dbt.type_string() %}
with source_data as (
Expand All @@ -293,7 +236,7 @@
)
}}
{% set string_type = 'varchar(10)' %}
{% set string_type = dbt.type_string() %}
WITH source_data AS (SELECT * FROM {{ ref('model_a') }} )
Expand Down Expand Up @@ -327,7 +270,7 @@
)
{% set string_type = 'varchar(10)' %}
{% set string_type = dbt.type_string() %}
select id
,cast(field1 as {{string_type}}) as field1
Expand All @@ -344,7 +287,7 @@
config(materialized='table')
}}
{% set string_type = 'varchar(10)' %}
{% set string_type = dbt.type_string() %}
with source_data as (
Expand All @@ -360,35 +303,3 @@
from source_data
"""

#
# Tests
#

_TESTS__SELECT_FROM_INCREMENTAL_IGNORE = """
select * from {{ ref('incremental_ignore') }} where false
"""

_TESTS__SELECT_FROM_A = """
select * from {{ ref('model_a') }} where false
"""

_TESTS__SELECT_FROM_INCREMENTAL_APPEND_NEW_COLUMNS_TARGET = """
select * from {{ ref('incremental_append_new_columns_target') }} where false
"""

_TESTS__SELECT_FROM_INCREMENTAL_SYNC_ALL_COLUMNS = """
select * from {{ ref('incremental_sync_all_columns') }} where false
"""

_TESTS__SELECT_FROM_INCREMENTAL_SYNC_ALL_COLUMNS_TARGET = """
select * from {{ ref('incremental_sync_all_columns_target') }} where false
"""

_TESTS__SELECT_FROM_INCREMENTAL_IGNORE_TARGET = """
select * from {{ ref('incremental_ignore_target') }} where false
"""

_TESTS__SELECT_FROM_INCREMENTAL_APPEND_NEW_COLUMNS = """
select * from {{ ref('incremental_append_new_columns') }} where false
"""
36 changes: 8 additions & 28 deletions ...l_schema_tests/test_incremental_schema.py → ...ntal/test_incremental_on_schema_change.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,8 +5,7 @@
run_dbt,
)

from tests.functional.incremental_schema_tests.fixtures import (
_PROPERTIES__SCHEMA,
from dbt.tests.adapter.incremental.fixtures import (
_MODELS__INCREMENTAL_SYNC_REMOVE_ONLY,
_MODELS__INCREMENTAL_IGNORE,
_MODELS__INCREMENTAL_SYNC_REMOVE_ONLY_TARGET,
Expand All @@ -19,23 +18,10 @@
_MODELS__INCREMENTAL_APPEND_NEW_COLUMNS,
_MODELS__INCREMENTAL_SYNC_ALL_COLUMNS_TARGET,
_MODELS__INCREMENTAL_APPEND_NEW_COLUMNS_REMOVE_ONE_TARGET,
_TESTS__SELECT_FROM_INCREMENTAL_IGNORE,
_TESTS__SELECT_FROM_A,
_TESTS__SELECT_FROM_INCREMENTAL_APPEND_NEW_COLUMNS_TARGET,
_TESTS__SELECT_FROM_INCREMENTAL_SYNC_ALL_COLUMNS,
_TESTS__SELECT_FROM_INCREMENTAL_SYNC_ALL_COLUMNS_TARGET,
_TESTS__SELECT_FROM_INCREMENTAL_IGNORE_TARGET,
_TESTS__SELECT_FROM_INCREMENTAL_APPEND_NEW_COLUMNS,
)


class TestIncrementalSchemaChange:
@pytest.fixture(scope="class")
def properties(self):
return {
"schema.yml": _PROPERTIES__SCHEMA,
}

class BaseIncrementalOnSchemaChangeSetup:
@pytest.fixture(scope="class")
def models(self):
return {
Expand All @@ -53,18 +39,6 @@ def models(self):
"incremental_append_new_columns_remove_one_target.sql": _MODELS__INCREMENTAL_APPEND_NEW_COLUMNS_REMOVE_ONE_TARGET,
}

@pytest.fixture(scope="class")
def tests(self):
return {
"select_from_incremental.sql": _TESTS__SELECT_FROM_INCREMENTAL_IGNORE,
"select_from_a.sql": _TESTS__SELECT_FROM_A,
"select_from_incremental_append_new_columns_target.sql": _TESTS__SELECT_FROM_INCREMENTAL_APPEND_NEW_COLUMNS_TARGET,
"select_from_incremental_sync_all_columns.sql": _TESTS__SELECT_FROM_INCREMENTAL_SYNC_ALL_COLUMNS,
"select_from_incremental_sync_all_columns_target.sql": _TESTS__SELECT_FROM_INCREMENTAL_SYNC_ALL_COLUMNS_TARGET,
"select_from_incremental_ignore_target.sql": _TESTS__SELECT_FROM_INCREMENTAL_IGNORE_TARGET,
"select_from_incremental_append_new_columns.sql": _TESTS__SELECT_FROM_INCREMENTAL_APPEND_NEW_COLUMNS,
}

def run_twice_and_assert(self, include, compare_source, compare_target, project):

# dbt run (twice)
Expand Down Expand Up @@ -103,6 +77,8 @@ def run_incremental_sync_remove_only(self, project):
compare_target = "incremental_sync_remove_only_target"
self.run_twice_and_assert(select, compare_source, compare_target, project)


class BaseIncrementalOnSchemaChange(BaseIncrementalOnSchemaChangeSetup):
def test_run_incremental_ignore(self, project):
select = "model_a incremental_ignore incremental_ignore_target"
compare_source = "incremental_ignore"
Expand All @@ -122,3 +98,7 @@ def test_run_incremental_fail_on_schema_change(self, project):
run_dbt(["run", "--models", select, "--full-refresh"])
results_two = run_dbt(["run", "--models", select], expect_pass=False)
assert "Compilation Error" in results_two[1].message


class TestIncrementalOnSchemaChange(BaseIncrementalOnSchemaChange):
pass

0 comments on commit b05582d

Please sign in to comment.