Skip to content

Commit

Permalink
Add BaseDocsGenerate and BaseDocsGenReferences Tests (#52)
Browse files Browse the repository at this point in the history 
### Summary

Added these tests as part of the requirements for upgrading to dbt-core
1.2.

### Description
#### BaseDocsGenerate
- Modify second_model.sql to explicitly use alternate schema
- Modify models fixture to use our version of second_model.sql
- Modify verify_catalog from dbt-core to exclude source table
verification
- Modify unique_schema fixture to prepend schema with rav-test folder
- Modify project_config_update fixture to prevent the seed being created
as a view
- Modify profiles fixture to make root_path the same as schema
- Modify expected_catalog fixture to accommodate dremio data types
- Include test_run_and_generate methods so they use our version of
verify_catalog

#### BaseDocsGenReferences
- Modify project_config_update fixture to allow creating a view for
seeds. This is because the ephemeral_summary models looks for the seed
under database.schema (which is only used for views).
- Modify unique_schema, profiles, and expected catalog fixtures for same
reasons as above

#### Utilities
- Modify base_expected_catalog to look for datalake instead of database 
- Modify expected_references_catalog for the same reason above

#### Other
- Relevant imports
- Remove staging snapshot macro

### Related Issue

#43
#36

### Additional Reviewers

@jlarue26 
@ArgusLi
  • Loading branch information
@ravjotbrar
ravjotbrar authored Nov 3, 2022
1 parent d8a3b7a commit a373dd9
Show file tree
Hide file tree
Showing 13 changed files with 977 additions and 613 deletions.
116 changes: 0 additions & 116 deletions dbt/include/dremio/macros/adapters/snapshot.sql
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,119 +28,3 @@ limitations under the License.*/
values ({{ insert_cols_csv }})

{% endmacro %}

{% macro dremio__snapshot_staging_table(strategy, source_sql, target_relation) -%}

with snapshot_query as (

{{ source_sql }}

),

snapshotted_data as (

select *,
{{ strategy.unique_key }} as dbt_unique_key

from {{ target_relation }}
where dbt_valid_to is null

),

insertions_source_data as (

select
*,
{{ strategy.unique_key }} as dbt_unique_key,
{{ strategy.updated_at }} as dbt_updated_at,
{{ strategy.updated_at }} as dbt_valid_from,
nullif({{ strategy.updated_at }}, {{ strategy.updated_at }}) as dbt_valid_to,
{{ strategy.scd_id }} as dbt_scd_id

from snapshot_query
),

updates_source_data as (

select
*,
{{ strategy.unique_key }} as dbt_unique_key,
{{ strategy.updated_at }} as dbt_updated_at,
{{ strategy.updated_at }} as dbt_valid_from,
{{ strategy.updated_at }} as dbt_valid_to

from snapshot_query
),

{%- if strategy.invalidate_hard_deletes %}

deletes_source_data as (

select
*,
{{ strategy.unique_key }} as dbt_unique_key
from snapshot_query
),
{% endif %}

insertions as (

select
'insert' as dbt_change_type,
source_data.*

from insertions_source_data as source_data
left outer join snapshotted_data on snapshotted_data.dbt_unique_key = source_data.dbt_unique_key
where snapshotted_data.dbt_unique_key is null
or (
snapshotted_data.dbt_unique_key is not null
and (
{{ strategy.row_changed }}
)
)

),

updates as (

select
'update' as dbt_change_type,
source_data.*,
snapshotted_data.dbt_scd_id

from updates_source_data as source_data
join snapshotted_data on snapshotted_data.dbt_unique_key = source_data.dbt_unique_key
where (
{{ strategy.row_changed }}
)
)

{%- if strategy.invalidate_hard_deletes -%}
,

deletes as (

select
'delete' as dbt_change_type,
source_data.*,
{{ snapshot_get_time() }} as dbt_valid_from,
{{ snapshot_get_time() }} as dbt_updated_at,
{{ snapshot_get_time() }} as dbt_valid_to,
snapshotted_data.dbt_scd_id

from snapshotted_data
left join deletes_source_data as source_data on snapshotted_data.dbt_unique_key = source_data.dbt_unique_key
where source_data.dbt_unique_key is null
)
{%- endif %}

select * from insertions
union all
select * from updates
{%- if strategy.invalidate_hard_deletes %}
union all
select * from deletes
{%- endif %}

{%- endmacro %}

87 changes: 87 additions & 0 deletions tests/functional/adapter/basic/test_adapter_methods.py
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,87 @@
import pytest
from dbt.tests.adapter.basic.test_adapter_methods import BaseAdapterMethod
from dbt.tests.adapter.basic.test_adapter_methods import models__upstream_sql
from tests.functional.adapter.utils.test_utils import DATALAKE

models__my_model_sql = """
{% set upstream = ref('upstream_view') %}
{% if execute %}
{# don't ever do any of this #}
{%- do adapter.drop_schema(upstream) -%}
{% set existing = adapter.get_relation(upstream.database, upstream.schema, upstream.identifier) %}
{% if existing is not defined %}
{% do exceptions.raise_compiler_error('expected ' ~ ' to not exist, but it did') %}
{% endif %}
{%- do adapter.create_schema(upstream) -%}
{% set sql = create_view_as(upstream, 'select 2 as id') %}
{% do run_query(sql) %}
{% endif %}
select * from {{ upstream }}
"""

models__expected_sql = """
-- make sure this runs after 'model'
-- {{ ref('model_view') }}
select 2 as id
"""


class TestBaseAdapterMethodDremio(BaseAdapterMethod):
@pytest.fixture(scope="class")
def project_config_update(self):
return {
"models": {
"+twin_strategy": "clone",
},
"name": "adapter_methods",
}

@pytest.fixture(scope="class")
def models(self):
return {
"upstream_view.sql": models__upstream_sql,
"expected_view.sql": models__expected_sql,
"model_view.sql": models__my_model_sql,
}

@pytest.fixture(scope="class")
def unique_schema(self, request, prefix) -> str:
test_file = request.module.__name__
# We only want the last part of the name
test_file = test_file.split(".")[-1]
unique_schema = f"{DATALAKE}.{prefix}_{test_file}"
return unique_schema

@pytest.fixture(scope="class")
def dbt_profile_data(
self, unique_schema, dbt_profile_target, profiles_config_update
):
profile = {
"config": {"send_anonymous_usage_stats": False},
"test": {
"outputs": {
"default": {},
},
"target": "default",
},
}
target = dbt_profile_target
target["schema"] = unique_schema
target["root_path"] = unique_schema
profile["test"]["outputs"]["default"] = target

if profiles_config_update:
profile.update(profiles_config_update)
return profile

@pytest.fixture(scope="class")
def equal_tables(self):
return ["model_view", "expected_view"]
165 changes: 165 additions & 0 deletions tests/functional/adapter/basic/test_base_mat.py
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,165 @@
import pytest
from dbt.tests.adapter.basic.test_base import BaseSimpleMaterializations
from tests.functional.adapter.utils.test_utils import (
relation_from_name,
check_relations_equal,
check_relation_types,
)
from dbt.tests.adapter.basic.files import (
base_view_sql,
base_table_sql,
base_materialized_var_sql,
)
from dbt.tests.util import (
run_dbt,
check_result_nodes_by_name,
)
from tests.functional.adapter.utils.test_utils import DATALAKE

# Unable to insert variable into docstring, so "rav-test" is hardcoded
schema_base_yml = """
version: 2
sources:
- name: raw
database: "rav-test"
schema: "{{ target.schema }}"
tables:
- name: seed
identifier: "{{ var('seed_name', 'base') }}"
"""


class TestSimpleMaterializationsDremio(BaseSimpleMaterializations):
@pytest.fixture(scope="class")
def models(self):
return {
"view_model.sql": base_view_sql,
"table_model.sql": base_table_sql,
"swappable.sql": base_materialized_var_sql,
"schema.yml": schema_base_yml,
}

@pytest.fixture(scope="class")
def project_config_update(self):
return {
"models": {
"+twin_strategy": "prevent",
},
"seeds": {"+twin_strategy": "allow"},
"name": "base",
"vars": {"dremio:reflections": "false"},
}

@pytest.fixture(scope="class")
def unique_schema(self, request, prefix) -> str:
test_file = request.module.__name__
# We only want the last part of the name
test_file = test_file.split(".")[-1]
unique_schema = f"{DATALAKE}.{prefix}_{test_file}"
return unique_schema

@pytest.fixture(scope="class")
def dbt_profile_data(
self, unique_schema, dbt_profile_target, profiles_config_update
):
profile = {
"config": {"send_anonymous_usage_stats": False},
"test": {
"outputs": {
"default": {},
},
"target": "default",
},
}
target = dbt_profile_target
target["schema"] = unique_schema
target["root_path"] = unique_schema
profile["test"]["outputs"]["default"] = target

if profiles_config_update:
profile.update(profiles_config_update)
return profile

def test_base(self, project):

# seed command
results = run_dbt(["seed"])
# seed result length
assert len(results) == 1

# run command
results = run_dbt()
# run result length
assert len(results) == 3

# names exist in result nodes
check_result_nodes_by_name(results, ["view_model", "table_model", "swappable"])

# check relation types
expected = {
"base": "table",
"view_model": "view",
"table_model": "table",
"swappable": "table",
}
check_relation_types(project.adapter, expected)

# base table rowcount
relation = relation_from_name(project.adapter, "base")
result = project.run_sql(
f"select count(*) as num_rows from {relation}", fetch="one"
)
assert result[0] == 10

# relations_equal
check_relations_equal(
project.adapter, ["base", "view_model", "table_model", "swappable"]
)

# check relations in catalog
catalog = run_dbt(["docs", "generate"])
assert len(catalog.nodes) == 4
assert len(catalog.sources) == 1

# run_dbt changing materialized_var to view
# required for BigQuery
if project.test_config.get("require_full_refresh", False):
results = run_dbt(
[
"run",
"--full-refresh",
"-m",
"swappable",
"--vars",
"materialized_var: view",
]
)
else:
results = run_dbt(
["run", "-m", "swappable", "--vars", "materialized_var: view"]
)
assert len(results) == 1
# check relation types, swappable is view
expected = {
"base": "table",
"view_model": "view",
"table_model": "table",
"swappable": "view",
}

check_relation_types(project.adapter, expected)

# run_dbt changing materialized_var to incremental
results = run_dbt(
["run", "-m", "swappable", "--vars", "materialized_var: incremental"]
)
assert len(results) == 1

# check relation types, swappable is table
expected = {
"base": "table",
"view_model": "view",
"table_model": "table",
"swappable": "table",
}
check_relation_types(project.adapter, expected)
Loading

0 comments on commit a373dd9

Please sign in to comment.