From b6f45de9dfcd54ba1194dc6cf1be616abc6f10ee Mon Sep 17 00:00:00 2001 From: Yasuhisa Yoshida Date: Wed, 18 Dec 2024 22:56:14 +0900 Subject: [PATCH] fix: populate database and schema values for bigquery in exported dbt sources (#543) * Improve the specification of database and schema when the server type is BigQuery * Make tables appear last * update CHANGELOG --------- Co-authored-by: Dr. Simon Harrer --- CHANGELOG.md | 1 + datacontract/export/dbt_converter.py | 11 ++++++++--- tests/fixtures/dbt/export/datacontract.yaml | 4 ++-- 3 files changed, 11 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2402307a..648a24a7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,6 +17,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed - Modify the arguments to narrow down the import target with `--dbt-model` (#532) - SodaCL: Prevent `KeyError: 'fail'` from happening when testing with SodaCL +- fix: populate database and schema values for bigquery in exported dbt sources (#543) - Fixing the options for importing and exporting to standard output (#544) ## [0.10.15] - 2024-10-26 diff --git a/datacontract/export/dbt_converter.py b/datacontract/export/dbt_converter.py index 48f8e0af..383bf597 100644 --- a/datacontract/export/dbt_converter.py +++ b/datacontract/export/dbt_converter.py @@ -59,7 +59,7 @@ def to_dbt_staging_sql(data_contract_spec: DataContractSpecification, model_name def to_dbt_sources_yaml(data_contract_spec: DataContractSpecification, server: str = None): - source = {"name": data_contract_spec.id, "tables": []} + source = {"name": data_contract_spec.id} dbt = { "version": 2, "sources": [source], @@ -72,9 +72,14 @@ def to_dbt_sources_yaml(data_contract_spec: DataContractSpecification, server: s adapter_type = None if found_server is not None: adapter_type = found_server.type - source["database"] = found_server.database - source["schema"] = found_server.schema_ + if adapter_type == "bigquery": + source["database"] = found_server.project + source["schema"] = found_server.dataset + else: + source["database"] = found_server.database + source["schema"] = found_server.schema_ + source["tables"] = [] for model_key, model_value in data_contract_spec.models.items(): dbt_model = _to_dbt_source_table(model_key, model_value, adapter_type) source["tables"].append(dbt_model) diff --git a/tests/fixtures/dbt/export/datacontract.yaml b/tests/fixtures/dbt/export/datacontract.yaml index a3335891..046ef09a 100644 --- a/tests/fixtures/dbt/export/datacontract.yaml +++ b/tests/fixtures/dbt/export/datacontract.yaml @@ -20,8 +20,8 @@ servers: type: bigquery environment: production account: my-account - database: my-database - schema: my-schema + project: my-database + dataset: my-schema roles: - name: analyst_us description: Access to the data for US region