From 67071a1a314db4a98225f3500c2f9630ddf251da Mon Sep 17 00:00:00 2001 From: Pankaj Koti Date: Fri, 7 Jun 2024 18:10:36 +0530 Subject: [PATCH] Bring back `dataset` as a required field for BigQuery profile (#1033) In PR #1017, we attempted to remove `dataset` from the required fields list for the BigQuery profile. However, we realised that this is failing BiqQuery dbt operations as it indeed is a required field. Hence, bring back the same as a required field. This is also necessary for building the mock profile where we construct the profile by taking in consideration only the required fields. Closes: #1031 (cherry picked from commit 803776abbeaf1f1f8ea739f7e73c324e7e8edebc) --- cosmos/profiles/bigquery/service_account_keyfile_dict.py | 3 +++ .../profiles/bigquery/test_bq_service_account_keyfile_dict.py | 1 + 2 files changed, 4 insertions(+) diff --git a/cosmos/profiles/bigquery/service_account_keyfile_dict.py b/cosmos/profiles/bigquery/service_account_keyfile_dict.py index 17858d7bb..480db669b 100644 --- a/cosmos/profiles/bigquery/service_account_keyfile_dict.py +++ b/cosmos/profiles/bigquery/service_account_keyfile_dict.py @@ -20,8 +20,11 @@ class GoogleCloudServiceAccountDictProfileMapping(BaseProfileMapping): dbt_profile_type: str = "bigquery" dbt_profile_method: str = "service-account-json" + # Do not remove dataset as a required field form the below list. Although it's observed that it's not a required + # field for some databases like Postgres, it's required for BigQuery. required_fields = [ "project", + "dataset", "keyfile_json", ] diff --git a/tests/profiles/bigquery/test_bq_service_account_keyfile_dict.py b/tests/profiles/bigquery/test_bq_service_account_keyfile_dict.py index 6f0d60b8d..d30c90021 100755 --- a/tests/profiles/bigquery/test_bq_service_account_keyfile_dict.py +++ b/tests/profiles/bigquery/test_bq_service_account_keyfile_dict.py @@ -96,6 +96,7 @@ def test_mock_profile(mock_bigquery_conn_with_dict: Connection): "type": "bigquery", "method": "service-account-json", "project": "mock_value", + "dataset": "mock_value", "threads": 1, "keyfile_json": None, }