From 9b933274b6d37bda1758748a49a4e152e01ac5e2 Mon Sep 17 00:00:00 2001 From: matt-winkler Date: Sat, 21 Oct 2023 15:46:01 -0600 Subject: [PATCH 1/6] patch for --show with json --- .changes/unreleased/Fixes-20231021-154237.yaml | 6 ++++++ dbt/adapters/bigquery/connections.py | 11 ++++++++++- 2 files changed, 16 insertions(+), 1 deletion(-) create mode 100644 .changes/unreleased/Fixes-20231021-154237.yaml diff --git a/.changes/unreleased/Fixes-20231021-154237.yaml b/.changes/unreleased/Fixes-20231021-154237.yaml new file mode 100644 index 000000000..b4b54acd9 --- /dev/null +++ b/.changes/unreleased/Fixes-20231021-154237.yaml @@ -0,0 +1,6 @@ +kind: Fixes +body: Patch for json inline --show +time: 2023-10-21T15:42:37.406853-06:00 +custom: + Author: matt.winkler@dbtlabs.com + Issue: "972" diff --git a/dbt/adapters/bigquery/connections.py b/dbt/adapters/bigquery/connections.py index 132854748..44662fdc6 100644 --- a/dbt/adapters/bigquery/connections.py +++ b/dbt/adapters/bigquery/connections.py @@ -15,7 +15,7 @@ import google.auth import google.auth.exceptions -import google.cloud.bigquery +import google.cloud.bigquery as bigquery import google.cloud.exceptions from google.api_core import retry, client_info from google.auth import impersonated_credentials @@ -63,6 +63,15 @@ ) +# Override broken json deserializer for dbt show --inline +def _json_from_json(value, _): + """NOOP string -> string coercion""" + return json.loads(value) + + +bigquery._helpers._CELLDATA_FROM_JSON["JSON"] = _json_from_json + + @lru_cache() def get_bigquery_defaults(scopes=None) -> Tuple[Any, Optional[str]]: """ From a5c9b8c2aa2f92e8aa03d39bf419b5a4f842db63 Mon Sep 17 00:00:00 2001 From: matt-winkler Date: Mon, 23 Oct 2023 08:23:35 -0600 Subject: [PATCH 2/6] update matts handle in changelog --- .changes/unreleased/Fixes-20231023-082312.yaml | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 .changes/unreleased/Fixes-20231023-082312.yaml diff --git a/.changes/unreleased/Fixes-20231023-082312.yaml b/.changes/unreleased/Fixes-20231023-082312.yaml new file mode 100644 index 000000000..368c58e95 --- /dev/null +++ b/.changes/unreleased/Fixes-20231023-082312.yaml @@ -0,0 +1,6 @@ +kind: Fixes +body: Patch for json inline --show +time: 2023-10-23T08:23:12.245223-06:00 +custom: + Author: matt-winkler + Issue: "972" From e49cfc0494b249054c7f30a0c481762c5271ea05 Mon Sep 17 00:00:00 2001 From: matt-winkler <75497565+matt-winkler@users.noreply.github.com> Date: Mon, 23 Oct 2023 10:27:58 -0600 Subject: [PATCH 3/6] Update dbt/adapters/bigquery/connections.py Co-authored-by: colin-rogers-dbt <111200756+colin-rogers-dbt@users.noreply.github.com> --- dbt/adapters/bigquery/connections.py | 1 + 1 file changed, 1 insertion(+) diff --git a/dbt/adapters/bigquery/connections.py b/dbt/adapters/bigquery/connections.py index 44662fdc6..de84e4bf8 100644 --- a/dbt/adapters/bigquery/connections.py +++ b/dbt/adapters/bigquery/connections.py @@ -64,6 +64,7 @@ # Override broken json deserializer for dbt show --inline +# can remove once this is fixed: https://github.com/googleapis/python-bigquery/issues/1500 def _json_from_json(value, _): """NOOP string -> string coercion""" return json.loads(value) From a32baf12d9578dc3ad6aa51ff7a32ba472a3308a Mon Sep 17 00:00:00 2001 From: Colin Date: Mon, 23 Oct 2023 09:32:36 -0700 Subject: [PATCH 4/6] add json struct functional test --- .../adapter/dbt_show/test_dbt_show.py | 35 +++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/tests/functional/adapter/dbt_show/test_dbt_show.py b/tests/functional/adapter/dbt_show/test_dbt_show.py index c60a26aec..76d6efc8e 100644 --- a/tests/functional/adapter/dbt_show/test_dbt_show.py +++ b/tests/functional/adapter/dbt_show/test_dbt_show.py @@ -1,5 +1,29 @@ +import pytest from dbt.tests.adapter.dbt_show.test_dbt_show import BaseShowSqlHeader, BaseShowLimit +from dbt.tests.util import run_dbt + +model_with_json_struct = """ + select * + from ( + select + struct< + k array< + struct + > + >( + [ + struct( + 1 as c1, + to_json(struct(1 as a)) as c2 + ) + ] + ) + as v + ) as model_limit_subq + limit 5 + """ + class TestBigQueryShowLimit(BaseShowLimit): pass @@ -7,3 +31,14 @@ class TestBigQueryShowLimit(BaseShowLimit): class TestBigQueryShowSqlHeader(BaseShowSqlHeader): pass + + +class TestBigQueryShowSqlWorksWithJSONStruct: + @pytest.fixture(scope="class") + def models(self): + return { + "json_struct_model.sql": model_with_json_struct, + } + + def test_sql_header(self, project): + run_dbt(["show", "--select", "json_struct_model"]) From 30f817e595a7138bb059d0924bbfbb94e2100439 Mon Sep 17 00:00:00 2001 From: matt-winkler Date: Mon, 23 Oct 2023 10:52:43 -0600 Subject: [PATCH 5/6] remove old change log --- .changes/unreleased/Fixes-20231021-154237.yaml | 6 ------ 1 file changed, 6 deletions(-) delete mode 100644 .changes/unreleased/Fixes-20231021-154237.yaml diff --git a/.changes/unreleased/Fixes-20231021-154237.yaml b/.changes/unreleased/Fixes-20231021-154237.yaml deleted file mode 100644 index b4b54acd9..000000000 --- a/.changes/unreleased/Fixes-20231021-154237.yaml +++ /dev/null @@ -1,6 +0,0 @@ -kind: Fixes -body: Patch for json inline --show -time: 2023-10-21T15:42:37.406853-06:00 -custom: - Author: matt.winkler@dbtlabs.com - Issue: "972" From b01397e6b5fe61bd2df72dd081988e203fe99f79 Mon Sep 17 00:00:00 2001 From: Colin Date: Mon, 23 Oct 2023 10:33:27 -0700 Subject: [PATCH 6/6] add comment to test --- tests/functional/adapter/dbt_show/test_dbt_show.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/functional/adapter/dbt_show/test_dbt_show.py b/tests/functional/adapter/dbt_show/test_dbt_show.py index 76d6efc8e..203d7031b 100644 --- a/tests/functional/adapter/dbt_show/test_dbt_show.py +++ b/tests/functional/adapter/dbt_show/test_dbt_show.py @@ -33,6 +33,8 @@ class TestBigQueryShowSqlHeader(BaseShowSqlHeader): pass +# Added to check if dbt show works with JSON struct +# Addresses: https://github.com/dbt-labs/dbt-bigquery/issues/972 class TestBigQueryShowSqlWorksWithJSONStruct: @pytest.fixture(scope="class") def models(self):