Merge branch 'main' into feature/python_models_partioning_clustering

.bumpversion.cfg

@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = v1.6.0b3
+current_version = 1.6.0b4
 parse = (?P<major>[\d]+) # major version number
 	\.(?P<minor>[\d]+) # minor version number
 	\.(?P<patch>[\d]+) # patch version number

.changes/1.6.0-b4.md

@@ -0,0 +1,14 @@
+## dbt-bigquery 1.6.0-b4 - June 23, 2023
+
+### Features
+
+- Support model contracts + constraints on nested columns ([#673](https://github.com/dbt-labs/dbt-bigquery/issues/673))
+
+### Fixes
+
+- test model constraints with sql headers ([#7714](https://github.com/dbt-labs/dbt-bigquery/issues/7714))
+- Pass python model timeout to polling operation so model execution times out as expected. ([#577](https://github.com/dbt-labs/dbt-bigquery/issues/577))
+
+### Dependencies
+
+- Rm explicit agate pin, in favor of transitive dependency from dbt-core ([#777](https://github.com/dbt-labs/dbt-bigquery/pull/777))

.changes/1.6.0/Dependencies-20230618-220517.yaml

@@ -0,0 +1,6 @@
+kind: Dependencies
+body: Rm explicit agate pin, in favor of transitive dependency from dbt-core
+time: 2023-06-18T22:05:17.020743-04:00
+custom:
+  Author: jtcohen6
+  PR: "777"

.changes/1.6.0/Fixes-20230601-120430.yaml

@@ -0,0 +1,6 @@
+kind: Fixes
+body: test model constraints with sql headers
+time: 2023-06-01T12:04:30.876751-04:00
+custom:
+  Author: michelleark
+  Issue: "7714"

.changes/1.6.0/Fixes-20230609-132727.yaml

@@ -0,0 +1,7 @@
+kind: Fixes
+body: Pass python model timeout to polling operation so model execution times out
+  as expected.
+time: 2023-06-09T13:27:27.279842-07:00
+custom:
+  Author: colin-rogers-dbt
+  Issue: "577"

.changes/unreleased/Dependencies-20230606-003859.yaml

@@ -0,0 +1,6 @@
+kind: "Dependencies"
+body: "Update types-requests requirement from ~=2.28 to ~=2.31"
+time: 2023-06-06T00:38:59.00000Z
+custom:
+  Author: dependabot[bot]
+  PR: 737

.changes/unreleased/Dependencies-20230606-005912.yaml

@@ -0,0 +1,6 @@
+kind: "Dependencies"
+body: "Update tox requirement from ~=4.4 to ~=4.6"
+time: 2023-06-06T00:59:12.00000Z
+custom:
+  Author: dependabot[bot]
+  PR: 757

.changes/unreleased/Dependencies-20230621-005934.yaml

@@ -0,0 +1,6 @@
+kind: "Dependencies"
+body: "Bump mypy from 1.2.0 to 1.4.0"
+time: 2023-06-21T00:59:34.00000Z
+custom:
+  Author: dependabot[bot]
+  PR: 780

CHANGELOG.md

@@ -5,6 +5,23 @@
 - "Breaking changes" listed under a version may require action from end users or external maintainers when upgrading to that version.
 - Do not edit this file directly. This file is auto-generated using [changie](https://github.com/miniscruff/changie). For details on how to document a change, see [the contributing guide](https://github.com/dbt-labs/dbt-bigquery/blob/main/CONTRIBUTING.md#adding-changelog-entry)
 
+## dbt-bigquery 1.6.0-b4 - June 23, 2023
+
+### Features
+
+- Support model contracts + constraints on nested columns ([#673](https://github.com/dbt-labs/dbt-bigquery/issues/673))
+
+### Fixes
+
+- test model constraints with sql headers ([#7714](https://github.com/dbt-labs/dbt-bigquery/issues/7714))
+- Pass python model timeout to polling operation so model execution times out as expected. ([#577](https://github.com/dbt-labs/dbt-bigquery/issues/577))
+
+### Dependencies
+
+- Rm explicit agate pin, in favor of transitive dependency from dbt-core ([#777](https://github.com/dbt-labs/dbt-bigquery/pull/777))
+
+
+
 ## dbt-bigquery 1.6.0-b3 - June 08, 2023
 
 ### Breaking Changes
@@ -19,8 +36,6 @@
 
 - test foreign key constraint rendering ([#7512](https://github.com/dbt-labs/dbt-bigquery/issues/7512))
 
-
-
 ## dbt-bigquery 1.6.0-b2 - May 25, 2023
 
 ## dbt-bigquery 1.6.0-b1 - May 12, 2023

dbt/adapters/bigquery/__version__.py

@@ -1 +1 @@
-version = "1.6.0b3"
+version = "1.6.0b4"

dbt/adapters/bigquery/python_submissions.py

@@ -1,6 +1,8 @@
 from typing import Dict, Union
 
 from dbt.adapters.base import PythonJobHelper
+from google.api_core.future.polling import POLLING_PREDICATE
+
 from dbt.adapters.bigquery import BigQueryConnectionManager, BigQueryCredentials
 from dbt.adapters.bigquery.connections import DataprocBatchConfig
 from google.api_core import retry
@@ -43,7 +45,9 @@ def __init__(self, parsed_model: Dict, credential: BigQueryCredentials) -> None:
         self.timeout = self.parsed_model["config"].get(
             "timeout", self.credential.job_execution_timeout_seconds or 60 * 60 * 24
         )
-        self.retry = retry.Retry(maximum=10.0, deadline=self.timeout)
+        self.result_polling_policy = retry.Retry(
+            predicate=POLLING_PREDICATE, maximum=10.0, timeout=self.timeout
+        )
         self.client_options = ClientOptions(
             api_endpoint="{}-dataproc.googleapis.com:443".format(self.credential.dataproc_region)
         )
@@ -98,7 +102,7 @@ def _submit_dataproc_job(self) -> dataproc_v1.types.jobs.Job:
                 "job": job,
             }
         )
-        response = operation.result(retry=self.retry)
+        response = operation.result(polling=self.result_polling_policy)
         # check if job failed
         if response.status.state == 6:
             raise ValueError(response.status.details)
@@ -123,7 +127,7 @@ def _submit_dataproc_job(self) -> dataproc_v1.types.jobs.Job:
         operation = self.job_client.create_batch(request=request)  # type: ignore
         # this takes quite a while, waiting on GCP response to resolve
         # (not a google-api-core issue, more likely a dataproc serverless issue)
-        response = operation.result(retry=self.retry)
+        response = operation.result(polling=self.result_polling_policy)
         return response
         # there might be useful results here that we can parse and return
         # Dataproc job output is saved to the Cloud Storage bucket

dev-requirements.txt

@@ -12,7 +12,7 @@ flake8~=6.0
 flaky~=3.7
 freezegun~=1.2
 ipdb~=0.13.13
-mypy==1.2.0  # patch updates have historically introduced breaking changes
+mypy==1.4.0  # patch updates have historically introduced breaking changes
 pip-tools~=6.13
 pre-commit~=3.2
 pre-commit-hooks~=4.4
@@ -22,9 +22,9 @@ pytest-dotenv~=0.5.2
 pytest-logbook~=1.2
 pytest-xdist~=3.2
 pytz~=2023.3
-tox~=4.4
+tox~=4.6
 types-pytz~=2023.3
-types-requests~=2.28
+types-requests~=2.31
 types-protobuf~=4.22
 twine~=4.0
 wheel~=0.40

setup.py

@@ -58,7 +58,7 @@ def _dbt_core_version(plugin_version: str) -> str:
 
 
 package_name = "dbt-bigquery"
-package_version = "1.6.0b3"
+package_version = "1.6.0b4"
 dbt_core_version = _dbt_core_version(_dbt_bigquery_version())
 description = """The BigQuery adapter plugin for dbt"""
 
@@ -78,7 +78,6 @@ def _dbt_core_version(plugin_version: str) -> str:
         "google-cloud-bigquery~=3.0",
         "google-cloud-storage~=2.4",
         "google-cloud-dataproc~=5.0",
-        "agate~=1.6.3",
     ],
     zip_safe=False,
     classifiers=[

tests/functional/adapter/constraints/test_constraints.py

@@ -3,6 +3,8 @@
     BaseTableConstraintsColumnsEqual,
     BaseViewConstraintsColumnsEqual,
     BaseIncrementalConstraintsColumnsEqual,
+    BaseTableContractSqlHeader,
+    BaseIncrementalContractSqlHeader,
     BaseConstraintsRuntimeDdlEnforcement,
     BaseConstraintsRollback,
     BaseIncrementalConstraintsRuntimeDdlEnforcement,
@@ -22,6 +24,7 @@
     my_model_with_quoted_column_name_sql,
     model_schema_yml,
     constrained_model_schema_yml,
+    model_contract_header_schema_yml,
     model_quoted_column_schema_yml,
     model_fk_constraint_schema_yml,
     my_model_wrong_order_depends_on_fk_sql,
@@ -79,9 +82,39 @@
 # - does not support a data type named 'text' (TODO handle this via type translation/aliasing!)
 constraints_yml = model_schema_yml.replace("text", "string")
 model_constraints_yml = constrained_model_schema_yml.replace("text", "string")
+model_contract_header_schema_yml = model_contract_header_schema_yml.replace("text", "string")
 model_fk_constraint_schema_yml = model_fk_constraint_schema_yml.replace("text", "string")
 constrained_model_schema_yml = constrained_model_schema_yml.replace("text", "string")
 
+my_model_contract_sql_header_sql = """
+{{
+  config(
+    materialized = "table"
+  )
+}}
+
+{% call set_sql_header(config) %}
+DECLARE DEMO STRING DEFAULT 'hello world';
+{% endcall %}
+
+SELECT DEMO as column_name
+"""
+
+my_model_incremental_contract_sql_header_sql = """
+{{
+  config(
+    materialized = "incremental",
+    on_schema_change="append_new_columns"
+  )
+}}
+
+{% call set_sql_header(config) %}
+DECLARE DEMO STRING DEFAULT 'hello world';
+{% endcall %}
+
+SELECT DEMO as column_name
+"""
+
 
 class BigQueryColumnEqualSetup:
     @pytest.fixture
@@ -144,6 +177,24 @@ def models(self):
         }
 
 
+class TestBigQueryTableContractsSqlHeader(BaseTableContractSqlHeader):
+    @pytest.fixture(scope="class")
+    def models(self):
+        return {
+            "my_model_contract_sql_header.sql": my_model_contract_sql_header_sql,
+            "constraints_schema.yml": model_contract_header_schema_yml,
+        }
+
+
+class TestBigQueryIncrementalContractsSqlHeader(BaseIncrementalContractSqlHeader):
+    @pytest.fixture(scope="class")
+    def models(self):
+        return {
+            "my_model_contract_sql_header.sql": my_model_incremental_contract_sql_header_sql,
+            "constraints_schema.yml": model_contract_header_schema_yml,
+        }
+
+
 class BaseStructContract:
     @pytest.fixture
     def wrong_schema_data_type(self):

tests/functional/adapter/test_python_model.py

@@ -1,14 +1,38 @@
 import os
 import pytest
-from dbt.tests.util import run_dbt, write_file
+from dbt.tests.util import run_dbt, run_dbt_and_capture, write_file
 import dbt.tests.adapter.python_model.test_python_model as dbt_tests
 
 TEST_SKIP_MESSAGE = (
     "Skipping the Tests since Dataproc serverless is not stable. " "TODO: Fix later"
 )
 
+blocks_for_thirty_sec = """
+def model(dbt, _):
+    dbt.config(
+        materialized='table',
+        timeout=5
+    )
+    import pandas as pd
+    data = {'col_1': [3, 2, 1, 0], 'col_2': ['a', 'b', 'c', 'd']}
+    df = pd.DataFrame.from_dict(data)
+    import time
+    time.sleep(30)
+    return df
+"""
+
+
+class TestPythonModelDataprocTimeoutTest:
+    @pytest.fixture(scope="class")
+    def models(self):
+        return {"30_sec_python_model.py": blocks_for_thirty_sec}
+
+    def test_model_times_out(self, project):
+        result, output = run_dbt_and_capture(["run"], expect_pass=False)
+        assert len(result) == 1
+        assert "Operation did not complete within the designated timeout of 5 seconds." in output
+
 
-@pytest.mark.skip(reason=TEST_SKIP_MESSAGE)
 class TestPythonModelDataproc(dbt_tests.BasePythonModelTests):
     pass
 

tox.ini

@@ -23,7 +23,7 @@ passenv =
     DATAPROC_*
     GCS_BUCKET
 commands =
-  bigquery: {envpython} -m pytest {posargs} -vv tests/functional --profile service_account
+  bigquery: {envpython} -m pytest {posargs} -vv tests/functional -k "not TestPython" --profile service_account
 deps =
   -rdev-requirements.txt
   -e.