From 4932b9679ae7af5dfefcca623cc5cf33a66c7ecc Mon Sep 17 00:00:00 2001 From: Michelle Ark Date: Thu, 26 Sep 2024 01:51:21 +0100 Subject: [PATCH 1/4] Microbatch Strategy (#1334) * poc: microbatch using merge * update base tests * use dynamic insert_overwrite under the hood for bigquery * changelog entry * clean up validation + add testing --- .../unreleased/Features-20240925-232238.yaml | 6 ++ .../macros/materializations/incremental.sql | 15 ++++- .../incremental_strategy/microbatch.sql | 28 ++++++++++ .../incremental_strategy_fixtures.py | 56 +++++++++++++++++++ .../test_incremental_microbatch.py | 55 ++++++++++++++++++ 5 files changed, 157 insertions(+), 3 deletions(-) create mode 100644 .changes/unreleased/Features-20240925-232238.yaml create mode 100644 dbt/include/bigquery/macros/materializations/incremental_strategy/microbatch.sql create mode 100644 tests/functional/adapter/incremental/test_incremental_microbatch.py diff --git a/.changes/unreleased/Features-20240925-232238.yaml b/.changes/unreleased/Features-20240925-232238.yaml new file mode 100644 index 000000000..903884196 --- /dev/null +++ b/.changes/unreleased/Features-20240925-232238.yaml @@ -0,0 +1,6 @@ +kind: Features +body: Add Microbatch Strategy to dbt-spark +time: 2024-09-25T23:22:38.216277+01:00 +custom: + Author: michelleark + Issue: "1354" diff --git a/dbt/include/bigquery/macros/materializations/incremental.sql b/dbt/include/bigquery/macros/materializations/incremental.sql index 3908bedc2..935280d63 100644 --- a/dbt/include/bigquery/macros/materializations/incremental.sql +++ b/dbt/include/bigquery/macros/materializations/incremental.sql @@ -4,12 +4,16 @@ {% set invalid_strategy_msg -%} Invalid incremental strategy provided: {{ strategy }} - Expected one of: 'merge', 'insert_overwrite' + Expected one of: 'merge', 'insert_overwrite', 'microbatch' {%- endset %} - {% if strategy not in ['merge', 'insert_overwrite'] %} + {% if strategy not in ['merge', 'insert_overwrite', 'microbatch'] %} {% do exceptions.raise_compiler_error(invalid_strategy_msg) %} {% endif %} + {% if strategy == 'microbatch' %} + {% do bq_validate_microbatch_config(config) %} + {% endif %} + {% do return(strategy) %} {% endmacro %} @@ -48,8 +52,13 @@ tmp_relation, target_relation, sql, unique_key, partition_by, partitions, dest_columns, tmp_relation_exists, copy_partitions ) %} - {% else %} {# strategy == 'merge' #} + {% elif strategy == 'microbatch' %} + {% set build_sql = bq_generate_microbatch_build_sql( + tmp_relation, target_relation, sql, unique_key, partition_by, partitions, dest_columns, tmp_relation_exists, copy_partitions + ) %} + + {% else %} {# strategy == 'merge' #} {% set build_sql = bq_generate_incremental_merge_build_sql( tmp_relation, target_relation, sql, unique_key, partition_by, dest_columns, tmp_relation_exists, incremental_predicates ) %} diff --git a/dbt/include/bigquery/macros/materializations/incremental_strategy/microbatch.sql b/dbt/include/bigquery/macros/materializations/incremental_strategy/microbatch.sql new file mode 100644 index 000000000..d4c4b7453 --- /dev/null +++ b/dbt/include/bigquery/macros/materializations/incremental_strategy/microbatch.sql @@ -0,0 +1,28 @@ +{% macro bq_validate_microbatch_config(config) %} + {% if config.get("partition_by") is none %} + {% set missing_partition_msg -%} + The 'microbatch' strategy requires a `partition_by` config. + {%- endset %} + {% do exceptions.raise_compiler_error(missing_partition_msg) %} + {% endif %} + + {% if config.get("partition_by").granularity != config.get('batch_size') %} + {% set invalid_partition_by_granularity_msg -%} + The 'microbatch' strategy requires a `partition_by` config with the same granularity as its configured `batch_size`. + Got: + `batch_size`: {{ config.get('batch_size') }} + `partition_by.granularity`: {{ config.get("partition_by").granularity }} + {%- endset %} + {% do exceptions.raise_compiler_error(invalid_partition_by_granularity_msg) %} + {% endif %} +{% endmacro %} + +{% macro bq_generate_microbatch_build_sql( + tmp_relation, target_relation, sql, unique_key, partition_by, partitions, dest_columns, tmp_relation_exists, copy_partitions +) %} + {% set build_sql = bq_insert_overwrite_sql( + tmp_relation, target_relation, sql, unique_key, partition_by, partitions, dest_columns, tmp_relation_exists, copy_partitions + ) %} + + {{ return(build_sql) }} +{% endmacro %} diff --git a/tests/functional/adapter/incremental/incremental_strategy_fixtures.py b/tests/functional/adapter/incremental/incremental_strategy_fixtures.py index 17391b48d..02efbb6c2 100644 --- a/tests/functional/adapter/incremental/incremental_strategy_fixtures.py +++ b/tests/functional/adapter/incremental/incremental_strategy_fixtures.py @@ -555,3 +555,59 @@ select * from data """.lstrip() + +microbatch_model_no_unique_id_sql = """ +{{ config( + materialized='incremental', + incremental_strategy='microbatch', + partition_by={ + 'field': 'event_time', + 'data_type': 'timestamp', + 'granularity': 'day' + }, + event_time='event_time', + batch_size='day', + begin=modules.datetime.datetime(2020, 1, 1, 0, 0, 0) + ) +}} +select * from {{ ref('input_model') }} +""" + +microbatch_input_sql = """ +{{ config(materialized='table', event_time='event_time') }} +select 1 as id, TIMESTAMP '2020-01-01 00:00:00-0' as event_time +union all +select 2 as id, TIMESTAMP '2020-01-02 00:00:00-0' as event_time +union all +select 3 as id, TIMESTAMP '2020-01-03 00:00:00-0' as event_time +""" + +microbatch_model_no_partition_by_sql = """ +{{ config( + materialized='incremental', + incremental_strategy='microbatch', + event_time='event_time', + batch_size='day', + begin=modules.datetime.datetime(2020, 1, 1, 0, 0, 0) + ) +}} +select * from {{ ref('input_model') }} +""" + + +microbatch_model_invalid_partition_by_sql = """ +{{ config( + materialized='incremental', + incremental_strategy='microbatch', + event_time='event_time', + batch_size='day', + begin=modules.datetime.datetime(2020, 1, 1, 0, 0, 0), + partition_by={ + 'field': 'event_time', + 'data_type': 'timestamp', + 'granularity': 'hour' + } + ) +}} +select * from {{ ref('input_model') }} +""" diff --git a/tests/functional/adapter/incremental/test_incremental_microbatch.py b/tests/functional/adapter/incremental/test_incremental_microbatch.py new file mode 100644 index 000000000..d1bbbcea3 --- /dev/null +++ b/tests/functional/adapter/incremental/test_incremental_microbatch.py @@ -0,0 +1,55 @@ +import os +import pytest +from unittest import mock + +from dbt.tests.util import run_dbt_and_capture +from dbt.tests.adapter.incremental.test_incremental_microbatch import ( + BaseMicrobatch, + patch_microbatch_end_time, +) + +from tests.functional.adapter.incremental.incremental_strategy_fixtures import ( + microbatch_model_no_unique_id_sql, + microbatch_input_sql, + microbatch_model_no_partition_by_sql, + microbatch_model_invalid_partition_by_sql, +) + + +class TestBigQueryMicrobatch(BaseMicrobatch): + @pytest.fixture(scope="class") + def microbatch_model_sql(self) -> str: + return microbatch_model_no_unique_id_sql + + +class TestBigQueryMicrobatchMissingPartitionBy: + @pytest.fixture(scope="class") + def models(self) -> str: + return { + "microbatch.sql": microbatch_model_no_partition_by_sql, + "input_model.sql": microbatch_input_sql, + } + + @mock.patch.dict(os.environ, {"DBT_EXPERIMENTAL_MICROBATCH": "True"}) + def test_execution_failure_no_partition_by(self, project): + with patch_microbatch_end_time("2020-01-03 13:57:00"): + _, stdout = run_dbt_and_capture(["run"], expect_pass=False) + assert "The 'microbatch' strategy requires a `partition_by` config" in stdout + + +class TestBigQueryMicrobatchInvalidPartitionByGranularity: + @pytest.fixture(scope="class") + def models(self) -> str: + return { + "microbatch.sql": microbatch_model_invalid_partition_by_sql, + "input_model.sql": microbatch_input_sql, + } + + @mock.patch.dict(os.environ, {"DBT_EXPERIMENTAL_MICROBATCH": "True"}) + def test_execution_failure_no_partition_by(self, project): + with patch_microbatch_end_time("2020-01-03 13:57:00"): + _, stdout = run_dbt_and_capture(["run"], expect_pass=False) + assert ( + "The 'microbatch' strategy requires a `partition_by` config with the same granularity as its configured `batch_size`" + in stdout + ) From ae0f91c7a9dac303632eba2a99161251be6396f7 Mon Sep 17 00:00:00 2001 From: Colin Rogers <111200756+colin-rogers-dbt@users.noreply.github.com> Date: Fri, 27 Sep 2024 15:33:24 -0700 Subject: [PATCH 2/4] update dbt-common dependency to 1.10 and dbt-adapters to 1.7 (#1356) * use dynamic schema in test_grant_access_to.py * use dynamic schema in test_grant_access_to.py * revert setup * update dbt-common dependency to 1.10 and dbt-adapters to 1.7 * run integration.yml on change to any *.py file --- .github/workflows/integration.yml | 1 + setup.py | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index fd22b1d9a..43ac4ecb3 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -128,6 +128,7 @@ jobs: - 'tests/**' - 'dev-requirements.txt' - '.github/**' + - '*.py' - name: Generate integration test matrix id: generate-matrix diff --git a/setup.py b/setup.py index 97a5e96a4..ab89a3c39 100644 --- a/setup.py +++ b/setup.py @@ -50,8 +50,8 @@ def _dbt_bigquery_version() -> str: packages=find_namespace_packages(include=["dbt", "dbt.*"]), include_package_data=True, install_requires=[ - "dbt-common>=1.0.4,<2.0", - "dbt-adapters>=1.1.1,<2.0", + "dbt-common>=1.10,<2.0", + "dbt-adapters>=1.7,<2.0", # 3.20 introduced pyarrow>=3.0 under the `pandas` extra "google-cloud-bigquery[pandas]>=3.0,<4.0", "google-cloud-storage~=2.4", From 8833c5450a5f1f59b688a5c488f27864dc02f93c Mon Sep 17 00:00:00 2001 From: Mike Alfare <13974384+mikealfare@users.noreply.github.com> Date: Tue, 1 Oct 2024 21:03:14 -0400 Subject: [PATCH 3/4] Catch google `NotFound` exception as a `DbtDatabaseError` (#1360) * catch google NotFound exception as a DbtDatabaseError * changelog --- .changes/unreleased/Fixes-20241001-193207.yaml | 7 +++++++ dbt/adapters/bigquery/connections.py | 4 ++++ 2 files changed, 11 insertions(+) create mode 100644 .changes/unreleased/Fixes-20241001-193207.yaml diff --git a/.changes/unreleased/Fixes-20241001-193207.yaml b/.changes/unreleased/Fixes-20241001-193207.yaml new file mode 100644 index 000000000..584445a5b --- /dev/null +++ b/.changes/unreleased/Fixes-20241001-193207.yaml @@ -0,0 +1,7 @@ +kind: Fixes +body: Catch additional database error exception, NotFound, as a DbtDatabaseError instead + of defaulting to a DbtRuntimeError +time: 2024-10-01T19:32:07.304353-04:00 +custom: + Author: mikealfare + Issue: "1360" diff --git a/dbt/adapters/bigquery/connections.py b/dbt/adapters/bigquery/connections.py index cdd9d17dc..d3eee3ef3 100644 --- a/dbt/adapters/bigquery/connections.py +++ b/dbt/adapters/bigquery/connections.py @@ -268,6 +268,10 @@ def exception_handler(self, sql): message = "Access denied while running query" self.handle_error(e, message) + except google.cloud.exceptions.NotFound as e: + message = "Not found while running query" + self.handle_error(e, message) + except google.auth.exceptions.RefreshError as e: message = ( "Unable to generate access token, if you're using " From bc93052b81a674cfa5ead0b72427d60b60f79d8b Mon Sep 17 00:00:00 2001 From: Github Build Bot Date: Wed, 2 Oct 2024 01:06:42 +0000 Subject: [PATCH 4/4] Bumping version to 1.9.0b1 and generate changelog --- .bumpversion.cfg | 2 +- .changes/1.9.0-b1.md | 44 +++++++++++++++++ .../Dependencies-20231211-001048.yaml | 0 .../Dependencies-20231220-002130.yaml | 0 .../Dependencies-20231222-002351.yaml | 0 .../Dependencies-20240105-004800.yaml | 0 .../Dependencies-20240429-005158.yaml | 0 .../Dependencies-20240429-005159.yaml | 0 .../Dependencies-20240520-230208.yaml | 0 .../Dependencies-20240718-005755.yaml | 0 .../Dependencies-20240718-005756.yaml | 0 .../Dependencies-20240718-005757.yaml | 0 .../Dependencies-20240719-003740.yaml | 0 .../Features-20240426-105319.yaml | 0 .../Features-20240430-185650.yaml | 0 .../Features-20240501-151902.yaml | 0 .../Features-20240516-125735.yaml | 0 .../Features-20240730-135911.yaml | 0 .../Features-20240925-232238.yaml | 0 .../Fixes-20240120-180818.yaml | 0 .../Fixes-20240201-145323.yaml | 0 .../Fixes-20240226-233024.yaml | 0 .../Fixes-20240426-105224.yaml | 0 .../Fixes-20241001-193207.yaml | 0 .../Under the Hood-20240331-101418.yaml | 0 .../Under the Hood-20240718-193206.yaml | 0 CHANGELOG.md | 48 ++++++++++++++++++- dbt/adapters/bigquery/__version__.py | 2 +- 28 files changed, 92 insertions(+), 4 deletions(-) create mode 100644 .changes/1.9.0-b1.md rename .changes/{unreleased => 1.9.0}/Dependencies-20231211-001048.yaml (100%) rename .changes/{unreleased => 1.9.0}/Dependencies-20231220-002130.yaml (100%) rename .changes/{unreleased => 1.9.0}/Dependencies-20231222-002351.yaml (100%) rename .changes/{unreleased => 1.9.0}/Dependencies-20240105-004800.yaml (100%) rename .changes/{unreleased => 1.9.0}/Dependencies-20240429-005158.yaml (100%) rename .changes/{unreleased => 1.9.0}/Dependencies-20240429-005159.yaml (100%) rename .changes/{unreleased => 1.9.0}/Dependencies-20240520-230208.yaml (100%) rename .changes/{unreleased => 1.9.0}/Dependencies-20240718-005755.yaml (100%) rename .changes/{unreleased => 1.9.0}/Dependencies-20240718-005756.yaml (100%) rename .changes/{unreleased => 1.9.0}/Dependencies-20240718-005757.yaml (100%) rename .changes/{unreleased => 1.9.0}/Dependencies-20240719-003740.yaml (100%) rename .changes/{unreleased => 1.9.0}/Features-20240426-105319.yaml (100%) rename .changes/{unreleased => 1.9.0}/Features-20240430-185650.yaml (100%) rename .changes/{unreleased => 1.9.0}/Features-20240501-151902.yaml (100%) rename .changes/{unreleased => 1.9.0}/Features-20240516-125735.yaml (100%) rename .changes/{unreleased => 1.9.0}/Features-20240730-135911.yaml (100%) rename .changes/{unreleased => 1.9.0}/Features-20240925-232238.yaml (100%) rename .changes/{unreleased => 1.9.0}/Fixes-20240120-180818.yaml (100%) rename .changes/{unreleased => 1.9.0}/Fixes-20240201-145323.yaml (100%) rename .changes/{unreleased => 1.9.0}/Fixes-20240226-233024.yaml (100%) rename .changes/{unreleased => 1.9.0}/Fixes-20240426-105224.yaml (100%) rename .changes/{unreleased => 1.9.0}/Fixes-20241001-193207.yaml (100%) rename .changes/{unreleased => 1.9.0}/Under the Hood-20240331-101418.yaml (100%) rename .changes/{unreleased => 1.9.0}/Under the Hood-20240718-193206.yaml (100%) diff --git a/.bumpversion.cfg b/.bumpversion.cfg index 24e904ac0..bd9430cbe 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 1.9.0a1 +current_version = 1.9.0b1 parse = (?P[\d]+) # major version number \.(?P[\d]+) # minor version number \.(?P[\d]+) # patch version number diff --git a/.changes/1.9.0-b1.md b/.changes/1.9.0-b1.md new file mode 100644 index 000000000..7d0dd2c8f --- /dev/null +++ b/.changes/1.9.0-b1.md @@ -0,0 +1,44 @@ +## dbt-bigquery 1.9.0-b1 - October 02, 2024 + +### Features + +- Add configuration options `enable_list_inference` and `intermediate_format` for python models ([#1047](https://github.com/dbt-labs/dbt-bigquery/issues/1047), [#1114](https://github.com/dbt-labs/dbt-bigquery/issues/1114)) +- Add tests for cross-database `cast` macro ([#1214](https://github.com/dbt-labs/dbt-bigquery/issues/1214)) +- Cross-database `date` macro ([#1221](https://github.com/dbt-labs/dbt-bigquery/issues/1221)) +- Add support for base 64 encoded json keyfile credentials ([#923](https://github.com/dbt-labs/dbt-bigquery/issues/923)) +- Add support for cancelling queries on keyboard interrupt ([#917](https://github.com/dbt-labs/dbt-bigquery/issues/917)) +- Add Microbatch Strategy to dbt-spark ([#1354](https://github.com/dbt-labs/dbt-bigquery/issues/1354)) + +### Fixes + +- Drop intermediate objects created in BigQuery for incremental models ([#1036](https://github.com/dbt-labs/dbt-bigquery/issues/1036)) +- Fix null column index issue during `dbt docs generate` for external tables ([#1079](https://github.com/dbt-labs/dbt-bigquery/issues/1079)) +- make seed delimiter configurable via `field_delimeter` in model config ([#1119](https://github.com/dbt-labs/dbt-bigquery/issues/1119)) +- Default `enableListInference` to `True` for python models to support nested lists ([#1047](https://github.com/dbt-labs/dbt-bigquery/issues/1047), [#1114](https://github.com/dbt-labs/dbt-bigquery/issues/1114)) +- Catch additional database error exception, NotFound, as a DbtDatabaseError instead of defaulting to a DbtRuntimeError ([#1360](https://github.com/dbt-labs/dbt-bigquery/issues/1360)) + +### Under the Hood + +- Lazy load `agate` ([#1162](https://github.com/dbt-labs/dbt-bigquery/issues/1162)) +- Simplify linting environment and dev dependencies ([#1291](https://github.com/dbt-labs/dbt-bigquery/issues/1291)) + +### Dependencies + +- Update pre-commit requirement from ~=3.5 to ~=3.7 ([#1052](https://github.com/dbt-labs/dbt-bigquery/pull/1052)) +- Update freezegun requirement from ~=1.3 to ~=1.4 ([#1062](https://github.com/dbt-labs/dbt-bigquery/pull/1062)) +- Bump mypy from 1.7.1 to 1.8.0 ([#1064](https://github.com/dbt-labs/dbt-bigquery/pull/1064)) +- Update flake8 requirement from ~=6.1 to ~=7.0 ([#1069](https://github.com/dbt-labs/dbt-bigquery/pull/1069)) +- Bump actions/download-artifact from 3 to 4 ([#1209](https://github.com/dbt-labs/dbt-bigquery/pull/1209)) +- Bump actions/upload-artifact from 3 to 4 ([#1210](https://github.com/dbt-labs/dbt-bigquery/pull/1210)) +- Bump ubuntu from 22.04 to 24.04 in /docker ([#1247](https://github.com/dbt-labs/dbt-bigquery/pull/1247)) +- Update pre-commit-hooks requirement from ~=4.5 to ~=4.6 ([#1281](https://github.com/dbt-labs/dbt-bigquery/pull/1281)) +- Update pytest-xdist requirement from ~=3.5 to ~=3.6 ([#1282](https://github.com/dbt-labs/dbt-bigquery/pull/1282)) +- Update flaky requirement from ~=3.7 to ~=3.8 ([#1283](https://github.com/dbt-labs/dbt-bigquery/pull/1283)) +- Update twine requirement from ~=4.0 to ~=5.1 ([#1293](https://github.com/dbt-labs/dbt-bigquery/pull/1293)) + +### Contributors +- [@d-cole](https://github.com/d-cole) ([#917](https://github.com/dbt-labs/dbt-bigquery/issues/917)) +- [@dwreeves](https://github.com/dwreeves) ([#1162](https://github.com/dbt-labs/dbt-bigquery/issues/1162)) +- [@robeleb1](https://github.com/robeleb1) ([#923](https://github.com/dbt-labs/dbt-bigquery/issues/923)) +- [@salimmoulouel](https://github.com/salimmoulouel) ([#1119](https://github.com/dbt-labs/dbt-bigquery/issues/1119)) +- [@vinit2107](https://github.com/vinit2107) ([#1036](https://github.com/dbt-labs/dbt-bigquery/issues/1036)) diff --git a/.changes/unreleased/Dependencies-20231211-001048.yaml b/.changes/1.9.0/Dependencies-20231211-001048.yaml similarity index 100% rename from .changes/unreleased/Dependencies-20231211-001048.yaml rename to .changes/1.9.0/Dependencies-20231211-001048.yaml diff --git a/.changes/unreleased/Dependencies-20231220-002130.yaml b/.changes/1.9.0/Dependencies-20231220-002130.yaml similarity index 100% rename from .changes/unreleased/Dependencies-20231220-002130.yaml rename to .changes/1.9.0/Dependencies-20231220-002130.yaml diff --git a/.changes/unreleased/Dependencies-20231222-002351.yaml b/.changes/1.9.0/Dependencies-20231222-002351.yaml similarity index 100% rename from .changes/unreleased/Dependencies-20231222-002351.yaml rename to .changes/1.9.0/Dependencies-20231222-002351.yaml diff --git a/.changes/unreleased/Dependencies-20240105-004800.yaml b/.changes/1.9.0/Dependencies-20240105-004800.yaml similarity index 100% rename from .changes/unreleased/Dependencies-20240105-004800.yaml rename to .changes/1.9.0/Dependencies-20240105-004800.yaml diff --git a/.changes/unreleased/Dependencies-20240429-005158.yaml b/.changes/1.9.0/Dependencies-20240429-005158.yaml similarity index 100% rename from .changes/unreleased/Dependencies-20240429-005158.yaml rename to .changes/1.9.0/Dependencies-20240429-005158.yaml diff --git a/.changes/unreleased/Dependencies-20240429-005159.yaml b/.changes/1.9.0/Dependencies-20240429-005159.yaml similarity index 100% rename from .changes/unreleased/Dependencies-20240429-005159.yaml rename to .changes/1.9.0/Dependencies-20240429-005159.yaml diff --git a/.changes/unreleased/Dependencies-20240520-230208.yaml b/.changes/1.9.0/Dependencies-20240520-230208.yaml similarity index 100% rename from .changes/unreleased/Dependencies-20240520-230208.yaml rename to .changes/1.9.0/Dependencies-20240520-230208.yaml diff --git a/.changes/unreleased/Dependencies-20240718-005755.yaml b/.changes/1.9.0/Dependencies-20240718-005755.yaml similarity index 100% rename from .changes/unreleased/Dependencies-20240718-005755.yaml rename to .changes/1.9.0/Dependencies-20240718-005755.yaml diff --git a/.changes/unreleased/Dependencies-20240718-005756.yaml b/.changes/1.9.0/Dependencies-20240718-005756.yaml similarity index 100% rename from .changes/unreleased/Dependencies-20240718-005756.yaml rename to .changes/1.9.0/Dependencies-20240718-005756.yaml diff --git a/.changes/unreleased/Dependencies-20240718-005757.yaml b/.changes/1.9.0/Dependencies-20240718-005757.yaml similarity index 100% rename from .changes/unreleased/Dependencies-20240718-005757.yaml rename to .changes/1.9.0/Dependencies-20240718-005757.yaml diff --git a/.changes/unreleased/Dependencies-20240719-003740.yaml b/.changes/1.9.0/Dependencies-20240719-003740.yaml similarity index 100% rename from .changes/unreleased/Dependencies-20240719-003740.yaml rename to .changes/1.9.0/Dependencies-20240719-003740.yaml diff --git a/.changes/unreleased/Features-20240426-105319.yaml b/.changes/1.9.0/Features-20240426-105319.yaml similarity index 100% rename from .changes/unreleased/Features-20240426-105319.yaml rename to .changes/1.9.0/Features-20240426-105319.yaml diff --git a/.changes/unreleased/Features-20240430-185650.yaml b/.changes/1.9.0/Features-20240430-185650.yaml similarity index 100% rename from .changes/unreleased/Features-20240430-185650.yaml rename to .changes/1.9.0/Features-20240430-185650.yaml diff --git a/.changes/unreleased/Features-20240501-151902.yaml b/.changes/1.9.0/Features-20240501-151902.yaml similarity index 100% rename from .changes/unreleased/Features-20240501-151902.yaml rename to .changes/1.9.0/Features-20240501-151902.yaml diff --git a/.changes/unreleased/Features-20240516-125735.yaml b/.changes/1.9.0/Features-20240516-125735.yaml similarity index 100% rename from .changes/unreleased/Features-20240516-125735.yaml rename to .changes/1.9.0/Features-20240516-125735.yaml diff --git a/.changes/unreleased/Features-20240730-135911.yaml b/.changes/1.9.0/Features-20240730-135911.yaml similarity index 100% rename from .changes/unreleased/Features-20240730-135911.yaml rename to .changes/1.9.0/Features-20240730-135911.yaml diff --git a/.changes/unreleased/Features-20240925-232238.yaml b/.changes/1.9.0/Features-20240925-232238.yaml similarity index 100% rename from .changes/unreleased/Features-20240925-232238.yaml rename to .changes/1.9.0/Features-20240925-232238.yaml diff --git a/.changes/unreleased/Fixes-20240120-180818.yaml b/.changes/1.9.0/Fixes-20240120-180818.yaml similarity index 100% rename from .changes/unreleased/Fixes-20240120-180818.yaml rename to .changes/1.9.0/Fixes-20240120-180818.yaml diff --git a/.changes/unreleased/Fixes-20240201-145323.yaml b/.changes/1.9.0/Fixes-20240201-145323.yaml similarity index 100% rename from .changes/unreleased/Fixes-20240201-145323.yaml rename to .changes/1.9.0/Fixes-20240201-145323.yaml diff --git a/.changes/unreleased/Fixes-20240226-233024.yaml b/.changes/1.9.0/Fixes-20240226-233024.yaml similarity index 100% rename from .changes/unreleased/Fixes-20240226-233024.yaml rename to .changes/1.9.0/Fixes-20240226-233024.yaml diff --git a/.changes/unreleased/Fixes-20240426-105224.yaml b/.changes/1.9.0/Fixes-20240426-105224.yaml similarity index 100% rename from .changes/unreleased/Fixes-20240426-105224.yaml rename to .changes/1.9.0/Fixes-20240426-105224.yaml diff --git a/.changes/unreleased/Fixes-20241001-193207.yaml b/.changes/1.9.0/Fixes-20241001-193207.yaml similarity index 100% rename from .changes/unreleased/Fixes-20241001-193207.yaml rename to .changes/1.9.0/Fixes-20241001-193207.yaml diff --git a/.changes/unreleased/Under the Hood-20240331-101418.yaml b/.changes/1.9.0/Under the Hood-20240331-101418.yaml similarity index 100% rename from .changes/unreleased/Under the Hood-20240331-101418.yaml rename to .changes/1.9.0/Under the Hood-20240331-101418.yaml diff --git a/.changes/unreleased/Under the Hood-20240718-193206.yaml b/.changes/1.9.0/Under the Hood-20240718-193206.yaml similarity index 100% rename from .changes/unreleased/Under the Hood-20240718-193206.yaml rename to .changes/1.9.0/Under the Hood-20240718-193206.yaml diff --git a/CHANGELOG.md b/CHANGELOG.md index 4a408c580..b9bda350a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,10 +5,54 @@ - "Breaking changes" listed under a version may require action from end users or external maintainers when upgrading to that version. - Do not edit this file directly. This file is auto-generated using [changie](https://github.com/miniscruff/changie). For details on how to document a change, see [the contributing guide](https://github.com/dbt-labs/dbt-bigquery/blob/main/CONTRIBUTING.md#adding-changelog-entry) +## dbt-bigquery 1.9.0-b1 - October 02, 2024 + +### Features + +- Add configuration options `enable_list_inference` and `intermediate_format` for python models ([#1047](https://github.com/dbt-labs/dbt-bigquery/issues/1047), [#1114](https://github.com/dbt-labs/dbt-bigquery/issues/1114)) +- Add tests for cross-database `cast` macro ([#1214](https://github.com/dbt-labs/dbt-bigquery/issues/1214)) +- Cross-database `date` macro ([#1221](https://github.com/dbt-labs/dbt-bigquery/issues/1221)) +- Add support for base 64 encoded json keyfile credentials ([#923](https://github.com/dbt-labs/dbt-bigquery/issues/923)) +- Add support for cancelling queries on keyboard interrupt ([#917](https://github.com/dbt-labs/dbt-bigquery/issues/917)) +- Add Microbatch Strategy to dbt-spark ([#1354](https://github.com/dbt-labs/dbt-bigquery/issues/1354)) + +### Fixes + +- Drop intermediate objects created in BigQuery for incremental models ([#1036](https://github.com/dbt-labs/dbt-bigquery/issues/1036)) +- Fix null column index issue during `dbt docs generate` for external tables ([#1079](https://github.com/dbt-labs/dbt-bigquery/issues/1079)) +- make seed delimiter configurable via `field_delimeter` in model config ([#1119](https://github.com/dbt-labs/dbt-bigquery/issues/1119)) +- Default `enableListInference` to `True` for python models to support nested lists ([#1047](https://github.com/dbt-labs/dbt-bigquery/issues/1047), [#1114](https://github.com/dbt-labs/dbt-bigquery/issues/1114)) +- Catch additional database error exception, NotFound, as a DbtDatabaseError instead of defaulting to a DbtRuntimeError ([#1360](https://github.com/dbt-labs/dbt-bigquery/issues/1360)) + +### Under the Hood + +- Lazy load `agate` ([#1162](https://github.com/dbt-labs/dbt-bigquery/issues/1162)) +- Simplify linting environment and dev dependencies ([#1291](https://github.com/dbt-labs/dbt-bigquery/issues/1291)) + +### Dependencies + +- Update pre-commit requirement from ~=3.5 to ~=3.7 ([#1052](https://github.com/dbt-labs/dbt-bigquery/pull/1052)) +- Update freezegun requirement from ~=1.3 to ~=1.4 ([#1062](https://github.com/dbt-labs/dbt-bigquery/pull/1062)) +- Bump mypy from 1.7.1 to 1.8.0 ([#1064](https://github.com/dbt-labs/dbt-bigquery/pull/1064)) +- Update flake8 requirement from ~=6.1 to ~=7.0 ([#1069](https://github.com/dbt-labs/dbt-bigquery/pull/1069)) +- Bump actions/download-artifact from 3 to 4 ([#1209](https://github.com/dbt-labs/dbt-bigquery/pull/1209)) +- Bump actions/upload-artifact from 3 to 4 ([#1210](https://github.com/dbt-labs/dbt-bigquery/pull/1210)) +- Bump ubuntu from 22.04 to 24.04 in /docker ([#1247](https://github.com/dbt-labs/dbt-bigquery/pull/1247)) +- Update pre-commit-hooks requirement from ~=4.5 to ~=4.6 ([#1281](https://github.com/dbt-labs/dbt-bigquery/pull/1281)) +- Update pytest-xdist requirement from ~=3.5 to ~=3.6 ([#1282](https://github.com/dbt-labs/dbt-bigquery/pull/1282)) +- Update flaky requirement from ~=3.7 to ~=3.8 ([#1283](https://github.com/dbt-labs/dbt-bigquery/pull/1283)) +- Update twine requirement from ~=4.0 to ~=5.1 ([#1293](https://github.com/dbt-labs/dbt-bigquery/pull/1293)) + +### Contributors +- [@d-cole](https://github.com/d-cole) ([#917](https://github.com/dbt-labs/dbt-bigquery/issues/917)) +- [@dwreeves](https://github.com/dwreeves) ([#1162](https://github.com/dbt-labs/dbt-bigquery/issues/1162)) +- [@robeleb1](https://github.com/robeleb1) ([#923](https://github.com/dbt-labs/dbt-bigquery/issues/923)) +- [@salimmoulouel](https://github.com/salimmoulouel) ([#1119](https://github.com/dbt-labs/dbt-bigquery/issues/1119)) +- [@vinit2107](https://github.com/vinit2107) ([#1036](https://github.com/dbt-labs/dbt-bigquery/issues/1036)) + + ## Previous Releases For information on prior major and minor releases, see their changelogs: -- [1.8](https://github.com/dbt-labs/dbt-bigquery/blob/1.8.latest/CHANGELOG.md) -- [1.7](https://github.com/dbt-labs/dbt-bigquery/blob/1.7.latest/CHANGELOG.md) - [1.6](https://github.com/dbt-labs/dbt-bigquery/blob/1.6.latest/CHANGELOG.md) - [1.5](https://github.com/dbt-labs/dbt-bigquery/blob/1.5.latest/CHANGELOG.md) - [1.4](https://github.com/dbt-labs/dbt-bigquery/blob/1.4.latest/CHANGELOG.md) diff --git a/dbt/adapters/bigquery/__version__.py b/dbt/adapters/bigquery/__version__.py index 6698ed64c..a4077fff2 100644 --- a/dbt/adapters/bigquery/__version__.py +++ b/dbt/adapters/bigquery/__version__.py @@ -1 +1 @@ -version = "1.9.0a1" +version = "1.9.0b1"