From 7e6af113178f9bc030aec619a0fb09ef1805939d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Borja=20V=C3=A1zquez-Barreiros?= Date: Mon, 2 Dec 2024 22:40:03 +0000 Subject: [PATCH 1/4] allowlist microbatch --- .changes/unreleased/Features-20241202-223835.yaml | 6 ++++++ .../bigquery/macros/materializations/incremental.sql | 4 ++-- 2 files changed, 8 insertions(+), 2 deletions(-) create mode 100644 .changes/unreleased/Features-20241202-223835.yaml diff --git a/.changes/unreleased/Features-20241202-223835.yaml b/.changes/unreleased/Features-20241202-223835.yaml new file mode 100644 index 000000000..ab59abd99 --- /dev/null +++ b/.changes/unreleased/Features-20241202-223835.yaml @@ -0,0 +1,6 @@ +kind: Features +body: Allow copy_partitions in microbatch +time: 2024-12-02T22:38:35.479052Z +custom: + Author: borjavb + Issue: "1414" diff --git a/dbt/include/bigquery/macros/materializations/incremental.sql b/dbt/include/bigquery/macros/materializations/incremental.sql index 935280d63..25a83b0c6 100644 --- a/dbt/include/bigquery/macros/materializations/incremental.sql +++ b/dbt/include/bigquery/macros/materializations/incremental.sql @@ -95,9 +95,9 @@ {{ run_hooks(pre_hooks) }} - {% if partition_by.copy_partitions is true and strategy != 'insert_overwrite' %} {#-- We can't copy partitions with merge strategy --#} + {% if partition_by.copy_partitions is true and strategy not in ['insert_overwrite', 'microbatch'] %} {#-- We can't copy partitions with merge strategy --#} {% set wrong_strategy_msg -%} - The 'copy_partitions' option requires the 'incremental_strategy' option to be set to 'insert_overwrite'. + The 'copy_partitions' option requires the 'incremental_strategy' option to be set to 'insert_overwrite' or 'microbatch'. {%- endset %} {% do exceptions.raise_compiler_error(wrong_strategy_msg) %} From d5435d3831cae2d3a9ea14d94eaf4bf1706e0dd4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Borja=20V=C3=A1zquez-Barreiros?= Date: Mon, 2 Dec 2024 22:47:01 +0000 Subject: [PATCH 2/4] add tests --- .../incremental_strategy_fixtures.py | 18 ++++++++++++++++++ .../incremental/test_incremental_microbatch.py | 6 ++++++ 2 files changed, 24 insertions(+) diff --git a/tests/functional/adapter/incremental/incremental_strategy_fixtures.py b/tests/functional/adapter/incremental/incremental_strategy_fixtures.py index 02efbb6c2..d442f948e 100644 --- a/tests/functional/adapter/incremental/incremental_strategy_fixtures.py +++ b/tests/functional/adapter/incremental/incremental_strategy_fixtures.py @@ -611,3 +611,21 @@ }} select * from {{ ref('input_model') }} """ + +microbatch_model_no_unique_id_copy_partitions_sql = """ +{{ config( + materialized='incremental', + incremental_strategy='microbatch', + partition_by={ + 'field': 'event_time', + 'data_type': 'timestamp', + 'granularity': 'day', + 'copy_partitions': true + }, + event_time='event_time', + batch_size='day', + begin=modules.datetime.datetime(2020, 1, 1, 0, 0, 0) + ) +}} +select * from {{ ref('input_model') }} +""" diff --git a/tests/functional/adapter/incremental/test_incremental_microbatch.py b/tests/functional/adapter/incremental/test_incremental_microbatch.py index d1bbbcea3..8ea073477 100644 --- a/tests/functional/adapter/incremental/test_incremental_microbatch.py +++ b/tests/functional/adapter/incremental/test_incremental_microbatch.py @@ -13,6 +13,7 @@ microbatch_input_sql, microbatch_model_no_partition_by_sql, microbatch_model_invalid_partition_by_sql, + microbatch_model_no_unique_id_copy_partitions_sql, ) @@ -53,3 +54,8 @@ def test_execution_failure_no_partition_by(self, project): "The 'microbatch' strategy requires a `partition_by` config with the same granularity as its configured `batch_size`" in stdout ) + +class TestBigQueryMicrobatchWithCopyPartitions(BaseMicrobatch): + @pytest.fixture(scope="class") + def microbatch_model_sql(self) -> str: + return microbatch_model_invalid_partition_by_sql From 634d79b19f286bc4805db2919303fb9eca695d4a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Borja=20V=C3=A1zquez-Barreiros?= Date: Sat, 7 Dec 2024 18:55:11 +0000 Subject: [PATCH 3/4] Update test_incremental_microbatch.py --- .../adapter/incremental/test_incremental_microbatch.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/functional/adapter/incremental/test_incremental_microbatch.py b/tests/functional/adapter/incremental/test_incremental_microbatch.py index a0ed85d8d..6e15d737b 100644 --- a/tests/functional/adapter/incremental/test_incremental_microbatch.py +++ b/tests/functional/adapter/incremental/test_incremental_microbatch.py @@ -81,6 +81,7 @@ def test_execution_failure_no_partition_by(self, project): in stdout ) + class TestBigQueryMicrobatchWithCopyPartitions(BaseMicrobatch): @pytest.fixture(scope="class") def microbatch_model_sql(self) -> str: From a86e930860cfb38b84b9b5c8fe629b2fde6f0df3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Borja=20V=C3=A1zquez-Barreiros?= Date: Sat, 7 Dec 2024 19:43:46 +0000 Subject: [PATCH 4/4] fix text --- .../adapter/incremental/test_incremental_microbatch.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/functional/adapter/incremental/test_incremental_microbatch.py b/tests/functional/adapter/incremental/test_incremental_microbatch.py index 6e15d737b..912f96eec 100644 --- a/tests/functional/adapter/incremental/test_incremental_microbatch.py +++ b/tests/functional/adapter/incremental/test_incremental_microbatch.py @@ -85,4 +85,4 @@ def test_execution_failure_no_partition_by(self, project): class TestBigQueryMicrobatchWithCopyPartitions(BaseMicrobatch): @pytest.fixture(scope="class") def microbatch_model_sql(self) -> str: - return microbatch_model_invalid_partition_by_sql + return microbatch_model_no_unique_id_copy_partitions_sql