diff --git a/.changes/unreleased/Features-20241202-223835.yaml b/.changes/unreleased/Features-20241202-223835.yaml new file mode 100644 index 000000000..ab59abd99 --- /dev/null +++ b/.changes/unreleased/Features-20241202-223835.yaml @@ -0,0 +1,6 @@ +kind: Features +body: Allow copy_partitions in microbatch +time: 2024-12-02T22:38:35.479052Z +custom: + Author: borjavb + Issue: "1414" diff --git a/dbt/include/bigquery/macros/materializations/incremental.sql b/dbt/include/bigquery/macros/materializations/incremental.sql index 935280d63..25a83b0c6 100644 --- a/dbt/include/bigquery/macros/materializations/incremental.sql +++ b/dbt/include/bigquery/macros/materializations/incremental.sql @@ -95,9 +95,9 @@ {{ run_hooks(pre_hooks) }} - {% if partition_by.copy_partitions is true and strategy != 'insert_overwrite' %} {#-- We can't copy partitions with merge strategy --#} + {% if partition_by.copy_partitions is true and strategy not in ['insert_overwrite', 'microbatch'] %} {#-- We can't copy partitions with merge strategy --#} {% set wrong_strategy_msg -%} - The 'copy_partitions' option requires the 'incremental_strategy' option to be set to 'insert_overwrite'. + The 'copy_partitions' option requires the 'incremental_strategy' option to be set to 'insert_overwrite' or 'microbatch'. {%- endset %} {% do exceptions.raise_compiler_error(wrong_strategy_msg) %} diff --git a/tests/functional/adapter/incremental/incremental_strategy_fixtures.py b/tests/functional/adapter/incremental/incremental_strategy_fixtures.py index 365aba8c8..21d5f15b6 100644 --- a/tests/functional/adapter/incremental/incremental_strategy_fixtures.py +++ b/tests/functional/adapter/incremental/incremental_strategy_fixtures.py @@ -629,3 +629,21 @@ }} select * from {{ ref('input_model') }} """ + +microbatch_model_no_unique_id_copy_partitions_sql = """ +{{ config( + materialized='incremental', + incremental_strategy='microbatch', + partition_by={ + 'field': 'event_time', + 'data_type': 'timestamp', + 'granularity': 'day', + 'copy_partitions': true + }, + event_time='event_time', + batch_size='day', + begin=modules.datetime.datetime(2020, 1, 1, 0, 0, 0) + ) +}} +select * from {{ ref('input_model') }} +""" diff --git a/tests/functional/adapter/incremental/test_incremental_microbatch.py b/tests/functional/adapter/incremental/test_incremental_microbatch.py index d0f8b62b7..912f96eec 100644 --- a/tests/functional/adapter/incremental/test_incremental_microbatch.py +++ b/tests/functional/adapter/incremental/test_incremental_microbatch.py @@ -13,6 +13,7 @@ microbatch_input_sql, microbatch_model_no_partition_by_sql, microbatch_model_invalid_partition_by_sql, + microbatch_model_no_unique_id_copy_partitions_sql, microbatch_input_event_time_date_sql, microbatch_input_event_time_datetime_sql, ) @@ -79,3 +80,9 @@ def test_execution_failure_no_partition_by(self, project): "The 'microbatch' strategy requires a `partition_by` config with the same granularity as its configured `batch_size`" in stdout ) + + +class TestBigQueryMicrobatchWithCopyPartitions(BaseMicrobatch): + @pytest.fixture(scope="class") + def microbatch_model_sql(self) -> str: + return microbatch_model_no_unique_id_copy_partitions_sql