[Feature] Allow copy_partitions when using microbatch (#1421)

dbt-labs · Dec 10, 2024 · b8d7224 · b8d7224
1 parent 12df8d6
commit b8d7224
Show file tree

Hide file tree

Showing 4 changed files with 33 additions and 2 deletions.
diff --git a/.changes/unreleased/Features-20241202-223835.yaml b/.changes/unreleased/Features-20241202-223835.yaml
@@ -0,0 +1,6 @@
+kind: Features
+body: Allow copy_partitions in microbatch
+time: 2024-12-02T22:38:35.479052Z
+custom:
+    Author: borjavb
+    Issue: "1414"
diff --git a/dbt/include/bigquery/macros/materializations/incremental.sql b/dbt/include/bigquery/macros/materializations/incremental.sql
@@ -95,9 +95,9 @@
 
   {{ run_hooks(pre_hooks) }}
 
-  {% if partition_by.copy_partitions is true and strategy != 'insert_overwrite' %} {#-- We can't copy partitions with merge strategy --#}
+  {% if partition_by.copy_partitions is true and strategy not in ['insert_overwrite', 'microbatch'] %} {#-- We can't copy partitions with merge strategy --#}
         {% set wrong_strategy_msg -%}
-        The 'copy_partitions' option requires the 'incremental_strategy' option to be set to 'insert_overwrite'.
+        The 'copy_partitions' option requires the 'incremental_strategy' option to be set to 'insert_overwrite' or 'microbatch'.
         {%- endset %}
         {% do exceptions.raise_compiler_error(wrong_strategy_msg) %}
 

diff --git a/tests/functional/adapter/incremental/incremental_strategy_fixtures.py b/tests/functional/adapter/incremental/incremental_strategy_fixtures.py
@@ -629,3 +629,21 @@
 }}
 select * from {{ ref('input_model') }}
 """
+
+microbatch_model_no_unique_id_copy_partitions_sql = """
+{{ config(
+    materialized='incremental',
+    incremental_strategy='microbatch',
+    partition_by={
+      'field': 'event_time',
+      'data_type': 'timestamp',
+      'granularity': 'day',
+      'copy_partitions': true
+    },
+    event_time='event_time',
+    batch_size='day',
+    begin=modules.datetime.datetime(2020, 1, 1, 0, 0, 0)
+    )
+}}
+select * from {{ ref('input_model') }}
+"""
diff --git a/tests/functional/adapter/incremental/test_incremental_microbatch.py b/tests/functional/adapter/incremental/test_incremental_microbatch.py
@@ -13,6 +13,7 @@
     microbatch_input_sql,
     microbatch_model_no_partition_by_sql,
     microbatch_model_invalid_partition_by_sql,
+    microbatch_model_no_unique_id_copy_partitions_sql,
     microbatch_input_event_time_date_sql,
     microbatch_input_event_time_datetime_sql,
 )
@@ -79,3 +80,9 @@ def test_execution_failure_no_partition_by(self, project):
             "The 'microbatch' strategy requires a `partition_by` config with the same granularity as its configured `batch_size`"
             in stdout
         )
+
+
+class TestBigQueryMicrobatchWithCopyPartitions(BaseMicrobatch):
+    @pytest.fixture(scope="class")
+    def microbatch_model_sql(self) -> str:
+        return microbatch_model_no_unique_id_copy_partitions_sql