Use Dedupe.

artie-labs · Jun 17, 2024 · a8f5de4 · a8f5de4
1 parent b2524fe
commit a8f5de4
Showing 1 changed file with 2 additions and 0 deletions.
diff --git a/clients/bigquery/merge.go b/clients/bigquery/merge.go
@@ -51,6 +51,8 @@ func (s *Store) Merge(tableData *optimization.TableData) error {
 		AdditionalEqualityStrings: additionalEqualityStrings,
 		// BigQuery has DDL quotas.
 		RetryColBackfill: true,
+		// We are using BigQuery's streaming API which doesn't guarantee exactly once semantics
+		SubQueryDedupe: true,
 	})
 }