Spark 3.5: Fix clobbering of files across streaming epochs when query…

… ID is reused (apache#9255)
geruh · Jan 25, 2024 · e8c3b5d · e8c3b5d
1 parent be65e0b
commit e8c3b5d
Showing 1 changed file with 2 additions and 2 deletions.
diff --git a/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/source/SparkWrite.java b/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/source/SparkWrite.java
@@ -673,11 +673,11 @@ public DataWriter<InternalRow> createWriter(int partitionId, long taskId, long e
       Table table = tableBroadcast.value();
       PartitionSpec spec = table.specs().get(outputSpecId);
       FileIO io = table.io();
-
+      String operationId = queryId + "-" + epochId;
       OutputFileFactory fileFactory =
           OutputFileFactory.builderFor(table, partitionId, taskId)
               .format(format)
-              .operationId(queryId)
+              .operationId(operationId)
               .build();
       SparkFileWriterFactory writerFactory =
           SparkFileWriterFactory.builderFor(table)