Spark 3.5: Fix flaky TestSparkExecutorCache (#9583)

apache · Jan 31, 2024 · f8866bf · f8866bf
1 parent a25f77d
commit f8866bf
Showing 1 changed file with 7 additions and 0 deletions.
diff --git a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/TestSparkExecutorCache.java b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/TestSparkExecutorCache.java
@@ -70,10 +70,12 @@
 import org.apache.iceberg.spark.SparkExecutorCache.Conf;
 import org.apache.iceberg.util.CharSequenceSet;
 import org.apache.iceberg.util.Pair;
+import org.apache.spark.SparkEnv;
 import org.apache.spark.sql.Dataset;
 import org.apache.spark.sql.Encoders;
 import org.apache.spark.sql.Row;
 import org.apache.spark.sql.catalyst.analysis.NoSuchTableException;
+import org.apache.spark.storage.memory.MemoryStore;
 import org.junit.jupiter.api.AfterEach;
 import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.TestTemplate;
@@ -391,6 +393,11 @@ private List<DeleteFile> createAndInitTable(String operation, RowLevelOperationM
 
     sql("REFRESH TABLE %s", targetTableName);
 
+    // invalidate the memory store to destroy all currently live table broadcasts
+    SparkEnv sparkEnv = SparkEnv.get();
+    MemoryStore memoryStore = sparkEnv.blockManager().memoryStore();
+    memoryStore.clear();
+
     return ImmutableList.of(posDeleteFile, eqDeleteFile);
   }