From ad730d1502512132a9b45074b678f6fc1aa02c54 Mon Sep 17 00:00:00 2001 From: chinmay-bhat <12948588+chinmay-bhat@users.noreply.github.com> Date: Mon, 15 Jan 2024 20:22:13 +0530 Subject: [PATCH] move subclasses of `TestBase` used in older Spark versions to JUnit5 --- .../SparkDistributedDataScanTestBase.java | 45 +++++++++---------- .../TestSparkDistributedDataScanDeletes.java | 45 +++++++++---------- ...stSparkDistributedDataScanFilterFiles.java | 26 +++++------ ...kDistributedDataScanJavaSerialization.java | 13 ++---- ...kDistributedDataScanKryoSerialization.java | 13 ++---- ...TestSparkDistributedDataScanReporting.java | 41 ++++++++--------- 6 files changed, 78 insertions(+), 105 deletions(-) diff --git a/spark/v3.4/spark/src/test/java/org/apache/iceberg/SparkDistributedDataScanTestBase.java b/spark/v3.4/spark/src/test/java/org/apache/iceberg/SparkDistributedDataScanTestBase.java index 47b8dbb1d997..404ba7284606 100644 --- a/spark/v3.4/spark/src/test/java/org/apache/iceberg/SparkDistributedDataScanTestBase.java +++ b/spark/v3.4/spark/src/test/java/org/apache/iceberg/SparkDistributedDataScanTestBase.java @@ -21,46 +21,41 @@ import static org.apache.iceberg.PlanningMode.DISTRIBUTED; import static org.apache.iceberg.PlanningMode.LOCAL; +import java.util.Arrays; +import java.util.List; import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap; import org.apache.iceberg.spark.SparkReadConf; import org.apache.spark.sql.SparkSession; import org.apache.spark.sql.internal.SQLConf; -import org.junit.Before; -import org.junit.runner.RunWith; -import org.junit.runners.Parameterized; -import org.junit.runners.Parameterized.Parameters; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.extension.ExtendWith; -@RunWith(Parameterized.class) +@ExtendWith(ParameterizedTestExtension.class) public abstract class SparkDistributedDataScanTestBase extends DataTableScanTestBase> { @Parameters(name = "formatVersion = {0}, dataMode = {1}, deleteMode = {2}") - public static Object[] parameters() { - return new Object[][] { - new Object[] {1, LOCAL, LOCAL}, - new Object[] {1, LOCAL, DISTRIBUTED}, - new Object[] {1, DISTRIBUTED, LOCAL}, - new Object[] {1, DISTRIBUTED, DISTRIBUTED}, - new Object[] {2, LOCAL, LOCAL}, - new Object[] {2, LOCAL, DISTRIBUTED}, - new Object[] {2, DISTRIBUTED, LOCAL}, - new Object[] {2, DISTRIBUTED, DISTRIBUTED} - }; + public static List parameters() { + return Arrays.asList( + new Object[] {1, LOCAL, LOCAL}, + new Object[] {1, LOCAL, DISTRIBUTED}, + new Object[] {1, DISTRIBUTED, LOCAL}, + new Object[] {1, DISTRIBUTED, DISTRIBUTED}, + new Object[] {2, LOCAL, LOCAL}, + new Object[] {2, LOCAL, DISTRIBUTED}, + new Object[] {2, DISTRIBUTED, LOCAL}, + new Object[] {2, DISTRIBUTED, DISTRIBUTED}); } protected static SparkSession spark = null; - private final PlanningMode dataMode; - private final PlanningMode deleteMode; + @Parameter(index = 1) + private PlanningMode dataMode; - public SparkDistributedDataScanTestBase( - int formatVersion, PlanningMode dataPlanningMode, PlanningMode deletePlanningMode) { - super(formatVersion); - this.dataMode = dataPlanningMode; - this.deleteMode = deletePlanningMode; - } + @Parameter(index = 2) + private PlanningMode deleteMode; - @Before + @BeforeEach public void configurePlanningModes() { table .updateProperties() diff --git a/spark/v3.4/spark/src/test/java/org/apache/iceberg/TestSparkDistributedDataScanDeletes.java b/spark/v3.4/spark/src/test/java/org/apache/iceberg/TestSparkDistributedDataScanDeletes.java index 8ed37db6426a..9361c63176e0 100644 --- a/spark/v3.4/spark/src/test/java/org/apache/iceberg/TestSparkDistributedDataScanDeletes.java +++ b/spark/v3.4/spark/src/test/java/org/apache/iceberg/TestSparkDistributedDataScanDeletes.java @@ -21,42 +21,39 @@ import static org.apache.iceberg.PlanningMode.DISTRIBUTED; import static org.apache.iceberg.PlanningMode.LOCAL; +import java.util.Arrays; +import java.util.List; import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap; import org.apache.iceberg.spark.SparkReadConf; import org.apache.spark.sql.SparkSession; import org.apache.spark.sql.internal.SQLConf; -import org.junit.AfterClass; -import org.junit.Before; -import org.junit.BeforeClass; -import org.junit.runner.RunWith; -import org.junit.runners.Parameterized; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.extension.ExtendWith; -@RunWith(Parameterized.class) +@ExtendWith(ParameterizedTestExtension.class) public class TestSparkDistributedDataScanDeletes extends DeleteFileIndexTestBase> { - @Parameterized.Parameters(name = "dataMode = {0}, deleteMode = {1}") - public static Object[] parameters() { - return new Object[][] { - new Object[] {LOCAL, LOCAL}, - new Object[] {LOCAL, DISTRIBUTED}, - new Object[] {DISTRIBUTED, LOCAL}, - new Object[] {DISTRIBUTED, DISTRIBUTED} - }; + @Parameters(name = "formatVersion = {0}, dataMode = {1}, deleteMode = {2}") + public static List parameters() { + return Arrays.asList( + new Object[] {2, LOCAL, LOCAL}, + new Object[] {2, LOCAL, DISTRIBUTED}, + new Object[] {2, DISTRIBUTED, LOCAL}, + new Object[] {2, LOCAL, DISTRIBUTED}); } private static SparkSession spark = null; - private final PlanningMode dataMode; - private final PlanningMode deleteMode; + @Parameter(index = 1) + private PlanningMode dataMode; - public TestSparkDistributedDataScanDeletes( - PlanningMode dataPlanningMode, PlanningMode deletePlanningMode) { - this.dataMode = dataPlanningMode; - this.deleteMode = deletePlanningMode; - } + @Parameter(index = 2) + private PlanningMode deleteMode; - @Before + @BeforeEach public void configurePlanningModes() { table .updateProperties() @@ -65,7 +62,7 @@ public void configurePlanningModes() { .commit(); } - @BeforeClass + @BeforeAll public static void startSpark() { TestSparkDistributedDataScanDeletes.spark = SparkSession.builder() @@ -75,7 +72,7 @@ public static void startSpark() { .getOrCreate(); } - @AfterClass + @AfterAll public static void stopSpark() { SparkSession currentSpark = TestSparkDistributedDataScanDeletes.spark; TestSparkDistributedDataScanDeletes.spark = null; diff --git a/spark/v3.4/spark/src/test/java/org/apache/iceberg/TestSparkDistributedDataScanFilterFiles.java b/spark/v3.4/spark/src/test/java/org/apache/iceberg/TestSparkDistributedDataScanFilterFiles.java index 510c130a5824..a218f965ea65 100644 --- a/spark/v3.4/spark/src/test/java/org/apache/iceberg/TestSparkDistributedDataScanFilterFiles.java +++ b/spark/v3.4/spark/src/test/java/org/apache/iceberg/TestSparkDistributedDataScanFilterFiles.java @@ -25,13 +25,11 @@ import org.apache.iceberg.spark.SparkReadConf; import org.apache.spark.sql.SparkSession; import org.apache.spark.sql.internal.SQLConf; -import org.junit.AfterClass; -import org.junit.BeforeClass; -import org.junit.runner.RunWith; -import org.junit.runners.Parameterized; -import org.junit.runners.Parameterized.Parameters; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.extension.ExtendWith; -@RunWith(Parameterized.class) +@ExtendWith(ParameterizedTestExtension.class) public class TestSparkDistributedDataScanFilterFiles extends FilterFilesTestBase> { @@ -51,17 +49,13 @@ public static Object[] parameters() { private static SparkSession spark = null; - private final PlanningMode dataMode; - private final PlanningMode deleteMode; + @Parameter(index = 1) + private PlanningMode dataMode; - public TestSparkDistributedDataScanFilterFiles( - int formatVersion, PlanningMode dataPlanningMode, PlanningMode deletePlanningMode) { - super(formatVersion); - this.dataMode = dataPlanningMode; - this.deleteMode = deletePlanningMode; - } + @Parameter(index = 2) + private PlanningMode deleteMode; - @BeforeClass + @BeforeAll public static void startSpark() { TestSparkDistributedDataScanFilterFiles.spark = SparkSession.builder() @@ -71,7 +65,7 @@ public static void startSpark() { .getOrCreate(); } - @AfterClass + @AfterAll public static void stopSpark() { SparkSession currentSpark = TestSparkDistributedDataScanFilterFiles.spark; TestSparkDistributedDataScanFilterFiles.spark = null; diff --git a/spark/v3.4/spark/src/test/java/org/apache/iceberg/TestSparkDistributedDataScanJavaSerialization.java b/spark/v3.4/spark/src/test/java/org/apache/iceberg/TestSparkDistributedDataScanJavaSerialization.java index ba1096ee36b9..b8bd6fb86747 100644 --- a/spark/v3.4/spark/src/test/java/org/apache/iceberg/TestSparkDistributedDataScanJavaSerialization.java +++ b/spark/v3.4/spark/src/test/java/org/apache/iceberg/TestSparkDistributedDataScanJavaSerialization.java @@ -19,24 +19,19 @@ package org.apache.iceberg; import org.apache.spark.sql.SparkSession; -import org.junit.AfterClass; -import org.junit.BeforeClass; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; public class TestSparkDistributedDataScanJavaSerialization extends SparkDistributedDataScanTestBase { - public TestSparkDistributedDataScanJavaSerialization( - int formatVersion, PlanningMode dataPlanningMode, PlanningMode deletePlanningMode) { - super(formatVersion, dataPlanningMode, deletePlanningMode); - } - - @BeforeClass + @BeforeAll public static void startSpark() { SparkDistributedDataScanTestBase.spark = initSpark("org.apache.spark.serializer.JavaSerializer"); } - @AfterClass + @AfterAll public static void stopSpark() { SparkSession currentSpark = SparkDistributedDataScanTestBase.spark; SparkDistributedDataScanTestBase.spark = null; diff --git a/spark/v3.4/spark/src/test/java/org/apache/iceberg/TestSparkDistributedDataScanKryoSerialization.java b/spark/v3.4/spark/src/test/java/org/apache/iceberg/TestSparkDistributedDataScanKryoSerialization.java index 7a795eb477bd..08d66cccb627 100644 --- a/spark/v3.4/spark/src/test/java/org/apache/iceberg/TestSparkDistributedDataScanKryoSerialization.java +++ b/spark/v3.4/spark/src/test/java/org/apache/iceberg/TestSparkDistributedDataScanKryoSerialization.java @@ -19,24 +19,19 @@ package org.apache.iceberg; import org.apache.spark.sql.SparkSession; -import org.junit.AfterClass; -import org.junit.BeforeClass; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; public class TestSparkDistributedDataScanKryoSerialization extends SparkDistributedDataScanTestBase { - public TestSparkDistributedDataScanKryoSerialization( - int formatVersion, PlanningMode dataPlanningMode, PlanningMode deletePlanningMode) { - super(formatVersion, dataPlanningMode, deletePlanningMode); - } - - @BeforeClass + @BeforeAll public static void startSpark() { SparkDistributedDataScanTestBase.spark = initSpark("org.apache.spark.serializer.KryoSerializer"); } - @AfterClass + @AfterAll public static void stopSpark() { SparkSession currentSpark = SparkDistributedDataScanTestBase.spark; SparkDistributedDataScanTestBase.spark = null; diff --git a/spark/v3.4/spark/src/test/java/org/apache/iceberg/TestSparkDistributedDataScanReporting.java b/spark/v3.4/spark/src/test/java/org/apache/iceberg/TestSparkDistributedDataScanReporting.java index 1ea4f990b272..acd4688440d1 100644 --- a/spark/v3.4/spark/src/test/java/org/apache/iceberg/TestSparkDistributedDataScanReporting.java +++ b/spark/v3.4/spark/src/test/java/org/apache/iceberg/TestSparkDistributedDataScanReporting.java @@ -21,41 +21,38 @@ import static org.apache.iceberg.PlanningMode.DISTRIBUTED; import static org.apache.iceberg.PlanningMode.LOCAL; +import java.util.Arrays; +import java.util.List; import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap; import org.apache.iceberg.spark.SparkReadConf; import org.apache.spark.sql.SparkSession; import org.apache.spark.sql.internal.SQLConf; -import org.junit.AfterClass; -import org.junit.BeforeClass; -import org.junit.runner.RunWith; -import org.junit.runners.Parameterized; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.extension.ExtendWith; -@RunWith(Parameterized.class) +@ExtendWith(ParameterizedTestExtension.class) public class TestSparkDistributedDataScanReporting extends ScanPlanningAndReportingTestBase> { - @Parameterized.Parameters(name = "dataMode = {0}, deleteMode = {1}") - public static Object[] parameters() { - return new Object[][] { - new Object[] {LOCAL, LOCAL}, - new Object[] {LOCAL, DISTRIBUTED}, - new Object[] {DISTRIBUTED, LOCAL}, - new Object[] {DISTRIBUTED, DISTRIBUTED} - }; + @Parameters(name = "formatVersion = {0}, dataMode = {1}, deleteMode = {2}") + public static List parameters() { + return Arrays.asList( + new Object[] {2, LOCAL, LOCAL}, + new Object[] {2, LOCAL, DISTRIBUTED}, + new Object[] {2, DISTRIBUTED, LOCAL}, + new Object[] {2, DISTRIBUTED, DISTRIBUTED}); } private static SparkSession spark = null; - private final PlanningMode dataMode; - private final PlanningMode deleteMode; + @Parameter(index = 1) + private PlanningMode dataMode; - public TestSparkDistributedDataScanReporting( - PlanningMode dataPlanningMode, PlanningMode deletePlanningMode) { - this.dataMode = dataPlanningMode; - this.deleteMode = deletePlanningMode; - } + @Parameter(index = 2) + private PlanningMode deleteMode; - @BeforeClass + @BeforeAll public static void startSpark() { TestSparkDistributedDataScanReporting.spark = SparkSession.builder() @@ -65,7 +62,7 @@ public static void startSpark() { .getOrCreate(); } - @AfterClass + @AfterAll public static void stopSpark() { SparkSession currentSpark = TestSparkDistributedDataScanReporting.spark; TestSparkDistributedDataScanReporting.spark = null;