Skip to content

Commit

Permalink
Spark 3.3, 3.4: Fix incorrect catalog loaded in TestCreateActions
Browse files Browse the repository at this point in the history 
Back-port of #10952
  • Loading branch information
@manuzhang
manuzhang committed Sep 2, 2024
1 parent a07f862 commit 5298462
Show file tree
Hide file tree
Showing 2 changed files with 30 additions and 0 deletions.
15 changes: 15 additions & 0 deletions spark/v3.3/spark/src/test/java/org/apache/iceberg/spark/actions/TestCreateActions.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -183,6 +183,11 @@ public void before() {
public void after() throws IOException {
// Drop the hive table.
spark.sql(String.format("DROP TABLE IF EXISTS %s", baseTableName));
spark.sessionState().catalogManager().reset();
spark.conf().unset("spark.sql.catalog.spark_catalog.type");
spark.conf().unset("spark.sql.catalog.spark_catalog.default-namespace");
spark.conf().unset("spark.sql.catalog.spark_catalog.parquet-enabled");
spark.conf().unset("spark.sql.catalog.spark_catalog.cache-enabled");
}

@Test
Expand Down Expand Up @@ -728,6 +733,8 @@ public void testStructOfThreeLevelLists() throws Exception {

@Test
public void testTwoLevelList() throws IOException {
Assume.assumeTrue("Cannot migrate to a hadoop based catalog", !type.equals("hadoop"));

spark.conf().set("spark.sql.parquet.writeLegacyFormat", true);

String tableName = sourceName("testTwoLevelList");
Expand Down Expand Up @@ -811,6 +818,8 @@ public boolean accept(File dir, String name) {
}

private void threeLevelList(boolean useLegacyMode) throws Exception {
Assume.assumeTrue("Cannot migrate to a hadoop based catalog", !type.equals("hadoop"));

spark.conf().set("spark.sql.parquet.writeLegacyFormat", useLegacyMode);

String tableName = sourceName(String.format("threeLevelList_%s", useLegacyMode));
Expand All @@ -833,6 +842,8 @@ private void threeLevelList(boolean useLegacyMode) throws Exception {
}

private void threeLevelListWithNestedStruct(boolean useLegacyMode) throws Exception {
Assume.assumeTrue("Cannot migrate to a hadoop based catalog", !type.equals("hadoop"));

spark.conf().set("spark.sql.parquet.writeLegacyFormat", useLegacyMode);

String tableName =
Expand All @@ -858,6 +869,8 @@ private void threeLevelListWithNestedStruct(boolean useLegacyMode) throws Except
}

private void threeLevelLists(boolean useLegacyMode) throws Exception {
Assume.assumeTrue("Cannot migrate to a hadoop based catalog", !type.equals("hadoop"));

spark.conf().set("spark.sql.parquet.writeLegacyFormat", useLegacyMode);

String tableName = sourceName(String.format("threeLevelLists_%s", useLegacyMode));
Expand Down Expand Up @@ -885,6 +898,8 @@ private void threeLevelLists(boolean useLegacyMode) throws Exception {
}

private void structOfThreeLevelLists(boolean useLegacyMode) throws Exception {
Assume.assumeTrue("Cannot migrate to a hadoop based catalog", !type.equals("hadoop"));

spark.conf().set("spark.sql.parquet.writeLegacyFormat", useLegacyMode);

String tableName = sourceName(String.format("structOfThreeLevelLists_%s", useLegacyMode));
Expand Down
15 changes: 15 additions & 0 deletions spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/actions/TestCreateActions.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -183,6 +183,11 @@ public void before() {
public void after() throws IOException {
// Drop the hive table.
spark.sql(String.format("DROP TABLE IF EXISTS %s", baseTableName));
spark.sessionState().catalogManager().reset();
spark.conf().unset("spark.sql.catalog.spark_catalog.type");
spark.conf().unset("spark.sql.catalog.spark_catalog.default-namespace");
spark.conf().unset("spark.sql.catalog.spark_catalog.parquet-enabled");
spark.conf().unset("spark.sql.catalog.spark_catalog.cache-enabled");
}

@Test
Expand Down Expand Up @@ -728,6 +733,8 @@ public void testStructOfThreeLevelLists() throws Exception {

@Test
public void testTwoLevelList() throws IOException {
Assume.assumeTrue("Cannot migrate to a hadoop based catalog", !type.equals("hadoop"));

spark.conf().set("spark.sql.parquet.writeLegacyFormat", true);

String tableName = sourceName("testTwoLevelList");
Expand Down Expand Up @@ -811,6 +818,8 @@ public boolean accept(File dir, String name) {
}

private void threeLevelList(boolean useLegacyMode) throws Exception {
Assume.assumeTrue("Cannot migrate to a hadoop based catalog", !type.equals("hadoop"));

spark.conf().set("spark.sql.parquet.writeLegacyFormat", useLegacyMode);

String tableName = sourceName(String.format("threeLevelList_%s", useLegacyMode));
Expand All @@ -833,6 +842,8 @@ private void threeLevelList(boolean useLegacyMode) throws Exception {
}

private void threeLevelListWithNestedStruct(boolean useLegacyMode) throws Exception {
Assume.assumeTrue("Cannot migrate to a hadoop based catalog", !type.equals("hadoop"));

spark.conf().set("spark.sql.parquet.writeLegacyFormat", useLegacyMode);

String tableName =
Expand All @@ -858,6 +869,8 @@ private void threeLevelListWithNestedStruct(boolean useLegacyMode) throws Except
}

private void threeLevelLists(boolean useLegacyMode) throws Exception {
Assume.assumeTrue("Cannot migrate to a hadoop based catalog", !type.equals("hadoop"));

spark.conf().set("spark.sql.parquet.writeLegacyFormat", useLegacyMode);

String tableName = sourceName(String.format("threeLevelLists_%s", useLegacyMode));
Expand Down Expand Up @@ -885,6 +898,8 @@ private void threeLevelLists(boolean useLegacyMode) throws Exception {
}

private void structOfThreeLevelLists(boolean useLegacyMode) throws Exception {
Assume.assumeTrue("Cannot migrate to a hadoop based catalog", !type.equals("hadoop"));

spark.conf().set("spark.sql.parquet.writeLegacyFormat", useLegacyMode);

String tableName = sourceName(String.format("structOfThreeLevelLists_%s", useLegacyMode));
Expand Down

0 comments on commit 5298462

Please sign in to comment.