From c3d0b8a790176e38b8fc761d64e3ef31b7d7f6e3 Mon Sep 17 00:00:00 2001 From: Cheng Pan Date: Mon, 9 Dec 2024 15:04:11 +0800 Subject: [PATCH 1/6] Bump Spark 3.5.4 RC1 --- build.gradle | 2 ++ gradle/libs.versions.toml | 2 +- .../spark/data/vectorized/IcebergArrowColumnVector.java | 5 +++++ 3 files changed, 8 insertions(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index eb63ce138ab2..57e72a4f1450 100644 --- a/build.gradle +++ b/build.gradle @@ -119,6 +119,8 @@ allprojects { repositories { mavenCentral() mavenLocal() + // Spark 3.5.4 RC1 staging repo + maven { url = "https://repository.apache.org/content/repositories/orgapachespark-1471/" } } } diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index 16bafc7a736a..1a2eea96ab11 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -82,7 +82,7 @@ slf4j = "2.0.16" snowflake-jdbc = "3.20.0" spark-hive33 = "3.3.4" spark-hive34 = "3.4.4" -spark-hive35 = "3.5.2" +spark-hive35 = "3.5.4" sqlite-jdbc = "3.47.1.0" testcontainers = "1.20.4" tez010 = "0.10.4" diff --git a/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/data/vectorized/IcebergArrowColumnVector.java b/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/data/vectorized/IcebergArrowColumnVector.java index 38ec3a0e838c..088a15a11375 100644 --- a/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/data/vectorized/IcebergArrowColumnVector.java +++ b/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/data/vectorized/IcebergArrowColumnVector.java @@ -59,6 +59,11 @@ public void close() { accessor.close(); } + // If a column vector is writable or constant, it should override this method and do nothing. + // See more details at SPARK-50235, SPARK-50463 (Fixed in Spark 3.5.4) + @Override + public void closeIfNotWritable() {} + @Override public boolean hasNull() { return nullabilityHolder.hasNulls(); From 18bb71149ff4b9da95be8f705679f8e7f4fe9ee6 Mon Sep 17 00:00:00 2001 From: Cheng Pan Date: Mon, 16 Dec 2024 17:10:47 +0800 Subject: [PATCH 2/6] 3.5.4 RC2 --- build.gradle | 4 ++-- .../spark/data/vectorized/IcebergArrowColumnVector.java | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/build.gradle b/build.gradle index 57e72a4f1450..54fdd61f699c 100644 --- a/build.gradle +++ b/build.gradle @@ -119,8 +119,8 @@ allprojects { repositories { mavenCentral() mavenLocal() - // Spark 3.5.4 RC1 staging repo - maven { url = "https://repository.apache.org/content/repositories/orgapachespark-1471/" } + // Spark 3.5.4 RC2 staging repo + maven { url = "https://repository.apache.org/content/repositories/orgapachespark-1473/" } } } diff --git a/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/data/vectorized/IcebergArrowColumnVector.java b/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/data/vectorized/IcebergArrowColumnVector.java index 088a15a11375..9b22a308d6b5 100644 --- a/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/data/vectorized/IcebergArrowColumnVector.java +++ b/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/data/vectorized/IcebergArrowColumnVector.java @@ -62,7 +62,7 @@ public void close() { // If a column vector is writable or constant, it should override this method and do nothing. // See more details at SPARK-50235, SPARK-50463 (Fixed in Spark 3.5.4) @Override - public void closeIfNotWritable() {} + public void closeIfFreeable() {} @Override public boolean hasNull() { From 2feb6b0624c269fd2c2e97745c3fb6e34f6dda0f Mon Sep 17 00:00:00 2001 From: Cheng Pan Date: Tue, 17 Dec 2024 21:55:15 +0800 Subject: [PATCH 3/6] Spark 3.5.4 RC3 --- build.gradle | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/build.gradle b/build.gradle index 54fdd61f699c..fa3abfa9ffa8 100644 --- a/build.gradle +++ b/build.gradle @@ -119,8 +119,8 @@ allprojects { repositories { mavenCentral() mavenLocal() - // Spark 3.5.4 RC2 staging repo - maven { url = "https://repository.apache.org/content/repositories/orgapachespark-1473/" } + // Spark 3.5.4 RC3 staging repo + maven { url = "https://repository.apache.org/content/repositories/orgapachespark-1474/" } } } From 0c2505492714f4e9af2ce6072b9ea8577471bf09 Mon Sep 17 00:00:00 2001 From: Cheng Pan Date: Wed, 18 Dec 2024 10:28:53 +0800 Subject: [PATCH 4/6] comment --- .../spark/data/vectorized/IcebergArrowColumnVector.java | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/data/vectorized/IcebergArrowColumnVector.java b/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/data/vectorized/IcebergArrowColumnVector.java index 9b22a308d6b5..bf6de0e1391f 100644 --- a/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/data/vectorized/IcebergArrowColumnVector.java +++ b/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/data/vectorized/IcebergArrowColumnVector.java @@ -59,10 +59,11 @@ public void close() { accessor.close(); } - // If a column vector is writable or constant, it should override this method and do nothing. - // See more details at SPARK-50235, SPARK-50463 (Fixed in Spark 3.5.4) @Override - public void closeIfFreeable() {} + public void closeIfFreeable() { + // If a column vector is writable or constant, it should override this method and do nothing. + // See more details at SPARK-50235, SPARK-50463 (Fixed in Spark 3.5.4) + } @Override public boolean hasNull() { From 7bb210061fe021eebd8abf48a986df52c78cab72 Mon Sep 17 00:00:00 2001 From: Cheng Pan Date: Fri, 20 Dec 2024 20:27:57 +0800 Subject: [PATCH 5/6] release --- build.gradle | 2 -- 1 file changed, 2 deletions(-) diff --git a/build.gradle b/build.gradle index fa3abfa9ffa8..eb63ce138ab2 100644 --- a/build.gradle +++ b/build.gradle @@ -119,8 +119,6 @@ allprojects { repositories { mavenCentral() mavenLocal() - // Spark 3.5.4 RC3 staging repo - maven { url = "https://repository.apache.org/content/repositories/orgapachespark-1474/" } } } From dc7aea5bdedc21cbca10ece52d50a9be0fd1e8d2 Mon Sep 17 00:00:00 2001 From: Cheng Pan Date: Fri, 20 Dec 2024 22:40:40 +0800 Subject: [PATCH 6/6] Update spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/data/vectorized/IcebergArrowColumnVector.java --- .../iceberg/spark/data/vectorized/IcebergArrowColumnVector.java | 1 - 1 file changed, 1 deletion(-) diff --git a/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/data/vectorized/IcebergArrowColumnVector.java b/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/data/vectorized/IcebergArrowColumnVector.java index bf6de0e1391f..dc118aebe374 100644 --- a/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/data/vectorized/IcebergArrowColumnVector.java +++ b/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/data/vectorized/IcebergArrowColumnVector.java @@ -59,7 +59,6 @@ public void close() { accessor.close(); } - @Override public void closeIfFreeable() { // If a column vector is writable or constant, it should override this method and do nothing. // See more details at SPARK-50235, SPARK-50463 (Fixed in Spark 3.5.4)