From dbfefb07312be8554438c1f16f1037ab22bf153b Mon Sep 17 00:00:00 2001 From: Cheng Pan Date: Sat, 4 Jan 2025 04:00:33 +0800 Subject: [PATCH] Bump Apache Spark to 3.5.4 (#11731) --- gradle/libs.versions.toml | 2 +- .../spark/data/vectorized/IcebergArrowColumnVector.java | 5 +++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index 556187ceecb8..d10bb876afdf 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -82,7 +82,7 @@ slf4j = "2.0.16" snowflake-jdbc = "3.20.0" spark-hive33 = "3.3.4" spark-hive34 = "3.4.4" -spark-hive35 = "3.5.2" +spark-hive35 = "3.5.4" sqlite-jdbc = "3.47.1.0" testcontainers = "1.20.4" tez010 = "0.10.4" diff --git a/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/data/vectorized/IcebergArrowColumnVector.java b/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/data/vectorized/IcebergArrowColumnVector.java index 38ec3a0e838c..dc118aebe374 100644 --- a/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/data/vectorized/IcebergArrowColumnVector.java +++ b/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/data/vectorized/IcebergArrowColumnVector.java @@ -59,6 +59,11 @@ public void close() { accessor.close(); } + public void closeIfFreeable() { + // If a column vector is writable or constant, it should override this method and do nothing. + // See more details at SPARK-50235, SPARK-50463 (Fixed in Spark 3.5.4) + } + @Override public boolean hasNull() { return nullabilityHolder.hasNulls();