[engine] Remove support for Flink 1.14 and Spark 2 (apache#2998)

liming30 · Mar 13, 2024 · 598ec08 · 598ec08
1 parent d84edd4
commit 598ec08
Show file tree

Hide file tree

Showing 51 changed files with 90 additions and 3,056 deletions.
diff --git a/.github/workflows/e2e-tests-1.14-jdk11.yml b/.github/workflows/e2e-tests-1.14-jdk11.yml
diff --git a/.github/workflows/e2e-tests-1.14.yml b/.github/workflows/e2e-tests-1.14.yml
diff --git a/.github/workflows/utitcase-flink.yml b/.github/workflows/utitcase-flink.yml
@@ -52,7 +52,7 @@ jobs:
           jvm_timezone=$(random_timezone)
           echo "JVM timezone is set to $jvm_timezone"
           test_modules=""
-          for suffix in 1.14 1.15 1.16 1.17 1.18 common; do 
+          for suffix in 1.15 1.16 1.17 1.18 common; do
             test_modules+="org.apache.paimon:paimon-flink-${suffix}," 
           done
           test_modules="${test_modules%,}"

diff --git a/docs/content/engines/flink.md b/docs/content/engines/flink.md
@@ -30,7 +30,7 @@ This documentation is a guide for using Paimon in Flink.
 
 ## Preparing Paimon Jar File
 
-Paimon currently supports Flink 1.17, 1.16, 1.15 and 1.14. We recommend the latest Flink version for a better experience.
+Paimon currently supports Flink 1.18, 1.17, 1.16, 1.15. We recommend the latest Flink version for a better experience.
 
 Download the jar file with corresponding version.
 
@@ -43,7 +43,6 @@ Download the jar file with corresponding version.
 | Flink 1.17 | Bundled Jar  | [paimon-flink-1.17-{{< version >}}.jar](https://repo.maven.apache.org/maven2/org/apache/paimon/paimon-flink-1.17/{{< version >}}/paimon-flink-1.17-{{< version >}}.jar) |
 | Flink 1.16 | Bundled Jar  | [paimon-flink-1.16-{{< version >}}.jar](https://repo.maven.apache.org/maven2/org/apache/paimon/paimon-flink-1.16/{{< version >}}/paimon-flink-1.16-{{< version >}}.jar) |
 | Flink 1.15 | Bundled Jar  | [paimon-flink-1.15-{{< version >}}.jar](https://repo.maven.apache.org/maven2/org/apache/paimon/paimon-flink-1.15/{{< version >}}/paimon-flink-1.15-{{< version >}}.jar) |
-| Flink 1.14 | Bundled Jar  | [paimon-flink-1.14-{{< version >}}.jar](https://repo.maven.apache.org/maven2/org/apache/paimon/paimon-flink-1.14/{{< version >}}/paimon-flink-1.14-{{< version >}}.jar) |
 | Flink Action | Action Jar  | [paimon-flink-action-{{< version >}}.jar](https://repo.maven.apache.org/maven2/org/apache/paimon/paimon-flink-action/{{< version >}}/paimon-flink-action-{{< version >}}.jar) |
 
 {{< /stable >}}
@@ -56,7 +55,6 @@ Download the jar file with corresponding version.
 | Flink 1.17 | Bundled Jar | [paimon-flink-1.17-{{< version >}}.jar](https://repository.apache.org/snapshots/org/apache/paimon/paimon-flink-1.17/{{< version >}}/) |
 | Flink 1.16 | Bundled Jar | [paimon-flink-1.16-{{< version >}}.jar](https://repository.apache.org/snapshots/org/apache/paimon/paimon-flink-1.16/{{< version >}}/) |
 | Flink 1.15 | Bundled Jar | [paimon-flink-1.15-{{< version >}}.jar](https://repository.apache.org/snapshots/org/apache/paimon/paimon-flink-1.15/{{< version >}}/) |
-| Flink 1.14 | Bundled Jar | [paimon-flink-1.14-{{< version >}}.jar](https://repository.apache.org/snapshots/org/apache/paimon/paimon-flink-1.14/{{< version >}}/) |
 | Flink Action | Action Jar | [paimon-flink-action-{{< version >}}.jar](https://repository.apache.org/snapshots/org/apache/paimon/paimon-flink-action/{{< version >}}/) |
 
 {{< /unstable >}}

diff --git a/docs/content/engines/overview.md b/docs/content/engines/overview.md
@@ -34,7 +34,7 @@ Apache Spark and Apache Hive.
 
 | Engine |    Version    | Batch Read | Batch Write | Create Table | Alter Table | Streaming Write | Streaming Read | Batch Overwrite |
 |:------:|:-------------:|:----------:|:-----------:|:------------:|:-----------:|:---------------:|:--------------:|:---------------:|
-| Flink  |  1.14 - 1.18  |     ✅      |      ✅      |      ✅       |  ✅(1.17+)   |        ✅        |       ✅        |        ✅        |
+| Flink  |  1.15 - 1.18  |     ✅      |      ✅      |      ✅       |  ✅(1.17+)   |        ✅        |       ✅        |        ✅        |
 | Spark  |   3.1 - 3.5   |     ✅      |      ✅      |      ✅       |      ✅      |        ✅        |    ✅(3.3+)     |        ✅        |
 |  Hive  |   2.1 - 3.1   |     ✅      |      ✅      |      ✅       |      ❌      |        ❌        |       ❌        |        ❌        |
 | Spark  |      2.4      |     ✅      |      ❌      |      ❌       |      ❌      |        ❌        |       ❌        |        ❌        |

diff --git a/docs/content/engines/spark.md b/docs/content/engines/spark.md
@@ -724,51 +724,3 @@ All Spark's data types are available in package `org.apache.spark.sql.types`.
 - Conversion between Spark's `UserDefinedType` and Paimon's `UserDefinedType` is not supported.
 
 {{< /hint >}}
-
-## Spark 2
-
-Paimon supports Spark 2.4+. We highly recommend using versions above Spark3, as Spark2 only provides reading capabilities.
-
-{{< stable >}}
-
-Download [paimon-spark-2-{{< version >}}.jar](https://repo.maven.apache.org/maven2/org/apache/paimon/paimon-spark-2/{{< version >}}/paimon-spark-2-{{< version >}}.jar).
-
-{{< /stable >}}
-
-{{< unstable >}}
-
-Download [paimon-spark-2-{{< version >}}.jar](https://repository.apache.org/snapshots/org/apache/paimon/paimon-spark-2/{{< version >}}/).
-
-{{< /unstable >}}
-
-{{< hint info >}}
-
-If you are using HDFS, make sure that the environment variable `HADOOP_HOME` or `HADOOP_CONF_DIR` is set.
-
-{{< /hint >}}
-
-**Step 1: Prepare Test Data**
-
-Paimon currently only supports reading tables through Spark2. To create a Paimon table with records, please follow our [Flink quick start guide]({{< ref "engines/flink#quick-start" >}}).
-
-After the guide, all table files should be stored under the path `/tmp/paimon`, or the warehouse path you've specified.
-
-**Step 2: Specify Paimon Jar File**
-
-You can append path to paimon jar file to the `--jars` argument when starting `spark-shell`.
-
-```bash
-spark-shell ... --jars /path/to/paimon-spark-2-{{< version >}}.jar
-```
-
-Alternatively, you can copy `paimon-spark-2-{{< version >}}.jar` under `spark/jars` in your Spark installation directory.
-
-**Step 3: Query Table**
-
-Paimon with Spark 2.4 does not support DDL. You can use the `Dataset` reader and register the `Dataset` as a temporary table. In spark shell:
-
-```scala
-val dataset = spark.read.format("paimon").load("file:/tmp/paimon/default.db/word_count")
-dataset.createOrReplaceTempView("word_count")
-spark.sql("SELECT * FROM word_count").show()
-```
diff --git a/docs/content/how-to/creating-tables.md b/docs/content/how-to/creating-tables.md
@@ -620,16 +620,6 @@ val dataset = spark.read.format("paimon").load("hdfs:///path/to/table")
 
 {{< /tab >}}
 
-{{< tab "Spark2" >}}
-
-Spark2 only supports creating external tables through Scala API. The following Scala code loads the table located at `hdfs:///path/to/table` into a `DataSet`.
-
-```scala
-val dataset = spark.read.format("paimon").load("hdfs:///path/to/table")
-```
-
-{{< /tab >}}
-
 {{< tab "Hive" >}}
 
 To access existing paimon table, you can also register them as external tables in Hive. The following SQL creates an

diff --git a/docs/content/project/download.md b/docs/content/project/download.md
@@ -38,14 +38,12 @@ This documentation is a guide for downloading Paimon Jars.
 | Flink 1.17       | [paimon-flink-1.17-{{< version >}}.jar](https://repository.apache.org/snapshots/org/apache/paimon/paimon-flink-1.17/{{< version >}}/)                                 |
 | Flink 1.16       | [paimon-flink-1.16-{{< version >}}.jar](https://repository.apache.org/snapshots/org/apache/paimon/paimon-flink-1.16/{{< version >}}/)                                 |
 | Flink 1.15       | [paimon-flink-1.15-{{< version >}}.jar](https://repository.apache.org/snapshots/org/apache/paimon/paimon-flink-1.15/{{< version >}}/)                                 |
-| Flink 1.14       | [paimon-flink-1.14-{{< version >}}.jar](https://repository.apache.org/snapshots/org/apache/paimon/paimon-flink-1.14/{{< version >}}/)                                 |
 | Flink Action     | [paimon-flink-action-{{< version >}}.jar](https://repository.apache.org/snapshots/org/apache/paimon/paimon-flink-action/{{< version >}}/)                             |
 | Spark 3.5        | [paimon-spark-3.5-{{< version >}}.jar](https://repository.apache.org/snapshots/org/apache/paimon/paimon-spark-3.5/{{< version >}}/)                                   |
 | Spark 3.4        | [paimon-spark-3.4-{{< version >}}.jar](https://repository.apache.org/snapshots/org/apache/paimon/paimon-spark-3.4/{{< version >}}/)                                   |
 | Spark 3.3        | [paimon-spark-3.3-{{< version >}}.jar](https://repository.apache.org/snapshots/org/apache/paimon/paimon-spark-3.3/{{< version >}}/)                                   |
 | Spark 3.2        | [paimon-spark-3.2-{{< version >}}.jar](https://repository.apache.org/snapshots/org/apache/paimon/paimon-spark-3.2/{{< version >}}/)                                   |
 | Spark 3.1        | [paimon-spark-3.1-{{< version >}}.jar](https://repository.apache.org/snapshots/org/apache/paimon/paimon-spark-3.1/{{< version >}}/)                                   |
-| Spark 2          | [paimon-spark-2-{{< version >}}.jar](https://repository.apache.org/snapshots/org/apache/paimon/paimon-spark-2/{{< version >}}/)                                       |
 | Hive 3.1         | [paimon-hive-connector-3.1-{{< version >}}.jar](https://repository.apache.org/snapshots/org/apache/paimon/paimon-hive-connector-3.1/{{< version >}}/)                 |
 | Hive 2.3         | [paimon-hive-connector-2.3-{{< version >}}.jar](https://repository.apache.org/snapshots/org/apache/paimon/paimon-hive-connector-2.3/{{< version >}}/)                 |
 | Hive 2.2         | [paimon-hive-connector-2.2-{{< version >}}.jar](https://repository.apache.org/snapshots/org/apache/paimon/paimon-hive-connector-2.2/{{< version >}}/)                 |
@@ -73,14 +71,12 @@ This documentation is a guide for downloading Paimon Jars.
 | Flink 1.17       | [paimon-flink-1.17-{{< version >}}.jar](https://repo.maven.apache.org/maven2/org/apache/paimon/paimon-flink-1.17/{{< version >}}/paimon-flink-1.17-{{< version >}}.jar)                                                 |
 | Flink 1.16       | [paimon-flink-1.16-{{< version >}}.jar](https://repo.maven.apache.org/maven2/org/apache/paimon/paimon-flink-1.16/{{< version >}}/paimon-flink-1.16-{{< version >}}.jar)                                                 |
 | Flink 1.15       | [paimon-flink-1.15-{{< version >}}.jar](https://repo.maven.apache.org/maven2/org/apache/paimon/paimon-flink-1.15/{{< version >}}/paimon-flink-1.15-{{< version >}}.jar)                                                 |
-| Flink 1.14       | [paimon-flink-1.14-{{< version >}}.jar](https://repo.maven.apache.org/maven2/org/apache/paimon/paimon-flink-1.14/{{< version >}}/paimon-flink-1.14-{{< version >}}.jar)                                                 |
 | Flink Action     | [paimon-flink-action-{{< version >}}.jar](https://repo.maven.apache.org/maven2/org/apache/paimon/paimon-flink-action/{{< version >}}/paimon-flink-action-{{< version >}}.jar)                                           |
 | Spark 3.5        | [paimon-spark-3.5-{{< version >}}.jar](https://repo.maven.apache.org/maven2/org/apache/paimon/paimon-spark-3.5/{{< version >}}/paimon-spark-3.5-{{< version >}}.jar)                                                    |
 | Spark 3.4        | [paimon-spark-3.4-{{< version >}}.jar](https://repo.maven.apache.org/maven2/org/apache/paimon/paimon-spark-3.4/{{< version >}}/paimon-spark-3.4-{{< version >}}.jar)                                                    |
 | Spark 3.3        | [paimon-spark-3.3-{{< version >}}.jar](https://repo.maven.apache.org/maven2/org/apache/paimon/paimon-spark-3.3/{{< version >}}/paimon-spark-3.3-{{< version >}}.jar)                                                    |
 | Spark 3.2        | [paimon-spark-3.2-{{< version >}}.jar](https://repo.maven.apache.org/maven2/org/apache/paimon/paimon-spark-3.2/{{< version >}}/paimon-spark-3.2-{{< version >}}.jar)                                                    |
 | Spark 3.1        | [paimon-spark-3.1-{{< version >}}.jar](https://repo.maven.apache.org/maven2/org/apache/paimon/paimon-spark-3.1/{{< version >}}/paimon-spark-3.1-{{< version >}}.jar)                                                    |
-| Spark 2          | [paimon-spark-2-{{< version >}}.jar](https://repo.maven.apache.org/maven2/org/apache/paimon/paimon-spark-2/{{< version >}}/paimon-spark-2-{{< version >}}.jar)                                                          |
 | Hive 3.1         | [paimon-hive-connector-3.1-{{< version >}}.jar](https://repo.maven.apache.org/maven2/org/apache/paimon/paimon-hive-connector-3.1/{{< version >}}/paimon-hive-connector-3.1-{{< version >}}.jar)                         |
 | Hive 2.3         | [paimon-hive-connector-2.3-{{< version >}}.jar](https://repo.maven.apache.org/maven2/org/apache/paimon/paimon-hive-connector-2.3/{{< version >}}/paimon-hive-connector-2.3-{{< version >}}.jar)                         |
 | Hive 2.2         | [paimon-hive-connector-2.2-{{< version >}}.jar](https://repo.maven.apache.org/maven2/org/apache/paimon/paimon-hive-connector-2.2/{{< version >}}/paimon-hive-connector-2.2-{{< version >}}.jar)                         |

diff --git a/paimon-e2e-tests/pom.xml b/paimon-e2e-tests/pom.xml
@@ -304,16 +304,5 @@ under the License.
                 <flink.sql.connector.hive>flink-sql-connector-hive-2.3.6_${scala.binary.version}</flink.sql.connector.hive>
             </properties>
         </profile>
-
-        <profile>
-            <id>flink-1.14</id>
-            <properties>
-                <test.flink.main.version>1.14</test.flink.main.version>
-                <test.flink.version>1.14.6</test.flink.version>
-                <test.flink.connector.kafka.version>${test.flink.version}</test.flink.connector.kafka.version>
-                <flink.sql.connector.kafka>flink-sql-connector-kafka_${scala.binary.version}</flink.sql.connector.kafka>
-                <flink.sql.connector.hive>flink-sql-connector-hive-2.3.6_${scala.binary.version}</flink.sql.connector.hive>
-            </properties>
-        </profile>
     </profiles>
 </project>
diff --git a/paimon-e2e-tests/src/test/java/org/apache/paimon/tests/HiveE2eTest.java b/paimon-e2e-tests/src/test/java/org/apache/paimon/tests/HiveE2eTest.java
@@ -20,7 +20,6 @@
 
 import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Test;
-import org.junit.jupiter.api.condition.DisabledIfSystemProperty;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.testcontainers.containers.Container;
@@ -36,7 +35,6 @@
  * <p>NOTE: This test runs a complete Hadoop cluster in Docker, which requires a lot of memory. If
  * you're running this test locally, make sure that the memory limit of your Docker is at least 8GB.
  */
-@DisabledIfSystemProperty(named = "test.flink.version", matches = "1.14.*")
 public class HiveE2eTest extends E2eReaderTestBase {
 
     private static final Logger LOG = LoggerFactory.getLogger(HiveE2eTest.class);

diff --git a/paimon-e2e-tests/src/test/java/org/apache/paimon/tests/LogStoreE2eTest.java b/paimon-e2e-tests/src/test/java/org/apache/paimon/tests/LogStoreE2eTest.java
@@ -20,12 +20,10 @@
 
 import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Test;
-import org.junit.jupiter.api.condition.DisabledIfSystemProperty;
 
 import java.util.UUID;
 
 /** Tests for reading and writing log store in stream jobs. */
-@DisabledIfSystemProperty(named = "test.flink.version", matches = "1.14.*")
 public class LogStoreE2eTest extends E2eTestBase {
 
     private String topicName;

diff --git a/paimon-e2e-tests/src/test/java/org/apache/paimon/tests/SparkE2eTest.java b/paimon-e2e-tests/src/test/java/org/apache/paimon/tests/SparkE2eTest.java
@@ -19,7 +19,6 @@
 package org.apache.paimon.tests;
 
 import org.junit.jupiter.api.Test;
-import org.junit.jupiter.api.condition.DisabledIfSystemProperty;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.testcontainers.containers.Container;
@@ -30,7 +29,6 @@
 import java.util.stream.Collectors;
 
 /** Tests for reading paimon from Spark3. */
-@DisabledIfSystemProperty(named = "test.flink.version", matches = "1.14.*")
 public class SparkE2eTest extends E2eReaderTestBase {
 
     private static final Logger LOG = LoggerFactory.getLogger(SparkE2eTest.class);
@@ -41,7 +39,7 @@ public SparkE2eTest() {
 
     @Test
     public void testFlinkWriteAndSparkRead() throws Exception {
-        String warehousePath = TEST_DATA_DIR + "/" + UUID.randomUUID().toString() + "_warehouse";
+        String warehousePath = TEST_DATA_DIR + "/" + UUID.randomUUID() + "_warehouse";
         final String table = "T";
         final String sparkTable = String.format("paimon.default.%s", table);
         runSql(