Add Spark 4.0 support

.github/workflows/data/clickhouse/matrix.yml

@@ -1,4 +1,4 @@
-min: &min
+2x: &2x
   # Clickhouse version with proper DateTime > DateTime64 comparison
   clickhouse-image: yandex/clickhouse-server
   clickhouse-version: '21.1'
@@ -8,7 +8,7 @@ min: &min
   java-version: 8
   os: ubuntu-latest
 
-max: &max
+3x: &3x
   clickhouse-image: clickhouse/clickhouse-server
   clickhouse-version: 24.8.2.3-alpine
   spark-version: 3.5.3
@@ -17,6 +17,15 @@ max: &max
   java-version: 20
   os: ubuntu-latest
 
+4x: &4x
+  clickhouse-image: clickhouse/clickhouse-server
+  clickhouse-version: 24.6.3.70-alpine
+  spark-version: 4.0.0
+  pydantic-version: 2
+  python-version: '3.12'
+  java-version: 22
+  os: ubuntu-latest
+
 latest: &latest
   clickhouse-image: clickhouse/clickhouse-server
   clickhouse-version: latest-alpine
@@ -27,6 +36,6 @@ latest: &latest
   os: ubuntu-latest
 
 matrix:
-  small: [*max]
-  full: [*min, *max]
-  nightly: [*min, *max, *latest]
+  small: [*3x]
+  full: [*2x, *3x, *4x]
+  nightly: [*2x, *3x, *latest]

.github/workflows/data/core/matrix.yml

@@ -1,17 +1,24 @@
-min: &min
+2x: &2x
   spark-version: 2.3.1
   pydantic-version: 1
   python-version: '3.7'
   java-version: 8
   os: ubuntu-latest
 
-max: &max
+3x: &3x
   spark-version: 3.5.3
   pydantic-version: 2
   python-version: '3.12'
   java-version: 20
   os: ubuntu-latest
 
+4x: &4x
+  spark-version: 4.0.0
+  pydantic-version: 2
+  python-version: '3.12'
+  java-version: 22
+  os: ubuntu-latest
+
 latest: &latest
   spark-version: latest
   pydantic-version: latest
@@ -20,6 +27,6 @@ latest: &latest
   os: ubuntu-latest
 
 matrix:
-  small: [*max]
-  full: [*min, *max]
-  nightly: [*min, *max, *latest]
+  small: [*3x]
+  full: [*2x, *3x, *4x]
+  nightly: [*2x, *3x, *latest]

.github/workflows/data/greenplum/matrix.yml

@@ -1,4 +1,4 @@
-min: &min
+23: &23
   greenplum-version: 6.23.1
   package-version: 2.2.0
   # Spark 2.3.0 does not support passing ivysettings.xml
@@ -8,7 +8,7 @@ min: &min
   java-version: 8
   os: ubuntu-latest
 
-max: &max
+32: &32
   greenplum-version: 7.0.0
   package-version: 2.3.1
   # Greenplum connector does not support Spark 3.3+
@@ -29,6 +29,6 @@ latest: &latest
   os: ubuntu-latest
 
 matrix:
-  small: [*max]
-  full: [*min, *max]
-  nightly: [*min, *max, *latest]
+  small: [*32]
+  full: [*23, *32]
+  nightly: [*23, *latest]

.github/workflows/data/hdfs/matrix.yml

@@ -1,19 +1,27 @@
-min: &min
+2x: &2x
   hadoop-version: hadoop2-hdfs
   spark-version: 2.3.1
   pydantic-version: 1
   python-version: '3.7'
   java-version: 8
   os: ubuntu-latest
 
-max: &max
+3x: &3x
   hadoop-version: hadoop3-hdfs
   spark-version: 3.5.3
   pydantic-version: 2
   python-version: '3.12'
   java-version: 20
   os: ubuntu-latest
 
+4x: &4x
+  hadoop-version: hadoop3-hdfs
+  spark-version: 4.0.0
+  pydantic-version: 2
+  python-version: '3.12'
+  java-version: 22
+  os: ubuntu-latest
+
 latest: &latest
   hadoop-version: hadoop3-hdfs
   spark-version: latest
@@ -23,6 +31,6 @@ latest: &latest
   os: ubuntu-latest
 
 matrix:
-  small: [*max]
-  full: [*min, *max]
-  nightly: [*min, *max, *latest]
+  small: [*3x]
+  full: [*2x, *3x, *4x]
+  nightly: [*2x, *3x, *latest]

.github/workflows/data/hive/matrix.yml

@@ -1,17 +1,24 @@
-min: &min
+2x: &2x
   spark-version: 2.3.1
   pydantic-version: 1
   python-version: '3.7'
   java-version: 8
   os: ubuntu-latest
 
-max: &max
+3x: &3x
   spark-version: 3.5.3
   pydantic-version: 2
   python-version: '3.12'
   java-version: 20
   os: ubuntu-latest
 
+4x: &4x
+  spark-version: 4.0.0
+  pydantic-version: 2
+  python-version: '3.12'
+  java-version: 22
+  os: ubuntu-latest
+
 latest: &latest
   spark-version: latest
   pydantic-version: latest
@@ -20,6 +27,6 @@ latest: &latest
   os: ubuntu-latest
 
 matrix:
-  small: [*max]
-  full: [*min, *max]
-  nightly: [*min, *latest]
+  small: [*3x]
+  full: [*2x, *3x, *4x]
+  nightly: [*2x, *3x, *latest]

.github/workflows/data/kafka/matrix.yml

@@ -1,5 +1,5 @@
-min: &min
-  # Headers are supported only since 2.x.
+2x: &2x
+  # Headers are supported only since Kafka 2x.
   # Images before 3.2.3 are not creating kafka_jaas.conf properly, and failing to start
   # https://github.com/bitnami/containers/blob/9db9064668365cac89bff58259f63eb78bb97e79/bitnami/kafka/README.md?plain=1#L933
   kafka-version: 3.2.3
@@ -9,14 +9,22 @@ min: &min
   java-version: 8
   os: ubuntu-latest
 
-max: &max
+3x: &3x
   kafka-version: 3.7.1
   pydantic-version: 2
   spark-version: 3.5.3
   python-version: '3.12'
   java-version: 20
   os: ubuntu-latest
 
+4x: &4x
+  kafka-version: 3.7.1
+  pydantic-version: 2
+  spark-version: 4.0.0
+  python-version: '3.12'
+  java-version: 22
+  os: ubuntu-latest
+
 latest: &latest
   kafka-version: latest
   pydantic-version: latest
@@ -26,6 +34,6 @@ latest: &latest
   os: ubuntu-latest
 
 matrix:
-  small: [*max]
-  full: [*min, *max]
-  nightly: [*min, *max, *latest]
+  small: [*3x]
+  full: [*2x, *3x, *4x]
+  nightly: [*2x, *3x, *latest]

.github/workflows/data/local-fs/matrix.yml

@@ -1,32 +1,41 @@
-min: &min
+23: &23
   spark-version: 2.3.1
   pydantic-version: 1
   python-version: '3.7'
   java-version: 8
   os: ubuntu-latest
 
-min_avro: &min_avro
+24: &24
+  # Avro supported only since Spark 2.4
   spark-version: 2.4.8
   pydantic-version: 1
   python-version: '3.7'
   java-version: 8
   os: ubuntu-latest
 
-min_excel: &min_excel
+32: &32
+  # Excel supported only since Spark 3.2
   spark-version: 3.2.4
   pydantic-version: 1
   python-version: '3.7'
   java-version: 8
   os: ubuntu-latest
 
-max: &max
+35: &35
   # Excel pagkage currently supports Spark 3.5.1 max
   spark-version: 3.5.1
   pydantic-version: 2
   python-version: '3.12'
   java-version: 20
   os: ubuntu-latest
 
+4x: &4x
+  spark-version: 4.0.0
+  pydantic-version: 2
+  python-version: '3.12'
+  java-version: 22
+  os: ubuntu-latest
+
 latest: &latest
   spark-version: latest
   pydantic-version: latest
@@ -35,15 +44,6 @@ latest: &latest
   os: ubuntu-latest
 
 matrix:
-  small:
-    - <<: *max
-  full:
-    - <<: *min
-    - <<: *min_avro
-    - <<: *min_excel
-    - <<: *max
-  nightly:
-    - <<: *min
-    - <<: *min_avro
-    - <<: *min_excel
-    - <<: *latest
+  small: [*35]
+  full: [*23, *24, *32, *35, *4x]
+  nightly: [*23, *24, *32, *35, *latest]

.github/workflows/data/mongodb/matrix.yml

@@ -1,20 +1,28 @@
-min: &min
+32: &32
   mongodb-version: 4.0.0
-  # MongoDB connector does not support Spark 2.x
+  # MongoDB connector does not support Spark 2x
   spark-version: 3.2.4
   pydantic-version: 1
   python-version: '3.7'
   java-version: 8
   os: ubuntu-latest
 
-max: &max
+35: &35
   mongodb-version: 7.0.14
   spark-version: 3.5.3
   pydantic-version: 2
   python-version: '3.12'
   java-version: 20
   os: ubuntu-latest
 
+4x: &4x
+  mongodb-version: 7.0.12
+  spark-version: 4.0.0
+  pydantic-version: 2
+  python-version: '3.12'
+  java-version: 20
+  os: ubuntu-latest
+
 latest: &latest
   mongodb-version: latest
   spark-version: latest
@@ -24,6 +32,6 @@ latest: &latest
   os: ubuntu-latest
 
 matrix:
-  small: [*max]
-  full: [*min, *max]
-  nightly: [*min, *latest]
+  small: [*35]
+  full: [*32, *35, *4x]
+  nightly: [*32, *35, *latest]

.github/workflows/data/mssql/matrix.yml

@@ -1,19 +1,27 @@
-min: &min
+2x: &2x
   mssql-version: 2017-latest
   spark-version: 2.3.1
   pydantic-version: 1
   python-version: '3.7'
   java-version: 8
   os: ubuntu-20.04
 
-max: &max
+3x: &3x
   mssql-version: 2022-latest
   spark-version: 3.5.3
   pydantic-version: 2
   python-version: '3.12'
   java-version: 20
   os: ubuntu-latest
 
+4x: &4x
+  mssql-version: 2022-CU14-ubuntu-22.04
+  spark-version: 4.0.0
+  pydantic-version: 2
+  python-version: '3.12'
+  java-version: 22
+  os: ubuntu-latest
+
 latest: &latest
   mssql-version: latest
   spark-version: latest
@@ -23,6 +31,6 @@ latest: &latest
   os: ubuntu-latest
 
 matrix:
-  small: [*max]
-  full: [*min, *max]
-  nightly: [*min, *latest]
+  small: [*3x]
+  full: [*2x, *3x, *4x]
+  nightly: [*2x, *3x, *latest]