Merge branch 'develop'

.env.dependencies

@@ -21,6 +21,9 @@ MONGO_INITDB_ROOT_PASSWORD=E4j7h!9A
 # MSSQL
 ACCEPT_EULA=Y
 MSSQL_SA_PASSWORD=2astazeY
+MSSQL_DATABASE=onetl
+MSSQL_USER=onetl
+MSSQL_PASSWORD=7ellowEl7akey
 
 # MySQL
 MYSQL_ROOT_PASSWORD=ohbuz9Eochaj9saibooK3thooGa5aesh

.github/workflows/data/clickhouse/matrix.yml

@@ -1,41 +1,32 @@
 min: &min
+  # Clickhouse version with proper DateTime > DateTime64 comparison
+  clickhouse-image: yandex/clickhouse-server
+  clickhouse-version: '21.1'
   spark-version: 2.3.1
   pydantic-version: 1
   python-version: '3.7'
   java-version: 8
   os: ubuntu-latest
 
 max: &max
-  spark-version: 3.5.1
+  clickhouse-image: clickhouse/clickhouse-server
+  clickhouse-version: 24.8.2.3-alpine
+  spark-version: 3.5.2
   pydantic-version: 2
   python-version: '3.12'
   java-version: 20
   os: ubuntu-latest
 
 latest: &latest
+  clickhouse-image: clickhouse/clickhouse-server
+  clickhouse-version: latest-alpine
   spark-version: latest
   pydantic-version: latest
   python-version: '3.12'
   java-version: 20
   os: ubuntu-latest
 
 matrix:
-  small:
-    - clickhouse-image: clickhouse/clickhouse-server
-      clickhouse-version: 24.3.2.23-alpine
-      <<: *max
-  full:
-    # Clickhouse version with proper DateTime > DateTime64 comparison
-    - clickhouse-image: yandex/clickhouse-server
-      clickhouse-version: '21.1'
-      <<: *min
-    - clickhouse-image: clickhouse/clickhouse-server
-      clickhouse-version: 24.3.2.23-alpine
-      <<: *max
-  nightly:
-    - clickhouse-image: yandex/clickhouse-server
-      clickhouse-version: '21.1'
-      <<: *min
-    - clickhouse-image: clickhouse/clickhouse-server
-      clickhouse-version: latest-alpine
-      <<: *latest
+  small: [*max]
+  full: [*min, *max]
+  nightly: [*min, *max, *latest]

.github/workflows/data/core/matrix.yml

@@ -6,7 +6,7 @@ min: &min
   os: ubuntu-latest
 
 max: &max
-  spark-version: 3.5.1
+  spark-version: 3.5.2
   pydantic-version: 2
   python-version: '3.12'
   java-version: 20

.github/workflows/data/core/tracked.txt

@@ -3,6 +3,5 @@ onetl/plugins/**
 onetl/impl/**
 onetl/hwm/**
 onetl/_util/**
-onetl/_internal.py
 onetl/log.py
 .github/workflows/data/core/**

.github/workflows/data/file-df/tracked.txt

@@ -1,6 +1,4 @@
 .github/workflows/data/file-df/**
-onetl/file_df_connection/spark_file_df_connection.py
-onetl/file/file_df_reader/**
-onetl/file/file_df_writer/**
 onetl/file/__init__.py
-tests/resources/file_df_connection/**
+**/*file_df*
+**/*file_df*/**

.github/workflows/data/greenplum/matrix.yml

@@ -1,4 +1,6 @@
 min: &min
+  greenplum-version: 6.23.1
+  package-version: 2.2.0
   # Spark 2.3.0 does not support passing ivysettings.xml
   spark-version: 2.3.1
   pydantic-version: 1
@@ -7,6 +9,8 @@ min: &min
   os: ubuntu-latest
 
 max: &max
+  greenplum-version: 7.0.0
+  package-version: 2.3.1
   # Greenplum connector does not support Spark 3.3+
   spark-version: 3.2.4
   pydantic-version: 2
@@ -15,6 +19,8 @@ max: &max
   os: ubuntu-latest
 
 latest: &latest
+  greenplum-version: 7.0.0
+  package-version: 2.3.1
   # Greenplum connector does not support Spark 3.3+
   spark-version: 3.2.4
   pydantic-version: latest
@@ -23,21 +29,6 @@ latest: &latest
   os: ubuntu-latest
 
 matrix:
-  small:
-    - greenplum-version: 7.0.0
-      package-version: 2.3.1
-      <<: *max
-  full:
-    - greenplum-version: 6.23.1
-      package-version: 2.2.0
-      <<: *min
-    - greenplum-version: 7.0.0
-      package-version: 2.3.1
-      <<: *max
-  nightly:
-    - greenplum-version: 6.23.1
-      package-version: 2.2.0
-      <<: *min
-    - greenplum-version: 7.0.0
-      package-version: 2.3.1
-      <<: *latest
+  small: [*max]
+  full: [*min, *max]
+  nightly: [*min, *max, *latest]

.github/workflows/data/hdfs/matrix.yml

@@ -8,7 +8,7 @@ min: &min
 
 max: &max
   hadoop-version: hadoop3-hdfs
-  spark-version: 3.5.1
+  spark-version: 3.5.2
   pydantic-version: 2
   python-version: '3.12'
   java-version: 20
@@ -23,11 +23,6 @@ latest: &latest
   os: ubuntu-latest
 
 matrix:
-  small:
-    - *max
-  full:
-    - *min
-    - *max
-  nightly:
-    - *min
-    - *latest
+  small: [*max]
+  full: [*min, *max]
+  nightly: [*min, *max, *latest]

.github/workflows/data/hive/matrix.yml

@@ -6,7 +6,7 @@ min: &min
   os: ubuntu-latest
 
 max: &max
-  spark-version: 3.5.1
+  spark-version: 3.5.2
   pydantic-version: 2
   python-version: '3.12'
   java-version: 20

.github/workflows/data/kafka/matrix.yml

@@ -10,9 +10,9 @@ min: &min
   os: ubuntu-latest
 
 max: &max
-  kafka-version: 3.7.0
+  kafka-version: 3.7.1
   pydantic-version: 2
-  spark-version: 3.5.1
+  spark-version: 3.5.2
   python-version: '3.12'
   java-version: 20
   os: ubuntu-latest
@@ -26,11 +26,6 @@ latest: &latest
   os: ubuntu-latest
 
 matrix:
-  small:
-    - *max
-  full:
-    - *min
-    - *max
-  nightly:
-    - *min
-    - *latest
+  small: [*max]
+  full: [*min, *max]
+  nightly: [*min, *max, *latest]

.github/workflows/data/local-fs/matrix.yml

@@ -20,8 +20,8 @@ min_excel: &min_excel
   os: ubuntu-latest
 
 max: &max
-  # Excel package currently has no release for 3.5.1
-  spark-version: 3.5.0
+  # Excel pagkage currently supports Spark 3.5.1 max
+  spark-version: 3.5.1
   pydantic-version: 2
   python-version: '3.12'
   java-version: 20

.github/workflows/data/mongodb/matrix.yml

@@ -1,36 +1,29 @@
 min: &min
-  # MongoDB connector does not support Spark 2
+  mongodb-version: 4.0.0
+  # MongoDB connector does not support Spark 2.x
   spark-version: 3.2.4
   pydantic-version: 1
   python-version: '3.7'
   java-version: 8
   os: ubuntu-latest
 
 max: &max
-  spark-version: 3.5.1
+  mongodb-version: 7.0.14
+  spark-version: 3.5.2
   pydantic-version: 2
   python-version: '3.12'
   java-version: 20
   os: ubuntu-latest
 
 latest: &latest
+  mongodb-version: latest
   spark-version: latest
   pydantic-version: latest
   python-version: '3.12'
   java-version: 20
   os: ubuntu-latest
 
 matrix:
-  small:
-    - mongodb-version: 7.0.9
-      <<: *max
-  full:
-    - mongodb-version: 4.0.0
-      <<: *min
-    - mongodb-version: 7.0.9
-      <<: *max
-  nightly:
-    - mongodb-version: 4.0.0
-      <<: *min
-    - mongodb-version: latest
-      <<: *latest
+  small: [*max]
+  full: [*min, *max]
+  nightly: [*min, *latest]

.github/workflows/data/mssql/matrix.yml

@@ -1,35 +1,28 @@
 min: &min
+  mssql-version: 2017-GA-ubuntu
   spark-version: 2.3.1
   pydantic-version: 1
   python-version: '3.7'
   java-version: 8
   os: ubuntu-latest
 
 max: &max
-  spark-version: 3.5.1
+  mssql-version: 2022-CU14-ubuntu-22.04
+  spark-version: 3.5.2
   pydantic-version: 2
   python-version: '3.12'
   java-version: 20
   os: ubuntu-latest
 
 latest: &latest
+  mssql-version: latest
   spark-version: latest
   pydantic-version: latest
   python-version: '3.12'
   java-version: 20
   os: ubuntu-latest
 
 matrix:
-  small:
-    - mssql-version: 2022-CU12-ubuntu-22.04
-      <<: *max
-  full:
-    - mssql-version: 2017-GA-ubuntu
-      <<: *min
-    - mssql-version: 2022-CU12-ubuntu-22.04
-      <<: *max
-  nightly:
-    - mssql-version: 2017-GA-ubuntu
-      <<: *min
-    - mssql-version: latest
-      <<: *latest
+  small: [*max]
+  full: [*min, *max]
+  nightly: [*min, *latest]

.github/workflows/data/mysql/matrix.yml

@@ -1,37 +1,30 @@
 min: &min
+  # Tags 5.7.6-5.6.12 cannot be downloaded since Docker v26:
+  # "Docker Image Format v1 and Docker Image manifest version 2, schema 1 support is disabled by default"
+  mysql-version: 5.7.13
   spark-version: 2.3.1
   pydantic-version: 1
   python-version: '3.7'
   java-version: 8
   os: ubuntu-latest
 
 max: &max
-  spark-version: 3.5.1
+  mysql-version: 9.0.1
+  spark-version: 3.5.2
   pydantic-version: 2
   python-version: '3.12'
   java-version: 20
   os: ubuntu-latest
 
 latest: &latest
+  mysql-version: latest
   spark-version: latest
   pydantic-version: latest
   python-version: '3.12'
   java-version: 20
   os: ubuntu-latest
 
 matrix:
-  small:
-    - mysql-version: 8.4.0
-      <<: *max
-  full:
-    # Min supported version by JDBC driver is 5.7
-    - mysql-version: 5.7.6
-      <<: *min
-    # Max supported version by JDBC driver is 8.3
-    - mysql-version: 8.4.0
-      <<: *max
-  nightly:
-    - mysql-version: 5.7.6
-      <<: *min
-    - mysql-version: latest
-      <<: *latest
+  small: [*max]
+  full: [*min, *max]
+  nightly: [*min, *latest]