Merge branch 'develop'

.env.dependencies

@@ -13,24 +13,14 @@ KAFKA_CFG_LISTENERS=INTERNAL_PLAINTEXT_ANONYMOUS://:9092,EXTERNAL_PLAINTEXT_ANON
 KAFKA_CFG_ADVERTISED_LISTENERS=INTERNAL_PLAINTEXT_ANONYMOUS://kafka:9092,EXTERNAL_PLAINTEXT_ANONYMOUS://localhost:9093,INTERNAL_PLAINTEXT_SASL://kafka:9094,EXTERNAL_PLAINTEXT_SASL://localhost:9095
 KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP=INTERNAL_PLAINTEXT_ANONYMOUS:PLAINTEXT,EXTERNAL_PLAINTEXT_ANONYMOUS:PLAINTEXT,INTERNAL_PLAINTEXT_SASL:SASL_PLAINTEXT,EXTERNAL_PLAINTEXT_SASL:SASL_PLAINTEXT
 KAFKA_CFG_SASL_ENABLED_MECHANISMS=PLAIN,SCRAM-SHA-256,SCRAM-SHA-512
-# old config names for <1.1.1
-KAFKA_ZOOKEEPER_CONNECT=zookeeper:2181
-KAFKA_INTER_BROKER_LISTENER_NAME=INTERNAL_PLAINTEXT_ANONYMOUS
-KAFKA_LISTENERS=INTERNAL_PLAINTEXT_ANONYMOUS://:9092,EXTERNAL_PLAINTEXT_ANONYMOUS://:9093,INTERNAL_PLAINTEXT_SASL://:9094,EXTERNAL_PLAINTEXT_SASL://:9095
-KAFKA_ADVERTISED_LISTENERS=INTERNAL_PLAINTEXT_ANONYMOUS://kafka:9092,EXTERNAL_PLAINTEXT_ANONYMOUS://localhost:9093,INTERNAL_PLAINTEXT_SASL://kafka:9094,EXTERNAL_PLAINTEXT_SASL://localhost:9095
-KAFKA_LISTENER_SECURITY_PROTOCOL_MAP=INTERNAL_PLAINTEXT_ANONYMOUS:PLAINTEXT,EXTERNAL_PLAINTEXT_ANONYMOUS:PLAINTEXT,INTERNAL_PLAINTEXT_SASL:SASL_PLAINTEXT,EXTERNAL_PLAINTEXT_SASL:SASL_PLAINTEXT
-KAFKA_SASL_ENABLED_MECHANISMS=PLAIN,SCRAM-SHA-256,SCRAM-SHA-512
 
 # Mongo
 MONGO_INITDB_ROOT_USERNAME=onetl
 MONGO_INITDB_ROOT_PASSWORD=E4j7h!9A
 
 # MSSQL
-MSSQL_DB=onetl
-MSSQL_USER=onetl
-MSSQL_PASSWORD=7ellowEl7akey
 ACCEPT_EULA=Y
-SA_PASSWORD=2astazeY
+MSSQL_SA_PASSWORD=2astazeY
 
 # MySQL
 MYSQL_ROOT_PASSWORD=ohbuz9Eochaj9saibooK3thooGa5aesh

.github/workflows/cache-cleanup.yml

@@ -0,0 +1,40 @@
+name: Cleanup caches after merge
+on:
+  pull_request:
+    types:
+      - closed
+  workflow_dispatch:
+
+jobs:
+  cleanup:
+    runs-on: ubuntu-latest
+    permissions:
+      # `actions:write` permission is required to delete caches
+      #  See also: https://docs.github.com/en/rest/actions/cache?apiVersion=2022-11-28#delete-a-github-actions-cache-for-a-repository-using-a-cache-id
+      actions: write
+      contents: read
+
+    steps:
+      - name: Check out code
+        uses: actions/checkout@v4
+
+      - name: Cleanup cache
+        run: |
+          gh extension install actions/gh-actions-cache
+
+          REPO=${{ github.repository }}
+          BRANCH=refs/pull/${{ github.event.pull_request.number }}/merge
+
+          echo "Fetching list of cache key"
+          cacheKeysForPR=$(gh actions-cache list -R $REPO -B $BRANCH --limit 100 --sort size | cut -f 1 )
+
+          ## Setting this to not fail the workflow while deleting cache keys.
+          set +e
+          echo "Deleting caches..."
+          for cacheKey in $cacheKeysForPR
+          do
+              gh actions-cache delete $cacheKey -R $REPO -B $BRANCH --confirm
+          done
+          echo "Done"
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/codeql-analysis.yml

@@ -49,8 +49,6 @@ jobs:
           restore-keys: |
             ${{ runner.os }}-python-${{ env.DEFAULT_PYTHON }}-codeql-${{ hashFiles('requirements*.txt') }}
             ${{ runner.os }}-python-${{ env.DEFAULT_PYTHON }}-codeql-
-            ${{ runner.os }}-python
-            ${{ runner.os }}-
 
       - name: Upgrade pip
         run: python -m pip install --upgrade pip setuptools wheel

.github/workflows/data/clickhouse/matrix.yml

@@ -6,7 +6,7 @@ min: &min
   os: ubuntu-latest
 
 max: &max
-  spark-version: 3.5.0
+  spark-version: 3.5.1
   pydantic-version: 2
   python-version: '3.12'
   java-version: 20
@@ -22,19 +22,19 @@ latest: &latest
 matrix:
   small:
     - clickhouse-image: clickhouse/clickhouse-server
-      clickhouse-version: 23.6.1-alpine
+      clickhouse-version: 24.3.2.23-alpine
       <<: *max
   full:
-  # the lowest supported Clickhouse version by JDBC driver
+    # Clickhouse version with proper DateTime > DateTime64 comparison
     - clickhouse-image: yandex/clickhouse-server
-      clickhouse-version: '20.7'
+      clickhouse-version: '21.1'
       <<: *min
     - clickhouse-image: clickhouse/clickhouse-server
-      clickhouse-version: 23.6.1-alpine
+      clickhouse-version: 24.3.2.23-alpine
       <<: *max
   nightly:
     - clickhouse-image: yandex/clickhouse-server
-      clickhouse-version: '20.7'
+      clickhouse-version: '21.1'
       <<: *min
     - clickhouse-image: clickhouse/clickhouse-server
       clickhouse-version: latest-alpine

.github/workflows/data/core/matrix.yml

@@ -6,7 +6,7 @@ min: &min
   os: ubuntu-latest
 
 max: &max
-  spark-version: 3.5.0
+  spark-version: 3.5.1
   pydantic-version: 2
   python-version: '3.12'
   java-version: 20

.github/workflows/data/core/tracked.txt

@@ -2,5 +2,7 @@ onetl/hooks/**
 onetl/plugins/**
 onetl/impl/**
 onetl/hwm/**
+onetl/_util/**
 onetl/_internal.py
 onetl/log.py
+.github/workflows/data/core/**

.github/workflows/data/ftp/matrix.yml

@@ -15,7 +15,7 @@ latest: &latest
 
 matrix:
   small:
-  # chonjay21/ftps image has only latest tag
+    # chonjay21/ftps image has only latest tag
     - ftp-version: latest
       <<: *max
   full:

.github/workflows/data/ftps/matrix.yml

@@ -15,7 +15,7 @@ latest: &latest
 
 matrix:
   small:
-  # chonjay21/ftps image has only latest tag
+    # chonjay21/ftps image has only latest tag
     - ftps-version: latest
       <<: *max
   full:

.github/workflows/data/greenplum/matrix.yml

@@ -28,14 +28,14 @@ matrix:
       package-version: 2.3.1
       <<: *max
   full:
-    - greenplum-version: 6.25.3
+    - greenplum-version: 6.23.1
       package-version: 2.2.0
       <<: *min
     - greenplum-version: 7.0.0
       package-version: 2.3.1
       <<: *max
   nightly:
-    - greenplum-version: 6.25.3
+    - greenplum-version: 6.23.1
       package-version: 2.2.0
       <<: *min
     - greenplum-version: 7.0.0

.github/workflows/data/hdfs/matrix.yml

@@ -8,7 +8,7 @@ min: &min
 
 max: &max
   hadoop-version: hadoop3-hdfs
-  spark-version: 3.5.0
+  spark-version: 3.5.1
   pydantic-version: 2
   python-version: '3.12'
   java-version: 20

.github/workflows/data/hive/matrix.yml

@@ -6,7 +6,7 @@ min: &min
   os: ubuntu-latest
 
 max: &max
-  spark-version: 3.5.0
+  spark-version: 3.5.1
   pydantic-version: 2
   python-version: '3.12'
   java-version: 20

.github/workflows/data/kafka/matrix.yml

@@ -1,16 +1,18 @@
 min: &min
-  # kafka_version: 0.10.2-1-r3
-  kafka-version: 3.5.1
+  # Headers are supported only since 2.x.
+  # Images before 3.2.3 are not creating kafka_jaas.conf properly, and failing to start
+  # https://github.com/bitnami/containers/blob/9db9064668365cac89bff58259f63eb78bb97e79/bitnami/kafka/README.md?plain=1#L933
+  kafka-version: 3.2.3
   pydantic-version: 1
   spark-version: 2.4.8
   python-version: '3.7'
   java-version: 8
   os: ubuntu-latest
 
 max: &max
-  kafka-version: 3.5.1
+  kafka-version: 3.7.0
   pydantic-version: 2
-  spark-version: 3.5.0
+  spark-version: 3.5.1
   python-version: '3.12'
   java-version: 20
   os: ubuntu-latest

.github/workflows/data/local-fs/matrix.yml

@@ -20,6 +20,7 @@ min_excel: &min_excel
   os: ubuntu-latest
 
 max: &max
+  # Excel package currently has no release for 3.5.1
   spark-version: 3.5.0
   pydantic-version: 2
   python-version: '3.12'

.github/workflows/data/local-fs/tracked.txt

@@ -2,3 +2,19 @@
 **/*local-fs*
 **/*local_fs*/**
 **/*local-fs*/**
+**/*csv*
+**/*csv*/**
+**/*json*
+**/*json*/**
+**/*xml*
+**/*xml*/**
+**/*excel*
+**/*excel*/**
+**/*avro*
+**/*avro*/**
+**/*orc*
+**/*orc*/**
+**/*parquet*
+**/*parquet*/**
+**/*file_format*
+**/*file_format*/**

.github/workflows/data/mongodb/matrix.yml

@@ -7,7 +7,7 @@ min: &min
   os: ubuntu-latest
 
 max: &max
-  spark-version: 3.4.2
+  spark-version: 3.5.1
   pydantic-version: 2
   python-version: '3.12'
   java-version: 20
@@ -22,12 +22,12 @@ latest: &latest
 
 matrix:
   small:
-    - mongodb-version: 6.0.7
+    - mongodb-version: 7.0.9
       <<: *max
   full:
     - mongodb-version: 4.0.0
       <<: *min
-    - mongodb-version: 6.0.7
+    - mongodb-version: 7.0.9
       <<: *max
   nightly:
     - mongodb-version: 4.0.0

.github/workflows/data/mssql/matrix.yml

@@ -6,7 +6,7 @@ min: &min
   os: ubuntu-latest
 
 max: &max
-  spark-version: 3.5.0
+  spark-version: 3.5.1
   pydantic-version: 2
   python-version: '3.12'
   java-version: 20
@@ -21,16 +21,15 @@ latest: &latest
 
 matrix:
   small:
-    - mssql-version: v2017.CU24.0
+    - mssql-version: 2022-CU12-ubuntu-22.04
       <<: *max
   full:
-    - mssql-version: v2017.CU24.0
+    - mssql-version: 2017-GA-ubuntu
       <<: *min
-  # v2019.CU4.0 is not very stable
-    - mssql-version: v2017.CU24.0
+    - mssql-version: 2022-CU12-ubuntu-22.04
       <<: *max
   nightly:
-    - mssql-version: v2017.CU24.0
+    - mssql-version: 2017-GA-ubuntu
       <<: *min
-    - mssql-version: v2017.CU24.0
+    - mssql-version: latest
       <<: *latest

.github/workflows/data/mysql/matrix.yml

@@ -6,7 +6,7 @@ min: &min
   os: ubuntu-latest
 
 max: &max
-  spark-version: 3.5.0
+  spark-version: 3.5.1
   pydantic-version: 2
   python-version: '3.12'
   java-version: 20
@@ -21,17 +21,17 @@ latest: &latest
 
 matrix:
   small:
-    - mysql-version: 8.0.33
+    - mysql-version: 8.4.0
       <<: *max
   full:
-  # Min supported version by JDBC driver is 5.7
-    - mysql-version: 5.7.42
+    # Min supported version by JDBC driver is 5.7
+    - mysql-version: 5.7.6
       <<: *min
-  # Max supported version by JDBC driver is 8.0
-    - mysql-version: 8.0.33
+    # Max supported version by JDBC driver is 8.3
+    - mysql-version: 8.4.0
       <<: *max
   nightly:
-    - mysql-version: 5.7.42
+    - mysql-version: 5.7.6
       <<: *min
     - mysql-version: latest
       <<: *latest

.github/workflows/data/oracle/matrix.yml

@@ -6,7 +6,7 @@ min: &min
   os: ubuntu-latest
 
 max: &max
-  spark-version: 3.5.0
+  spark-version: 3.5.1
   pydantic-version: 2
   python-version: '3.12'
   java-version: 20
@@ -30,10 +30,6 @@ matrix:
       oracle-version: 11.2.0.2-slim-faststart
       db-name: XE
       <<: *min
-    - oracle-image: gvenzl/oracle-xe
-      oracle-version: 21.3.0-slim-faststart
-      db-name: XEPDB1
-      <<: *max
     - oracle-image: gvenzl/oracle-free
       oracle-version: 23.3-slim-faststart
       db-name: FREEPDB1

.github/workflows/data/postgres/matrix.yml

@@ -6,7 +6,7 @@ min: &min
   os: ubuntu-latest
 
 max: &max
-  spark-version: 3.5.0
+  spark-version: 3.5.1
   pydantic-version: 2
   python-version: '3.12'
   java-version: 20
@@ -21,13 +21,13 @@ latest: &latest
 
 matrix:
   small:
-    - postgres-version: 15.2-alpine
+    - postgres-version: 16.2-alpine
       <<: *max
   full:
-  # Min supported version by JDBC driver is 8.4, but it is too ancient to be used by anyone in real life
+    # Min supported version by JDBC driver is 8.4, but it is too ancient to be used by anyone in real life
     - postgres-version: 9.4.26-alpine
       <<: *min
-    - postgres-version: 15.2-alpine
+    - postgres-version: 16.2-alpine
       <<: *max
   nightly:
     - postgres-version: 9.4.26-alpine

.github/workflows/data/s3/matrix.yml

@@ -9,8 +9,8 @@ min: &min
   os: ubuntu-latest
 
 max: &max
-  minio-version: 2023.7.18
-  spark-version: 3.5.0
+  minio-version: 2024.4.18
+  spark-version: 3.5.1
   pydantic-version: 2
   python-version: '3.12'
   java-version: 20

.github/workflows/data/samba/matrix.yml

@@ -15,6 +15,7 @@ latest: &latest
 
 matrix:
   small:
+    # elswork/samba image versions does not correlate with smbd version, it is always 4.x
     - server-version: latest
       <<: *max
   full: