Merge branch 'develop'

MobileTeleSystems · Feb 5, 2024 · 3214e75 · 3214e75
2 parents f5244e5 + 631cb1d
commit 3214e75
Show file tree

Hide file tree

Showing 327 changed files with 4,646 additions and 5,164 deletions.
diff --git a/.env.docker b/.env.docker
@@ -3,9 +3,9 @@ TZ=UTC
 # Greenplum
 ONETL_GP_HOST=greenplum
 ONETL_GP_PORT=5432
-ONETL_GP_DATABASE=testdb
-ONETL_GP_USER=tester
-ONETL_GP_PASSWORD=pivotal
+ONETL_GP_DATABASE=postgres
+ONETL_GP_USER=gpadmin
+ONETL_GP_PASSWORD=
 
 # ClickHouse
 ONETL_CH_HOST=clickhouse

diff --git a/.env.local b/.env.local
@@ -3,9 +3,9 @@ export TZ=UTC
 # Greenplum
 export ONETL_GP_HOST=localhost
 export ONETL_GP_PORT=5433
-export ONETL_GP_DATABASE=testdb
-export ONETL_GP_USER=tester
-export ONETL_GP_PASSWORD=pivotal
+export ONETL_GP_DATABASE=postgres
+export ONETL_GP_USER=gpadmin
+export ONETL_GP_PASSWORD=
 
 # ClickHouse
 export ONETL_CH_HOST=localhost

diff --git a/.github/dependabot.yml b/.github/dependabot.yml
@@ -0,0 +1,10 @@
+# Set update schedule for GitHub Actions
+
+version: 2
+updates:
+  - package-ecosystem: github-actions
+    directory: /
+    schedule:
+      interval: daily
+    labels:
+      - type:ci
diff --git a/.github/workflows/automerge.yml b/.github/workflows/automerge.yml
@@ -10,10 +10,10 @@ jobs:
     if: github.event.pull_request.user.login == 'pre-commit-ci[bot]' || github.event.pull_request.user.login == 'dependabot[bot]'
 
     steps:
-    - uses: alexwilson/[email protected]
-      with:
-        github-token: ${{ secrets.AUTOMERGE_TOKEN }}
-        merge-method: REBASE
+      - uses: alexwilson/[email protected]
+        with:
+          github-token: ${{ secrets.AUTOMERGE_TOKEN }}
+          merge-method: REBASE
 
   autoapprove:
     name: Automatically approve pull request
@@ -22,6 +22,6 @@ jobs:
     if: github.event.pull_request.user.login == 'pre-commit-ci[bot]' || github.event.pull_request.user.login == 'dependabot[bot]'
 
     steps:
-    - uses: hmarr/auto-approve-action@v3
-      with:
-        github-token: ${{ secrets.AUTOMERGE_TOKEN }}
+      - uses: hmarr/auto-approve-action@v4
+        with:
+          github-token: ${{ secrets.AUTOMERGE_TOKEN }}
diff --git a/.github/workflows/changelog.yml b/.github/workflows/changelog.yml
@@ -4,7 +4,7 @@ on:
   pull_request:
     types: [opened, synchronize, labeled, unlabeled, reopened]
     branches-ignore:
-    - master
+      - master
 
 env:
   DEFAULT_PYTHON: '3.12'
@@ -19,44 +19,44 @@ jobs:
     timeout-minutes: 10
     if: "!contains(github.event.pull_request.labels.*.name, 'ci:skip-changelog') && github.event.pull_request.user.login != 'pre-commit-ci[bot]' && github.event.pull_request.user.login != 'dependabot[bot]'"
     steps:
-    - name: Checkout code
-      uses: actions/checkout@v4
-      with:
-        fetch-depth: 0
-
-    - name: Set up Python ${{ env.DEFAULT_PYTHON }}
-      id: python
-      uses: actions/setup-python@v4
-      with:
-        python-version: ${{ env.DEFAULT_PYTHON }}
-
-    - name: Cache pip
-      uses: actions/cache@v3
-      with:
-        path: ~/.cache/pip
-        key: ${{ runner.os }}-python-${{ env.DEFAULT_PYTHON }}-changelog-${{ hashFiles('requirements/core.txt', 'requirements/docs.txt') }}
-        restore-keys: |
-          ${{ runner.os }}-python-${{ env.DEFAULT_PYTHON }}-changelog-${{ hashFiles('requirements/core.txt', 'requirements/docs.txt') }}
-          ${{ runner.os }}-python-${{ env.DEFAULT_PYTHON }}-changelog-
-
-    - name: Upgrade pip
-      run: python -m pip install --upgrade pip setuptools wheel
-
-    - name: Install dependencies
-      run: |
-        pip install -I  -r requirements/core.txt -r requirements/docs.txt
-
-    - name: Check changelog entry exists
-      run: |
-        if [ ! -s docs/changelog/next_release/${{ github.event.pull_request.number }}.*.rst ]; then
-            echo "Please add corresponding file 'docs/changelog/next_release/<issue number>.<change type>.rst' with changes description"
-            exit 1
-        fi
-
-    - name: Validate changelog
-      run: |
-        # Fetch the pull request' base branch so towncrier will be able to
-        # compare the current branch with the base branch.
-        git fetch --no-tags origin +refs/heads/${{ github.base_ref }}:refs/remotes/origin/${{ github.base_ref }}
-        towncrier check --compare-with origin/${{ github.base_ref }}
-        towncrier --draft
+      - name: Checkout code
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: Set up Python ${{ env.DEFAULT_PYTHON }}
+        id: python
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ env.DEFAULT_PYTHON }}
+
+      - name: Cache pip
+        uses: actions/cache@v4
+        with:
+          path: ~/.cache/pip
+          key: ${{ runner.os }}-python-${{ env.DEFAULT_PYTHON }}-changelog-${{ hashFiles('requirements/core.txt', 'requirements/docs.txt') }}
+          restore-keys: |
+            ${{ runner.os }}-python-${{ env.DEFAULT_PYTHON }}-changelog-${{ hashFiles('requirements/core.txt', 'requirements/docs.txt') }}
+            ${{ runner.os }}-python-${{ env.DEFAULT_PYTHON }}-changelog-
+
+      - name: Upgrade pip
+        run: python -m pip install --upgrade pip setuptools wheel
+
+      - name: Install dependencies
+        run: |
+          pip install -I  -r requirements/core.txt -r requirements/docs.txt
+
+      - name: Check changelog entry exists
+        run: |
+          if [ ! -s docs/changelog/next_release/${{ github.event.pull_request.number }}.*.rst ]; then
+              echo "Please add corresponding file 'docs/changelog/next_release/<issue number>.<change type>.rst' with changes description"
+              exit 1
+          fi
+
+      - name: Validate changelog
+        run: |
+          # Fetch the pull request' base branch so towncrier will be able to
+          # compare the current branch with the base branch.
+          git fetch --no-tags origin +refs/heads/${{ github.base_ref }}:refs/remotes/origin/${{ github.base_ref }}
+          towncrier check --compare-with origin/${{ github.base_ref }}
+          towncrier --draft
diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml
@@ -3,10 +3,10 @@ name: Code analysis
 on:
   push:
     branches:
-    - develop
+      - develop
   pull_request:
     branches-ignore:
-    - master
+      - master
   workflow_dispatch:
 
 concurrency:
@@ -28,55 +28,55 @@ jobs:
       security-events: write
 
     steps:
-    - name: Checkout repository
-      uses: actions/checkout@v4
-
-    - name: Set up Python ${{ env.DEFAULT_PYTHON }}
-      uses: actions/setup-python@v4
-      with:
-        python-version: ${{ env.DEFAULT_PYTHON }}
-
-    - name: Install Kerberos headers
-      run: |
-        sudo apt-get update
-        sudo apt-get install --no-install-recommends libkrb5-dev
-
-    - name: Cache pip
-      uses: actions/cache@v3
-      with:
-        path: ~/.cache/pip
-        key: ${{ runner.os }}-python-${{ env.DEFAULT_PYTHON }}-codeql-${{ hashFiles('requirements*.txt') }}
-        restore-keys: |
-          ${{ runner.os }}-python-${{ env.DEFAULT_PYTHON }}-codeql-${{ hashFiles('requirements*.txt') }}
-          ${{ runner.os }}-python-${{ env.DEFAULT_PYTHON }}-codeql-
-          ${{ runner.os }}-python
-          ${{ runner.os }}-
-
-    - name: Upgrade pip
-      run: python -m pip install --upgrade pip setuptools wheel
-
-    - name: Install dependencies
-      run: |
-        pip install -I \
-          -r requirements/core.txt \
-          -r requirements/dev.txt \
-          -r requirements/ftp.txt \
-          -r requirements/hdfs.txt \
-          -r requirements/kerberos.txt \
-          -r requirements/s3.txt \
-          -r requirements/sftp.txt \
-          -r requirements/spark.txt \
-          -r requirements/webdav.txt
-
-        # Set the `CODEQL-PYTHON` environment variable to the Python executable
-        # that includes the dependencies
-        echo "CODEQL_PYTHON=$(which python)" >> $GITHUB_ENV
-
-    - name: Run flake8
-      run: python3 -m flake8 --config setup.cfg .
-
-    - name: Run mypy
-      run: python3 -m mypy --config-file setup.cfg onetl
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Set up Python ${{ env.DEFAULT_PYTHON }}
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ env.DEFAULT_PYTHON }}
+
+      - name: Install Kerberos headers
+        run: |
+          sudo apt-get update
+          sudo apt-get install --no-install-recommends libkrb5-dev
+
+      - name: Cache pip
+        uses: actions/cache@v4
+        with:
+          path: ~/.cache/pip
+          key: ${{ runner.os }}-python-${{ env.DEFAULT_PYTHON }}-codeql-${{ hashFiles('requirements*.txt') }}
+          restore-keys: |
+            ${{ runner.os }}-python-${{ env.DEFAULT_PYTHON }}-codeql-${{ hashFiles('requirements*.txt') }}
+            ${{ runner.os }}-python-${{ env.DEFAULT_PYTHON }}-codeql-
+            ${{ runner.os }}-python
+            ${{ runner.os }}-
+
+      - name: Upgrade pip
+        run: python -m pip install --upgrade pip setuptools wheel
+
+      - name: Install dependencies
+        run: |
+          pip install -I \
+            -r requirements/core.txt \
+            -r requirements/dev.txt \
+            -r requirements/ftp.txt \
+            -r requirements/hdfs.txt \
+            -r requirements/kerberos.txt \
+            -r requirements/s3.txt \
+            -r requirements/sftp.txt \
+            -r requirements/spark.txt \
+            -r requirements/webdav.txt
+
+          # Set the `CODEQL-PYTHON` environment variable to the Python executable
+          # that includes the dependencies
+          echo "CODEQL_PYTHON=$(which python)" >> $GITHUB_ENV
+
+      - name: Run flake8
+        run: python3 -m flake8 --config setup.cfg .
+
+      - name: Run mypy
+        run: python3 -m mypy --config-file setup.cfg onetl
 
   codeql:
     name: CodeQL
@@ -87,21 +87,21 @@ jobs:
       security-events: write
 
     steps:
-    - name: Checkout repository
-      uses: actions/checkout@v4
+      - name: Checkout repository
+        uses: actions/checkout@v4
 
-    - name: Set up Python ${{ env.DEFAULT_PYTHON }}
-      uses: actions/setup-python@v4
-      with:
-        python-version: ${{ env.DEFAULT_PYTHON }}
+      - name: Set up Python ${{ env.DEFAULT_PYTHON }}
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ env.DEFAULT_PYTHON }}
 
     # Initializes the CodeQL tools for scanning.
-    - name: Initialize CodeQL
-      uses: github/codeql-action/init@v2
-      with:
-        languages: python
-
-    - name: Perform CodeQL Analysis
-      uses: github/codeql-action/analyze@v2
-      with:
-        category: /language:python
+      - name: Initialize CodeQL
+        uses: github/codeql-action/init@v3
+        with:
+          languages: python
+
+      - name: Perform CodeQL Analysis
+        uses: github/codeql-action/analyze@v3
+        with:
+          category: /language:python
diff --git a/.github/workflows/data/clickhouse/matrix.yml b/.github/workflows/data/clickhouse/matrix.yml
@@ -18,21 +18,21 @@ latest: &latest
 
 matrix:
   small:
-  - clickhouse-image: clickhouse/clickhouse-server
-    clickhouse-version: 23.6.1-alpine
-    <<: *max
+    - clickhouse-image: clickhouse/clickhouse-server
+      clickhouse-version: 23.6.1-alpine
+      <<: *max
   full:
   # the lowest supported Clickhouse version by JDBC driver
-  - clickhouse-image: yandex/clickhouse-server
-    clickhouse-version: '20.7'
-    <<: *min
-  - clickhouse-image: clickhouse/clickhouse-server
-    clickhouse-version: 23.6.1-alpine
-    <<: *max
+    - clickhouse-image: yandex/clickhouse-server
+      clickhouse-version: '20.7'
+      <<: *min
+    - clickhouse-image: clickhouse/clickhouse-server
+      clickhouse-version: 23.6.1-alpine
+      <<: *max
   nightly:
-  - clickhouse-image: yandex/clickhouse-server
-    clickhouse-version: '20.7'
-    <<: *min
-  - clickhouse-image: clickhouse/clickhouse-server
-    clickhouse-version: latest-alpine
-    <<: *latest
+    - clickhouse-image: yandex/clickhouse-server
+      clickhouse-version: '20.7'
+      <<: *min
+    - clickhouse-image: clickhouse/clickhouse-server
+      clickhouse-version: latest-alpine
+      <<: *latest
diff --git a/.github/workflows/data/clickhouse/tracked.txt b/.github/workflows/data/clickhouse/tracked.txt
@@ -1 +1,2 @@
-**/clickhouse*
+**/*clickhouse*
+**/*clickhouse*/**
diff --git a/.github/workflows/data/ftp/matrix.yml b/.github/workflows/data/ftp/matrix.yml
@@ -9,11 +9,11 @@ max: &max
 matrix:
   small:
   # chonjay21/ftps image has only latest tag
-  - ftp-version: latest
-    <<: *max
+    - ftp-version: latest
+      <<: *max
   full: &full
-  - ftp-version: latest
-    <<: *min
-  - ftp-version: latest
-    <<: *max
+    - ftp-version: latest
+      <<: *min
+    - ftp-version: latest
+      <<: *max
   nightly: *full
diff --git a/.github/workflows/data/ftp/tracked.txt b/.github/workflows/data/ftp/tracked.txt
@@ -1,3 +1,4 @@
-**/ftp*
+**/*ftp*
+**/*ftp*/**
 docker-compose.yml
 .env.dependencies
diff --git a/.github/workflows/data/ftps/matrix.yml b/.github/workflows/data/ftps/matrix.yml
@@ -9,11 +9,11 @@ max: &max
 matrix:
   small:
   # chonjay21/ftps image has only latest tag
-  - ftps-version: latest
-    <<: *max
+    - ftps-version: latest
+      <<: *max
   full: &full
-  - ftps-version: latest
-    <<: *min
-  - ftps-version: latest
-    <<: *max
+    - ftps-version: latest
+      <<: *min
+    - ftps-version: latest
+      <<: *max
   nightly: *full