Merge pull request #964 from exasol/develop

Changelist:

* #954: added release checklist (#955)
* #956: updated dependencies (#957)
* #950: Refactored Java Script Options parser (#960)
* #962: Added a GH workflow which uploads trivy databases (#963)

.github/pull_request_template.md

@@ -16,3 +16,7 @@
 ### When integrating to Main branch:
 
 1. [ ] Remember to merge with "Merge"
+2. [ ] Have you thought about version number? If there is a breaking change in the toolchain, need to update major version.
+3. [ ] If you plan a release, have you checked the Exasol packages for updates?
+   1. Websocket API (https://github.com/EXASOL/websocket-api)
+   2. [Exasol-BucketFS](https://pypi.org/project/exasol-bucketfs/)

.github/workflows/check_bazel_tests.yml

@@ -0,0 +1,42 @@
+name: Check Bazel Tests
+
+on:
+  push:
+    branches:
+      - develop
+  pull_request:
+
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+
+    env:
+      USE_BAZEL_VERSION: 7.2.1
+    steps:
+    - uses: actions/checkout@v4
+      with:
+        fetch-depth: 0
+        submodules: 'recursive'
+
+    - name: Install bazel
+      run: | 
+        curl -L -o bazel https://github.com/bazelbuild/bazelisk/releases/download/v1.19.0/bazelisk-linux-amd64
+        chmod +x bazel
+    - name: Install JDK and ZMQ
+      run: |
+        sudo apt-get update
+        sudo apt-get install -y openjdk-11-jdk libzmq3-dev
+    - name: Java Tests
+      run: |
+        bazel test //base/javacontainer/test/...
+      working-directory: ./exaudfclient/
+    - name: ExaudfLib Tests
+      run: |
+        bazel test //base/exaudflib/test/...
+      working-directory: ./exaudfclient/
+    - name: Script Options Parser Tests
+      run: |
+        bazel test //base/script_options_parser/test/...
+      working-directory: ./exaudfclient/
+

.github/workflows/check_current_gitmodules.yaml

@@ -14,8 +14,9 @@ jobs:
 
     steps:
     - uses: actions/checkout@v4
-    - name: Init submodules
-      run: git submodule update --init --recursive
+      with:
+        fetch-depth: 0
+        submodules: 'recursive'
     - name: Run update_current_submodules_file.sh
       run: bash githooks/update_current_submodules_file.sh
     - name: Show changes on working copy

.github/workflows/check_error_codes.yml

@@ -14,9 +14,9 @@ jobs:
 
     steps:
     - uses: actions/checkout@v4
-
-    - name: Init submodules
-      run: git submodule update --init --recursive
+      with:
+        fetch-depth: 0
+        submodules: 'recursive'
 
     - name: Search for duplicated error codes
       run: bash find_duplicate_error_codes.sh

.github/workflows/check_if_submodules_head_are_ancestors_of_origin_master.yaml

@@ -14,9 +14,9 @@ jobs:
 
     steps:
     - uses: actions/checkout@v4
-
-    - name: Init submodules
-      run: git submodule update --init --recursive
+      with:
+        fetch-depth: 0
+        submodules: 'recursive'
 
     - name: Check if submodules heads are ancestors of origin/master
       run: bash scripts/check_if_submodules_head_are_ancestors_of_origin_master.sh 

.github/workflows/check_markdown_links.yaml

@@ -12,9 +12,11 @@ jobs:
   markdown-link-check:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v4
-      - name: Init submodules
-        run: git submodule update --init --recursive
-      - uses: gaurav-nelson/github-action-markdown-link-check@v1
-        with:
-          config-file: '.github/markdown_check_config.json'
+    - uses: actions/checkout@v4
+      with:
+        fetch-depth: 0
+        submodules: 'recursive'
+
+    - uses: gaurav-nelson/github-action-markdown-link-check@v1
+      with:
+        config-file: '.github/markdown_check_config.json'

.github/workflows/check_package_diffs.yaml

@@ -11,9 +11,9 @@ jobs:
 
     steps:
     - uses: actions/checkout@v4
-
-    - name: Init submodules
-      run: git submodule update --init --recursive
+      with:
+        fetch-depth: 0
+        submodules: 'recursive'
 
     - name: Setup Python & Poetry Environment
       uses: exasol/python-toolbox/.github/actions/[email protected]

.github/workflows/check_pull_request.yml

@@ -8,10 +8,10 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
-    - uses: actions/checkout@v3
-
-    - name: Init submodules
-      run: git submodule update --init --recursive
+    - uses: actions/checkout@v4
+      with:
+        fetch-depth: 0
+        submodules: 'recursive'
 
     - name: Check if the branch triggers a rebuild
       run: "bash .github/workflows/check_pull_request.sh ${{ github.event.pull_request.head.ref }} ${{ github.event.pull_request.base.ref }}"

.github/workflows/nigthly.yaml

@@ -14,9 +14,9 @@ jobs:
 
     steps:
     - uses: actions/checkout@v4
-
-    - name: Init submodules
-      run: git submodule update --init --recursive
+      with:
+        fetch-depth: 0
+        submodules: 'recursive'
 
     - name: Get Time
       id: time

.github/workflows/push_docker_test_container.yaml

@@ -12,9 +12,10 @@ jobs:
     runs-on: ubuntu-latest
     environment: publish
     steps:
-      - uses: actions/checkout@v3
-      - name: Init submodules
-        run: git submodule update --init --recursive
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+          submodules: 'recursive'
       - name: Build TestContainer
         run: ./exaslct build-test-container --force-rebuild
       - name: Push TestContainer image

.github/workflows/release_droid_upload_github_release_assets.yml

@@ -13,21 +13,21 @@ jobs:
     environment: AWS
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v4
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+          submodules: 'recursive'
 
-    - name: Init submodules
-      run: git submodule update --init --recursive
+      - name: Setup Python & Poetry Environment
+        uses: exasol/python-toolbox/.github/actions/[email protected]
+        with:
+          python-version: "3.10"
+          poetry-version: '1.8.2'
 
-    - name: Setup Python & Poetry Environment
-      uses: exasol/python-toolbox/.github/actions/[email protected]
-      with:
-        python-version: "3.10"
-        poetry-version: '1.8.2'
-
-    - name: Start release
-      run: poetry run python -m exasol_script_languages_container_ci_setup.main start-release-build --log-level info --project "ScriptLanguagesRelease" --upload-url "${{ github.event.inputs.upload_url }}" --branch "$GITHUB_REF"
-      env: # Set the secret as an env variable
-        AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
-        AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_ACCESS_KEY_SECRET }}
-        AWS_DEFAULT_REGION: ${{ secrets.AWS_REGION }}
-        GITHUB_TOKEN: ${{ github.token }}
+      - name: Start release
+        run: poetry run python -m exasol_script_languages_container_ci_setup.main start-release-build --log-level info --project "ScriptLanguagesRelease" --upload-url "${{ github.event.inputs.upload_url }}" --branch "$GITHUB_REF"
+        env: # Set the secret as an env variable
+          AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_ACCESS_KEY_SECRET }}
+          AWS_DEFAULT_REGION: ${{ secrets.AWS_REGION }}
+          GITHUB_TOKEN: ${{ github.token }}

.github/workflows/test_release.yml

@@ -13,21 +13,21 @@ jobs:
     environment: AWS
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v4
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+          submodules: 'recursive'
 
-    - name: Init submodules
-      run: git submodule update --init --recursive
+      - name: Setup Python & Poetry Environment
+        uses: exasol/python-toolbox/.github/actions/[email protected]
+        with:
+          python-version: "3.10"
+          poetry-version: '1.8.2'
 
-    - name: Setup Python & Poetry Environment
-      uses: exasol/python-toolbox/.github/actions/[email protected]
-      with:
-        python-version: "3.10"
-        poetry-version: '1.8.2'
-
-    - name: Start test release
-      run: poetry run python -m exasol_script_languages_container_ci_setup.main start-test-release-build --log-level info --project ScriptLanguagesRelease --repo-name "$GITHUB_REPOSITORY" --branch "$GITHUB_REF" --release-title  "${{ github.event.inputs.release_title }}"
-      env: # Set the secret as an env variable
-        AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
-        AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_ACCESS_KEY_SECRET }}
-        AWS_DEFAULT_REGION: ${{ secrets.AWS_REGION }}
-        GITHUB_TOKEN: ${{ github.token }}
+      - name: Start test release
+        run: poetry run python -m exasol_script_languages_container_ci_setup.main start-test-release-build --log-level info --project ScriptLanguagesRelease --repo-name "$GITHUB_REPOSITORY" --branch "$GITHUB_REF" --release-title  "${{ github.event.inputs.release_title }}"
+        env: # Set the secret as an env variable
+          AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_ACCESS_KEY_SECRET }}
+          AWS_DEFAULT_REGION: ${{ secrets.AWS_REGION }}
+          GITHUB_TOKEN: ${{ github.token }}

.github/workflows/update_trivy_cache.yaml

@@ -0,0 +1,54 @@
+name: Update trivy cache
+
+on:
+  workflow_dispatch:
+  schedule:
+    - cron: "30 0 * * *"
+
+jobs:
+  update_trivy_cache:
+    environment: AWS
+    runs-on: ubuntu-latest
+    steps:
+      - name: Run install dependencies
+        run: |
+          sudo apt update && sudo apt install -y awscli curl wget apt-transport-https gnupg lsb-release
+          wget -qO - https://aquasecurity.github.io/trivy-repo/deb/public.key | sudo apt-key add -
+          echo deb https://aquasecurity.github.io/trivy-repo/deb $(lsb_release -sc) main | sudo tee -a /etc/apt/sources.list.d/trivy.list
+          sudo apt-get update
+          sudo apt-get install trivy
+        env:
+          DEBIAN_FRONTEND: noninteractive
+
+      - name: Fetch trivy databases
+        run: |
+          trivy image --download-java-db-only
+          trivy image --download-db-only
+
+      - name: Create tar gz databases
+        run: |
+          pushd $HOME/.cache
+          tar -czf trivy_cache.tar.gz ./trivy
+          popd
+
+      - name: Copy trivy databases
+        run: aws s3 cp  "$HOME/.cache/trivy_cache.tar.gz" "$TRIVY_CACHE_LOCATION/trivy_cache.tar.gz"
+        env: # Set the secret as an env variable
+          AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_ACCESS_KEY_SECRET }}
+          AWS_DEFAULT_REGION: ${{ secrets.AWS_REGION }}
+          TRIVY_CACHE_LOCATION: ${{ secrets.TRIVY_CACHE_LOCATION }}
+
+      - name: Report failure Status to Slack channel
+        id: report-failure-status-slack
+        # Also run this step in case of failures
+        if: ${{ always() }}
+        uses: ravsamhq/notify-slack-action@v2
+        with:
+          status: ${{ job.status }}
+          token: ${{ github.token }}
+          notification_title: "Update trivy build in {repo} has {status_message}"
+          message_format: "{emoji} *{workflow}* {status_message} in <{repo_url}|{repo}>"
+          notify_when: "failure,cancelled,warnings,skipped"
+        env:
+          SLACK_WEBHOOK_URL: ${{ secrets.INTEGRATION_TEAM_SLACK_NOTIFICATION_WEBHOOK }}

doc/changes/changelog.md

@@ -1,5 +1,6 @@
 # Changes
 
+* [8.4.0](changes_8.4.0.md)
 * [8.3.0](changes_8.3.0.md)
 * [8.2.0](changes_8.2.0.md)
 * [8.1.0](changes_8.1.0.md)

doc/changes/changes_8.4.0.md

@@ -0,0 +1,28 @@
+# script-languages-release 8.4.0, released t.b.d.
+
+Code name: t.b.d.
+
+## Summary
+
+t.b.d.
+
+
+## Script-Language-Container-Tool (Exaslct)
+
+This release uses version 1.0.0 of the container tool. 
+
+## Features
+
+ - #962: Add a GH workflow which uploads trivy databases and updated Ubuntu packages
+
+## Security Issues
+
+ - #956: Updated dependencies
+
+## Refactorings
+
+  - #950: Refactored existing script-options parser
+
+## Doc
+
+ - #954: added release checklist