ci.yml

# Copyright (C) 2020 Dremio
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

# Unifies main + PR workflow.
#
# The unified CI workflow consists of 2 "stages":
# - Checks - test, intTest, NesQuEIT, etc
# - Finalize - a "success" dummy job for PRs + a "save to github-cache" job for push-to-main
#
# Utilizes the Gradle build cache for all stages. The updated build cache
# of the jobs in the checks stage are saved as artifacts (with the minimum
# retention period). The updated build cache is pushed back to GigHub's
# cache when the checks have successfully finished.
#
# Java CI runs against Java 11 and Java 17, where it is meaningful.
# (Spark tests always run against 11, so not run against Java 17.)

name: CI build

on:
  push:
    branches:
      - main
      - release-*
    paths-ignore:
      - 'LICENSE'
      - 'NOTICE'
      - '**.md'
      - '!site/**'
      - '.github/renovate.json5'
      - '.github/workflows/release*.yml'
      - '.github/workflows/check*.yml'
      - '.idea/**'
      - '.editorconfig'
  pull_request:
    types: [labeled, opened, synchronize, reopened]

# For the main branch: let all CI runs complete, one after the other. This has a couple advantages:
# * Site deployments happen in commit-order
# * Saved Gradle cache are persisted in commit-order
# * (Potentially) more GH runners available for PRs
concurrency:
  # PRs: 1 CI run concurrently / older ones are cancelled
  # main branch: 1 CI run concurrently / all commits / no cancellation
  group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.ref }}
  cancel-in-progress: ${{ github.ref != 'refs/heads/main' }}

jobs:
  code-checks:
    name: CI Code Checks et al
    runs-on: ubuntu-22.04
    strategy:
      max-parallel: 2
      matrix:
        java-version: ['11'] # Ideally also '17', but GH concurrent job limit ... :(
    steps:
      - uses: actions/checkout@8ade135a41bc03ea155e62e844d188df1ea18608 # v4.1.0
      - name: Setup runner
        uses: ./.github/actions/setup-runner
      - name: Setup Java, Gradle
        uses: ./.github/actions/dev-tool-java
        with:
          java-version: ${{ matrix.java-version }}

      # Needed for the Quarkus plugin - can likely go away once we use Quarkus 3 or newer
      - name: Bump Gradle daemon heap
        run: sed -i 's/-Xms.*/-Xms6G -Xmx6G -XX:MaxMetaspaceSize=1g \\/' gradle.properties

      - name: Prepare Gradle build cache
        uses: ./.github/actions/ci-incr-build-cache-prepare
        with:
          java-version: ${{ matrix.java-version }}

      - name: Gradle / Compile
        uses: gradle/gradle-build-action@v2
        with:
          arguments: |
            spotlessCheck
            compileAll
            -x :nessie-quarkus:compileAll
            -x :nessie-quarkus-cli:compileAll
            -x :nessie-events-quarkus:compileAll
            --scan

      - name: Gradle / Compile Quarkus
        uses: gradle/gradle-build-action@v2
        with:
          arguments: |
            :nessie-quarkus:compileAll
            :nessie-quarkus-cli:compileAll
            :nessie-events-quarkus:compileAll
            --scan

      - name: Gradle / Checkstyle
        uses: gradle/gradle-build-action@v2
        with:
          arguments: checkstyle --scan

      - name: Gradle / Assemble
        uses: gradle/gradle-build-action@v2
        with:
          arguments: assemble --scan

      - name: Gradle / Publish to Maven local
        uses: gradle/gradle-build-action@v2
        with:
          arguments: publishToMavenLocal --scan

      # This is a rather quick one and uses the output of 'publishToMavenLocal', which uses the
      # outputs of 'assemble'
      - name: Gradle / build tools integration tests
        uses: gradle/gradle-build-action@v2
        with:
          arguments: buildToolsIntegrationTest

      - name: Save partial Gradle build cache
        uses: ./.github/actions/ci-incr-build-cache-save
        if: github.event_name == 'push' && github.ref == 'refs/heads/main'
        with:
          job-name: 'code-checks'
          java-version: ${{ matrix.java-version }}

  test:
    name: CI Test
    runs-on: ubuntu-22.04
    strategy:
      max-parallel: 2
      matrix:
        java-version: ['11'] # Ideally also '17', but GH concurrent job limit ... :(
    steps:
      - uses: actions/checkout@8ade135a41bc03ea155e62e844d188df1ea18608 # v4.1.0
      - name: Setup runner
        uses: ./.github/actions/setup-runner
      - name: Setup Java, Gradle
        uses: ./.github/actions/dev-tool-java
        with:
          java-version: ${{ matrix.java-version }}

      - name: Prepare Gradle build cache
        uses: ./.github/actions/ci-incr-build-cache-prepare
        with:
          java-version: ${{ matrix.java-version }}

      - name: Gradle / test
        uses: gradle/gradle-build-action@v2
        with:
          arguments: |
            test
            :nessie-client:check 
            -x :nessie-client:intTest 
            -x :nessie-quarkus:test
            -x :nessie-quarkus-cli:test
            -x :nessie-events-quarkus:test
            --scan

      - name: Capture Test Reports
        uses: actions/upload-artifact@v3
        if: ${{ failure() }}
        with:
          name: ci-test-reports
          path: |
            **/build/reports/*
            **/build/test-results/*
          retention-days: 7

      - name: Save partial Gradle build cache
        uses: ./.github/actions/ci-incr-build-cache-save
        if: github.event_name == 'push' && github.ref == 'refs/heads/main'
        with:
          job-name: 'test'
          java-version: ${{ matrix.java-version }}

  test-quarkus:
    name: CI Test Quarkus
    runs-on: ubuntu-22.04
    strategy:
      max-parallel: 2
      matrix:
        java-version: ['11'] # Ideally also '17', but GH concurrent job limit ... :(
    steps:
      - uses: actions/checkout@8ade135a41bc03ea155e62e844d188df1ea18608 # v4.1.0
      - name: Setup runner
        uses: ./.github/actions/setup-runner
      - name: Setup Java, Gradle
        uses: ./.github/actions/dev-tool-java
        with:
          java-version: ${{ matrix.java-version }}

      - name: Prepare Gradle build cache
        uses: ./.github/actions/ci-incr-build-cache-prepare
        with:
          java-version: ${{ matrix.java-version }}

      - name: Gradle / Test Quarkus
        uses: gradle/gradle-build-action@v2
        with:
          arguments: |
            :nessie-client:test
            :nessie-quarkus:test
            :nessie-events-quarkus:test
            --scan

      - name: Dump quarkus.log
        if: ${{ failure() }}
        run: |
          find . -path "**/build/quarkus.log" | while read ql ; do
            echo "::group::Quarkus build log $ql"
            cat $ql
            echo "::endgroup::"
          done

      - name: Capture Test Reports
        uses: actions/upload-artifact@v3
        if: ${{ failure() }}
        with:
          name: ci-test-quarkus-reports
          path: |
            **/build/quarkus.log
            **/build/reports/*
            **/build/test-results/*
          retention-days: 7

      - name: Save partial Gradle build cache
        uses: ./.github/actions/ci-incr-build-cache-save
        if: github.event_name == 'push' && github.ref == 'refs/heads/main'
        with:
          job-name: 'test-quarkus'
          java-version: ${{ matrix.java-version }}

  int-test:
    name: CI intTest
    runs-on: ubuntu-22.04
    strategy:
      max-parallel: 2
      matrix:
        java-version: ['11'] # Ideally also '17', but GH concurrent job limit ... :(
    env:
      SPARK_LOCAL_IP: localhost
    steps:
      - uses: actions/checkout@8ade135a41bc03ea155e62e844d188df1ea18608 # v4.1.0
      - name: Setup runner
        uses: ./.github/actions/setup-runner
      - name: Setup Java, Gradle
        uses: ./.github/actions/dev-tool-java
        with:
          java-version: ${{ matrix.java-version }}

      - name: Prepare Gradle build cache
        uses: ./.github/actions/ci-incr-build-cache-prepare

      - name: Gradle / intTest
        run: |
          echo "::group::Collect :nessie-versioned-storage projects"
          ./gradlew :listProjectsWithPrefix --prefix :nessie-versioned-persist- --output ../persist-prjs.txt --exclude
          echo "::endgroup::"

          echo "::group::Collect :nessie-versioned-persist projects"
          ./gradlew :listProjectsWithPrefix --prefix :nessie-versioned-storage- --output ../storage-prjs.txt --exclude
          echo "::endgroup::"

          echo "::group::Collect :nessie-spark-extensions projects"
          ./gradlew :listProjectsWithPrefix --prefix :nessie-spark-ext --output ../spark-prjs.txt --exclude
          echo "::endgroup::"

          ./gradlew intTest \
            -x :nessie-quarkus:intTest \
            -x :nessie-quarkus-cli:intTest \
            -x :nessie-events-quarkus:intTest \
            $(cat ../persist-prjs.txt) \
            $(cat ../storage-prjs.txt) \
            $(cat ../spark-prjs.txt) \
            --scan

      - name: Capture Test Reports
        uses: actions/upload-artifact@v3
        if: ${{ failure() }}
        with:
          name: ci-inttest-reports
          path: |
            **/build/reports/*
            **/build/test-results/*
          retention-days: 7

      - name: Save partial Gradle build cache
        uses: ./.github/actions/ci-incr-build-cache-save
        if: github.event_name == 'push' && github.ref == 'refs/heads/main'
        with:
          job-name: 'int-test'
          java-version: ${{ matrix.java-version }}

  int-test-stores:
    name: CI intTest versioned/stores
    runs-on: ubuntu-22.04
    strategy:
      max-parallel: 2
      matrix:
        java-version: ['11'] # Ideally also '17', but GH concurrent job limit ... :(
    steps:
      - uses: actions/checkout@8ade135a41bc03ea155e62e844d188df1ea18608 # v4.1.0
      - name: Setup runner
        uses: ./.github/actions/setup-runner
      - name: Setup Java, Gradle
        uses: ./.github/actions/dev-tool-java
        with:
          java-version: ${{ matrix.java-version }}

      - name: Prepare Gradle build cache
        uses: ./.github/actions/ci-incr-build-cache-prepare
        with:
          java-version: ${{ matrix.java-version }}

      - name: Gradle / intTest versioned/stores
        run: |
          echo "::group::Collect :nessie-versioned-storage projects"
          ./gradlew :listProjectsWithPrefix --prefix :nessie-versioned-storage- --output ../storage-prjs.txt
          echo "::endgroup::"

          echo "::group::Collect :nessie-versioned-persist projects"
          ./gradlew :listProjectsWithPrefix --prefix :nessie-versioned-persist- --output ../persist-prjs.txt
          echo "::endgroup::"

          ./gradlew $(cat ../persist-prjs.txt) $(cat ../storage-prjs.txt) --scan

      - name: Capture Test Reports
        uses: actions/upload-artifact@v3
        if: ${{ failure() }}
        with:
          name: ci-inttest-stores-reports
          path: |
            **/build/reports/*
            **/build/test-results/*
          retention-days: 7

      - name: Save partial Gradle build cache
        uses: ./.github/actions/ci-incr-build-cache-save
        if: github.event_name == 'push' && github.ref == 'refs/heads/main'
        with:
          job-name: 'int-test-stores'
          java-version: ${{ matrix.java-version }}

  int-test-integrations:
    name: CI intTest integrations
    runs-on: ubuntu-22.04
    env:
      SPARK_LOCAL_IP: localhost
    steps:
      - uses: actions/checkout@8ade135a41bc03ea155e62e844d188df1ea18608 # v4.1.0
      - name: Setup runner
        uses: ./.github/actions/setup-runner
      - name: Setup Java, Gradle
        uses: ./.github/actions/dev-tool-java

      - name: Prepare Gradle build cache
        uses: ./.github/actions/ci-incr-build-cache-prepare

      - name: Gradle / intTest integrations
        run: |
          echo "::group::Collect :nessie-spark-extensions projects"
          ./gradlew :listProjectsWithPrefix --prefix :nessie-spark-ext --output ../spark-prjs.txt
          echo "::endgroup::"

          ./gradlew $(cat ../spark-prjs.txt) --scan

      - name: Capture Test Reports
        uses: actions/upload-artifact@v3
        if: ${{ failure() }}
        with:
          name: ci-inttest-integrations-reports
          path: |
            **/build/reports/*
            **/build/test-results/*
          retention-days: 7

      - name: Save partial Gradle build cache
        uses: ./.github/actions/ci-incr-build-cache-save
        if: github.event_name == 'push' && github.ref == 'refs/heads/main'
        with:
          job-name: 'int-test-integrations'

  int-test-quarkus:
    name: CI intTest Quarkus
    runs-on: ubuntu-22.04
    strategy:
      max-parallel: 2
      matrix:
        java-version: ['11'] # Ideally also '17', but GH concurrent job limit ... :(
    steps:
      - uses: actions/checkout@8ade135a41bc03ea155e62e844d188df1ea18608 # v4.1.0
      - name: Setup runner
        uses: ./.github/actions/setup-runner
      - name: Setup Java, Gradle
        uses: ./.github/actions/dev-tool-java
        with:
          java-version: ${{ matrix.java-version }}

      - name: Prepare Gradle build cache
        uses: ./.github/actions/ci-incr-build-cache-prepare
        with:
          java-version: ${{ matrix.java-version }}

      - name: Gradle / intTest Quarkus
        uses: gradle/gradle-build-action@v2
        with:
          arguments: |
            :nessie-quarkus:intTest
            :nessie-quarkus-cli:intTest
            :nessie-events-quarkus:intTest
            --scan

      - name: Dump quarkus.log
        if: ${{ failure() }}
        run: |
          find . -path "**/build/quarkus.log" | while read ql ; do
            echo "::group::Quarkus build log $ql"
            cat $ql
            echo "::endgroup::"
          done

      - name: Capture Test Reports
        uses: actions/upload-artifact@v3
        if: ${{ failure() }}
        with:
          name: ci-inttest-quarkus-reports
          path: |
            **/build/quarkus.log
            **/build/reports/*
            **/build/test-results/*
          retention-days: 7

      - name: Save partial Gradle build cache
        uses: ./.github/actions/ci-incr-build-cache-save
        if: github.event_name == 'push' && github.ref == 'refs/heads/main'
        with:
          job-name: 'int-test-quarkus'
          java-version: ${{ matrix.java-version }}

  determine-jobs:
    name: CI Determine jobs
    runs-on: ubuntu-22.04
    outputs:
      # Each "conditional" job has a mapped output here, also a "non-PR" case and a "PR" case
      # with label and globs (at the end of the script).
      docker: ${{ steps.determine.outputs.docker }}
      nesqueit: ${{ steps.determine.outputs.nesqueit }}
      helm: ${{ steps.determine.outputs.helm }}
    steps:
      - name: install minimatch
        shell: bash
        run: npm install minimatch@"^7.4.3"
      - name: Determine conditional jobs to run
        id: determine
        uses: actions/github-script@v6
        with:
          script: |
            const {Minimatch} = require("minimatch");

            async function getChangedFiles(pull_request) {
              const listFilesOptions = github.rest.pulls.listFiles.endpoint.merge({
                owner: pull_request.base.repo.owner.login,
                repo: pull_request.base.repo.name,
                pull_number: pull_request.number,
              });
              return github.paginate(listFilesOptions).then(resp => resp.map(f => f.filename))
            }

            function isMatch(changedFile, matchers) {
              for (const matcher of matchers) {
                if (matcher.match(changedFile)) {
                  core.info("Match found for changed file " + changedFile);
                  return true;
                }
              }
              core.info("No changed file matched the globs");
              return false;
            }

            function filesMatchAnyGlob(changedFiles, globs) {
              const matchers = globs.map(g => new Minimatch(g));
              for (const changedFile of changedFiles) {
                if (isMatch(changedFile, matchers)) {
                  return true;
                }
              }
              return false;
            }

            function checkLabelOrGlobs(pr, changedFiles, labelName, globs) {
              core.info("Checking for " + labelName);

              // If the PR has the given label, then let the conditional job run.
              for (const label of pr.labels) {
                if (label.name === labelName) {
                  core.info("PR has label " + labelName);
                  return true;
                }
              }

              // For pull request, check the list of changed files against the given globs.
              // Let the conditional job run, if any of the globs matches and of the changed files.
              core.info("Checking changed files against globs " + globs);
              return filesMatchAnyGlob(changedFiles, globs);
            }

            ///////////////////////////////////////////////////////////////////////////////////////
            // "Non PR" case:
            // Behavior of the conditional jobs when the event that triggered the workflow is not
            // for a pull request (usually a push-to-main).
            //
            if (!context.payload || !context.payload.pull_request) {
              core.info("Not a pull-request, enabling all jobs.");

              // Yield default values for all event payload types, except pull_request
              core.setOutput('docker', true);
              core.setOutput('nesqueit', false); // handled in a scheduled job
              core.setOutput('helm', true);
              return true;
            }

            ///////////////////////////////////////////////////////////////////////////////////////
            // "PR" case:
            // Determine the whether the conditional jobs shall run, based on a label name, which
            // has been manually added, or, if the label is not present, based on some globs to
            // check.

            const pr = context.payload.pull_request
            core.info("pull-request #" + pr.number);

            // Retrieves the full list of changed files for the PR.
            const changedFiles = await getChangedFiles(pr);

            core.startGroup('PR job check / Docker');
            core.setOutput('docker', checkLabelOrGlobs(pr, changedFiles,
              'pr-docker',
              ['gradle/wrapper/**', 'tools/dockerbuild/**', '.github/**']));
            core.endGroup();

            core.startGroup('PR job check / NesQuEIT');
            core.setOutput('nesqueit', checkLabelOrGlobs(pr, changedFiles,
              'pr-integrations',
              ['gradle/wrapper/**', 'integrations/**', 'api/**', '.github/**']));
            core.endGroup();

            core.startGroup('PR job check / Helm');
            core.setOutput('helm', checkLabelOrGlobs(pr, changedFiles,
              'pr-helm',
              ['helm/**', '.github/**']));
            core.endGroup();

  docker-testing:
    name: CI Docker build and publishing
    runs-on: ubuntu-22.04
    needs:
      - determine-jobs
    if: needs.determine-jobs.outputs.docker == 'true'
    steps:
      - name: Checkout
        uses: actions/checkout@8ade135a41bc03ea155e62e844d188df1ea18608 # v4.1.0
        with:
          fetch-depth: 0
      - name: Setup runner
        uses: ./.github/actions/setup-runner
      - name: Setup Java, Gradle
        uses: ./.github/actions/dev-tool-java

      - name: Setup docker-registry
        run: |
          sudo apt-get install -y docker-registry apache2-utils 
          cat <<! > config.yml
          version: 0.1
          log:
            fields:
              service: registry
          storage:
            cache:
              blobdescriptor: inmemory
            filesystem:
              rootdirectory: /var/lib/docker-registry
            delete:
              enabled: true
          http:
            addr: 127.0.0.1:5000
            headers:
              X-Content-Type-Options: [nosniff]
          auth:
            htpasswd:
              realm: basic-realm
              path: /etc/docker/registry/htpasswd
          health:
            storagedriver:
              enabled: true
              interval: 10s
              threshold: 3
          !
          sudo mv config.yml /etc/docker/registry/config.yml
          sudo htpasswd -cBb /etc/docker/registry/htpasswd micky mouse 
          
          sudo service docker-registry restart
          
          echo mouse | docker login -u micky --password-stdin localhost:5000

          VERSION="$(cat version.txt)"
          DOCKER_VERSION="${VERSION%-SNAPSHOT}"
          echo "DOCKER_VERSION=${DOCKER_VERSION}" >> ${GITHUB_ENV}
          echo "DOCKER_IMAGE=localhost:5000/nessie-testing" >> ${GITHUB_ENV}

      - name: Prepare Gradle build cache
        uses: ./.github/actions/ci-incr-build-cache-prepare

      - name: Docker images publishing
        env:
          ARTIFACTS: ../build-artifacts
        run: |
          rm -rf "${ARTIFACTS}"
          mkdir -p "${ARTIFACTS}"

          tools/dockerbuild/build-push-images.sh \
            -a "${ARTIFACTS}" \
            -g ":nessie-quarkus" \
            -p "servers/quarkus-server" \
            ${DOCKER_IMAGE}
          rm -rf "${ARTIFACTS}"

      - name: Cleanup buildx
        run: |
          docker buildx use default
          docker buildx rm nessiebuild

      - name: Docker images exist test
        run: |
          docker pull ${DOCKER_IMAGE}:latest
          docker pull ${DOCKER_IMAGE}:latest-java
          docker pull ${DOCKER_IMAGE}:${DOCKER_VERSION}
          docker pull ${DOCKER_IMAGE}:${DOCKER_VERSION}-java
          cat <<! >> $GITHUB_STEP_SUMMARY
          ## Docker images
          
          \`\`\`
          $(docker images)
          \`\`\`
          !

      - name: Check if Docker Java image works
        run: |
          docker run --detach --name nessie ${DOCKER_IMAGE}:latest-java
          echo "Let Nessie Java Docker image run for one minute (to make sure it starts up fine)..."
          for i in {1..60}; do
            STATUS="$(docker container inspect nessie | jq -r '.[0].State.Status')"
            if [[ ${STATUS} != "running" ]] ; then
              echo "Nessie Java Docker image stopped on its own ... a bug?" > /dev/stderr
              docker logs nessie
              cat <<! >> $GITHUB_STEP_SUMMARY
              ## Nessie Java Docker image FAILED

              \`\`\`
              $(docker logs nessie)
              \`\`\`
          !
              exit 1
            fi
            sleep 1
          done
          echo "## Nessie Java Docker image smoke test: PASSED" >> $GITHUB_STEP_SUMMARY  
          echo "Nessie Java Docker image smoke test: PASSED"
          docker stop nessie
          docker rm nessie

  nesqueit:
    name: CI NesQuEIT
    runs-on: ubuntu-22.04
    needs:
      - determine-jobs
    # Only run NesQuEIT tests for PRs, if requested. This job can easily run for 30+ minutes.
    if: github.event_name == 'pull_request' && needs.determine-jobs.outputs.nesqueit == 'true'
    env:
      NESSIE_DIR: included-builds/nessie
      NESSIE_PATCH_REPOSITORY: ''
      NESSIE_PATCH_BRANCH: ''
      NESQUEIT_REPOSITORY: projectnessie/query-engine-integration-tests
      NESQUEIT_BRANCH: main
      ICEBERG_DIR: included-builds/iceberg
      ICEBERG_MAIN_REPOSITORY: apache/iceberg
      ICEBERG_MAIN_BRANCH: master
      ICEBERG_PATCH_REPOSITORY: snazy/iceberg
      ICEBERG_PATCH_BRANCH: iceberg-nesqueit
      SPARK_LOCAL_IP: localhost
    steps:
      - name: Prepare Git
        run: |
          git config --global user.email "integrations-testing@projectnessie.org"
          git config --global user.name "Integrations Testing [Bot]"

      - name: Checkout NeQuEIT repo
        uses: actions/checkout@8ade135a41bc03ea155e62e844d188df1ea18608 # v4.1.0
        with:
          repository: ${{env.NESQUEIT_REPOSITORY}}
          ref: ${{env.NESQUEIT_BRANCH}}

      - name: Free disk space
        uses: ./.github/actions/free-disk-space

      - name: Setup runner
        uses: ./.github/actions/setup-runner
        with:
          more-memory: 'true'

      - name: Checkout and patch Nessie PR
        uses: ./.github/actions/patch-git
        with:
          name: Nessie
          local-dir: ${{env.NESSIE_DIR}}
          main-repository: ${{ env.GITHUB_REPOSITORY }}
          patch-repository: ${{env.NESSIE_PATCH_REPOSITORY}}
          patch-branch: ${{env.NESSIE_PATCH_BRANCH}}
          work-branch: nessie-integration-patched

      - name: Checkout and patch Iceberg
        uses: ./.github/actions/patch-git
        with:
          name: Nessie
          local-dir: ${{env.ICEBERG_DIR}}
          main-repository: ${{env.ICEBERG_MAIN_REPOSITORY}}
          main-branch: ${{env.ICEBERG_MAIN_BRANCH}}
          patch-repository: ${{env.ICEBERG_PATCH_REPOSITORY}}
          patch-branch: ${{env.ICEBERG_PATCH_BRANCH}}
          work-branch: iceberg-integration-patched

      - name: Set up JDK ${{ matrix.java-version }}
        uses: actions/setup-java@v3
        with:
          distribution: 'temurin'
          java-version: 11

      - name: Iceberg Nessie test
        uses: gradle/gradle-build-action@v2
        with:
          cache-read-only: true
          arguments: :iceberg:iceberg-nessie:test --scan

      - name: Nessie Spark 3.2 / 2.12 Extensions test
        uses: gradle/gradle-build-action@v2
        with:
          arguments: :nessie:nessie-iceberg:nessie-spark-extensions-3.2_2.12:test :nessie:nessie-iceberg:nessie-spark-extensions-3.2_2.12:intTest --scan

      - name: Nessie Spark 3.3 / 2.12 Extensions test
        uses: gradle/gradle-build-action@v2
        with:
          arguments: :nessie:nessie-iceberg:nessie-spark-extensions-3.3_2.12:test :nessie:nessie-iceberg:nessie-spark-extensions-3.3_2.12:intTest --scan

      - name: Nessie Spark 3.4 / 2.13 Extensions test
        uses: gradle/gradle-build-action@v2
        with:
          arguments: :nessie:nessie-iceberg:nessie-spark-extensions-3.3_2.13:test :nessie:nessie-iceberg:nessie-spark-extensions-3.3_2.13:intTest --scan

      #- name: Publish Nessie + Iceberg to local Maven repo
      #  uses: gradle/gradle-build-action@v2
      #  with:
      #    arguments: publishLocal --scan
      #
      #- name: Gather locally published versions
      #  run: |
      #    NESSIE_VERSION="$(cat included-builds/nessie/version.txt)"
      #    ICEBERG_VERSION="$(cat included-builds/iceberg/build/iceberg-build.properties | grep '^git.build.version=' | cut -d= -f2)"
      #    echo "NESSIE_VERSION=${NESSIE_VERSION}" >> ${GITHUB_ENV}
      #    echo "ICEBERG_VERSION=${ICEBERG_VERSION}" >> ${GITHUB_ENV}
      #    cat <<! >> $GITHUB_STEP_SUMMARY
      #    ## Published versions
      #    | Published Nessie version | Published Iceberg version |
      #    | ------------------------ | ------------------------- |
      #    | ${NESSIE_VERSION}        | ${ICEBERG_VERSION}        |
      #    !

      - name: Tools & Integrations tests
        uses: gradle/gradle-build-action@v2
        with:
          arguments: intTest --scan

  helm-testing:
    name: CI Lint & Test Helm chart
    runs-on: ubuntu-22.04
    needs:
      - determine-jobs
    if: needs.determine-jobs.outputs.helm == 'true'
    steps:
      - name: Checkout
        uses: actions/checkout@8ade135a41bc03ea155e62e844d188df1ea18608 # v4.1.0
        with:
          fetch-depth: 0

      # Skip remaining steps, if current Nessie release version is not available
      - name: Precheck
        run: |
          EXPECT_VERSION="$(cat helm/nessie/Chart.yaml | grep -E '^version: ')"
          curl https://charts.projectnessie.org/index.yaml 2>/dev/null | grep -q "${EXPECT_VERSION}" || (
            echo "Skipping Helm checks, because Nessie Helm chart version ${EXPECT_VERSION} is not available"
            echo "WF_EXEC=false" >> ${GITHUB_ENV}  
          )

      - name: Set up Helm
        if: env.WF_EXEC != 'false'
        uses: azure/setup-helm@v3
        with:
          # https://helm.sh/docs/topics/version_skew/
          version: 'v3.11.3'

      - name: Setup Python
        if: env.WF_EXEC != 'false'
        uses: actions/setup-python@v4
        with:
          python-version: '3.11'

      - name: Set up chart-testing
        if: env.WF_EXEC != 'false'
        uses: helm/chart-testing-action@v2.4.0
        with:
          # https://github.com/adrienverge/yamllint/tags
          # default is 1.27.1, but leads to Python install issues
          yamllint_version: 1.31.0
          # https://github.com/23andMe/Yamale/releases
          # default is 3.0.4, just bump to current latest patch version
          yamale_version: 3.0.8

      - name: Run chart-testing (list-changed)
        if: env.WF_EXEC != 'false'
        run: |
          ct list-changed --target-branch ${{ github.event.repository.default_branch }}

      - name: Run chart-testing (lint)
        if: env.WF_EXEC != 'false'
        run: ct lint --debug --charts ./helm/nessie

      - name: Set up & Start Minikube
        if: env.WF_EXEC != 'false'
        uses: medyagh/setup-minikube@v0.0.14
        with:
          # If required, use the matrix strategy against this option to test against multiple Kubernetes versions:
          kubernetes-version: stable
          cache: false

      - name: Show pods
        if: env.WF_EXEC != 'false'
        run: kubectl get pods -A

      - name: Run chart-testing (install)
        if: env.WF_EXEC != 'false'
        run: ct install --debug --charts ./helm/nessie

  site:
    name: CI Website
    runs-on: ubuntu-22.04
    steps:
      - uses: actions/checkout@8ade135a41bc03ea155e62e844d188df1ea18608 # v4.1.0
      - name: Setup Python
        uses: ./.github/actions/dev-tool-python
        with:
          python-version: '3.8'
      - name: Upgrade pip
        run: |
          # install pip=>20.1 to use "pip cache dir"
          python3 -m pip install --upgrade pip
      - name: Install dependencies
        run: python3 -m pip install -r ./site/requirements.txt
      - name: Check code block indentation
        run: python3 check_code_indent.py
        working-directory: ./site
      - name: Generate Static Site
        run: mkdocs build
        working-directory: ./site
      - name: Deploy Static Site to GitHub
        if: github.event_name == 'push' && github.ref == 'refs/heads/main' && github.repository == 'projectnessie/nessie'
        uses: peaceiris/actions-gh-pages@v3
        with:
          external_repository: projectnessie/projectnessie.github.io
          publish_branch: main
          deploy_key: ${{ secrets.NESSIE_SITE_DEPLOY_KEY }}
          publish_dir: ./site/site
          cname: projectnessie.org

  store-cache:
    # Store the Gradle cache to GH cache as soon as all relevant Nessie/Gradle jobs have finished.
    name: CI Store Cache
    runs-on: ubuntu-22.04
    if: github.event_name == 'push' && github.ref == 'refs/heads/main'
    needs:
      # Only include jobs that use Nessie's Gradle cache, especially excluding NesQuEIT, which
      # is a "very special" citizen and also not run for "main" CI, which does
      # not add anything to the Gradle cache that's not already produced by other jobs.
      - code-checks
      - test
      - test-quarkus
      - int-test
      - int-test-stores
      - int-test-integrations
      - int-test-quarkus
    strategy:
      max-parallel: 2
      matrix:
        java-version: ['11'] # Ideally also '17', but GH concurrent job limit ... :(
    steps:
      - uses: actions/checkout@8ade135a41bc03ea155e62e844d188df1ea18608 # v4.1.0
      - name: Setup runner
        uses: ./.github/actions/setup-runner
      - name: Setup Java, Gradle
        uses: ./.github/actions/dev-tool-java
        with:
          java-version: ${{ matrix.java-version }}

      - name: Collect partial Gradle build caches
        uses: ./.github/actions/ci-incr-build-cache-prepare
        with:
          java-version: ${{ matrix.java-version }}
          cache-read-only: false

      - name: Trigger Gradle home cleanup
        run: ./gradlew --no-daemon :showVersion

      # Note: the "Post Gradle invocation" archives the updated build cache.