Update iceberg to v1.4.2 #5194
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # Copyright (C) 2020 Dremio | |
| # | |
| # Licensed under the Apache License, Version 2.0 (the "License"); | |
| # you may not use this file except in compliance with the License. | |
| # You may obtain a copy of the License at | |
| # | |
| # http://www.apache.org/licenses/LICENSE-2.0 | |
| # | |
| # Unless required by applicable law or agreed to in writing, software | |
| # distributed under the License is distributed on an "AS IS" BASIS, | |
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |
| # See the License for the specific language governing permissions and | |
| # limitations under the License. | |
| # Unifies main + PR workflow. | |
| # | |
| # The unified CI workflow consists of 2 "stages": | |
| # - Checks - test, intTest, NesQuEIT, etc | |
| # - Finalize - a "success" dummy job for PRs + a "save to github-cache" job for push-to-main | |
| # | |
| # Utilizes the Gradle build cache for all stages. The updated build cache | |
| # of the jobs in the checks stage are saved as artifacts (with the minimum | |
| # retention period). The updated build cache is pushed back to GigHub's | |
| # cache when the checks have successfully finished. | |
| # | |
| # Java CI runs against Java 11 and Java 17, where it is meaningful. | |
| # (Spark tests always run against 11, so not run against Java 17.) | |
| name: CI build | |
| on: | |
| push: | |
| branches: | |
| - main | |
| - release-* | |
| paths-ignore: | |
| - 'LICENSE' | |
| - 'NOTICE' | |
| - '**.md' | |
| - '!site/**' | |
| - '.github/renovate.json5' | |
| - '.github/workflows/release*.yml' | |
| - '.github/workflows/check*.yml' | |
| - '.idea/**' | |
| - '.editorconfig' | |
| pull_request: | |
| types: [labeled, opened, synchronize, reopened] | |
| # For the main branch: let all CI runs complete, one after the other. This has a couple advantages: | |
| # * Site deployments happen in commit-order | |
| # * Saved Gradle cache are persisted in commit-order | |
| # * (Potentially) more GH runners available for PRs | |
| concurrency: | |
| # PRs: 1 CI run concurrently / older ones are cancelled | |
| # main branch: 1 CI run concurrently / all commits / no cancellation | |
| group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.ref }} | |
| cancel-in-progress: ${{ github.ref != 'refs/heads/main' }} | |
| jobs: | |
| code-checks: | |
| name: CI Code Checks et al | |
| runs-on: ubuntu-22.04 | |
| strategy: | |
| max-parallel: 2 | |
| matrix: | |
| java-version: ['11'] # Ideally also '17', but GH concurrent job limit ... :( | |
| steps: | |
| - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 | |
| - name: Setup runner | |
| uses: ./.github/actions/setup-runner | |
| - name: Setup Java, Gradle | |
| uses: ./.github/actions/dev-tool-java | |
| with: | |
| java-version: ${{ matrix.java-version }} | |
| # Needed for the Quarkus plugin - can likely go away once we use Quarkus 3 or newer | |
| - name: Bump Gradle daemon heap | |
| run: sed -i 's/-Xms.*/-Xms6G -Xmx6G -XX:MaxMetaspaceSize=1g \\/' gradle.properties | |
| - name: Prepare Gradle build cache | |
| uses: ./.github/actions/ci-incr-build-cache-prepare | |
| with: | |
| java-version: ${{ matrix.java-version }} | |
| - name: Gradle / Compile | |
| uses: gradle/gradle-build-action@v2 | |
| with: | |
| arguments: | | |
| spotlessCheck | |
| compileAll | |
| -x :nessie-quarkus:compileAll | |
| -x :nessie-quarkus-cli:compileAll | |
| -x :nessie-events-quarkus:compileAll | |
| --scan | |
| - name: Gradle / Compile Quarkus | |
| uses: gradle/gradle-build-action@v2 | |
| with: | |
| arguments: | | |
| :nessie-quarkus:compileAll | |
| :nessie-quarkus-cli:compileAll | |
| :nessie-events-quarkus:compileAll | |
| --scan | |
| - name: Gradle / Checkstyle | |
| uses: gradle/gradle-build-action@v2 | |
| with: | |
| arguments: checkstyle --scan | |
| - name: Gradle / Assemble | |
| uses: gradle/gradle-build-action@v2 | |
| with: | |
| arguments: assemble --scan | |
| - name: Gradle / Publish to Maven local | |
| uses: gradle/gradle-build-action@v2 | |
| with: | |
| arguments: publishToMavenLocal --scan | |
| # This is a rather quick one and uses the output of 'publishToMavenLocal', which uses the | |
| # outputs of 'assemble' | |
| - name: Gradle / build tools integration tests | |
| uses: gradle/gradle-build-action@v2 | |
| with: | |
| arguments: buildToolsIntegrationTest | |
| - name: Save partial Gradle build cache | |
| uses: ./.github/actions/ci-incr-build-cache-save | |
| if: github.event_name == 'push' && github.ref == 'refs/heads/main' | |
| with: | |
| job-name: 'code-checks' | |
| java-version: ${{ matrix.java-version }} | |
| test: | |
| name: CI Test | |
| runs-on: ubuntu-22.04 | |
| strategy: | |
| max-parallel: 2 | |
| matrix: | |
| java-version: ['11'] # Ideally also '17', but GH concurrent job limit ... :( | |
| steps: | |
| - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 | |
| - name: Setup runner | |
| uses: ./.github/actions/setup-runner | |
| - name: Setup Java, Gradle | |
| uses: ./.github/actions/dev-tool-java | |
| with: | |
| java-version: ${{ matrix.java-version }} | |
| - name: Prepare Gradle build cache | |
| uses: ./.github/actions/ci-incr-build-cache-prepare | |
| with: | |
| java-version: ${{ matrix.java-version }} | |
| - name: Gradle / test | |
| uses: gradle/gradle-build-action@v2 | |
| with: | |
| arguments: | | |
| test | |
| :nessie-client:check | |
| -x :nessie-client:intTest | |
| -x :nessie-quarkus:test | |
| -x :nessie-quarkus-cli:test | |
| -x :nessie-events-quarkus:test | |
| --scan | |
| - name: Capture Test Reports | |
| uses: actions/upload-artifact@v3 | |
| if: ${{ failure() }} | |
| with: | |
| name: ci-test-reports | |
| path: | | |
| **/build/reports/* | |
| **/build/test-results/* | |
| retention-days: 7 | |
| - name: Save partial Gradle build cache | |
| uses: ./.github/actions/ci-incr-build-cache-save | |
| if: github.event_name == 'push' && github.ref == 'refs/heads/main' | |
| with: | |
| job-name: 'test' | |
| java-version: ${{ matrix.java-version }} | |
| test-quarkus: | |
| name: CI Test Quarkus | |
| runs-on: ubuntu-22.04 | |
| strategy: | |
| max-parallel: 2 | |
| matrix: | |
| java-version: ['11'] # Ideally also '17', but GH concurrent job limit ... :( | |
| steps: | |
| - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 | |
| - name: Setup runner | |
| uses: ./.github/actions/setup-runner | |
| - name: Setup Java, Gradle | |
| uses: ./.github/actions/dev-tool-java | |
| with: | |
| java-version: ${{ matrix.java-version }} | |
| - name: Prepare Gradle build cache | |
| uses: ./.github/actions/ci-incr-build-cache-prepare | |
| with: | |
| java-version: ${{ matrix.java-version }} | |
| - name: Gradle / Test Quarkus | |
| uses: gradle/gradle-build-action@v2 | |
| with: | |
| arguments: | | |
| :nessie-client:test | |
| :nessie-quarkus:test | |
| :nessie-events-quarkus:test | |
| --scan | |
| - name: Dump quarkus.log | |
| if: ${{ failure() }} | |
| run: | | |
| find . -path "**/build/quarkus.log" | while read ql ; do | |
| echo "::group::Quarkus build log $ql" | |
| cat $ql | |
| echo "::endgroup::" | |
| done | |
| - name: Capture Test Reports | |
| uses: actions/upload-artifact@v3 | |
| if: ${{ failure() }} | |
| with: | |
| name: ci-test-quarkus-reports | |
| path: | | |
| **/build/quarkus.log | |
| **/build/reports/* | |
| **/build/test-results/* | |
| retention-days: 7 | |
| - name: Save partial Gradle build cache | |
| uses: ./.github/actions/ci-incr-build-cache-save | |
| if: github.event_name == 'push' && github.ref == 'refs/heads/main' | |
| with: | |
| job-name: 'test-quarkus' | |
| java-version: ${{ matrix.java-version }} | |
| int-test: | |
| name: CI intTest | |
| runs-on: ubuntu-22.04 | |
| strategy: | |
| max-parallel: 2 | |
| matrix: | |
| java-version: ['11'] # Ideally also '17', but GH concurrent job limit ... :( | |
| env: | |
| SPARK_LOCAL_IP: localhost | |
| steps: | |
| - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 | |
| - name: Setup runner | |
| uses: ./.github/actions/setup-runner | |
| - name: Setup Java, Gradle | |
| uses: ./.github/actions/dev-tool-java | |
| with: | |
| java-version: ${{ matrix.java-version }} | |
| - name: Prepare Gradle build cache | |
| uses: ./.github/actions/ci-incr-build-cache-prepare | |
| - name: Gradle / intTest | |
| run: | | |
| echo "::group::Collect :nessie-versioned-storage projects" | |
| ./gradlew :listProjectsWithPrefix --prefix :nessie-versioned-persist- --output ../persist-prjs.txt --exclude | |
| echo "::endgroup::" | |
| echo "::group::Collect :nessie-versioned-persist projects" | |
| ./gradlew :listProjectsWithPrefix --prefix :nessie-versioned-storage- --output ../storage-prjs.txt --exclude | |
| echo "::endgroup::" | |
| echo "::group::Collect :nessie-spark-extensions projects" | |
| ./gradlew :listProjectsWithPrefix --prefix :nessie-spark-ext --output ../spark-prjs.txt --exclude | |
| echo "::endgroup::" | |
| ./gradlew intTest \ | |
| -x :nessie-quarkus:intTest \ | |
| -x :nessie-quarkus-cli:intTest \ | |
| -x :nessie-events-quarkus:intTest \ | |
| $(cat ../persist-prjs.txt) \ | |
| $(cat ../storage-prjs.txt) \ | |
| $(cat ../spark-prjs.txt) \ | |
| --scan | |
| - name: Capture Test Reports | |
| uses: actions/upload-artifact@v3 | |
| if: ${{ failure() }} | |
| with: | |
| name: ci-inttest-reports | |
| path: | | |
| **/build/reports/* | |
| **/build/test-results/* | |
| retention-days: 7 | |
| - name: Save partial Gradle build cache | |
| uses: ./.github/actions/ci-incr-build-cache-save | |
| if: github.event_name == 'push' && github.ref == 'refs/heads/main' | |
| with: | |
| job-name: 'int-test' | |
| java-version: ${{ matrix.java-version }} | |
| int-test-stores: | |
| name: CI intTest versioned/stores | |
| runs-on: ubuntu-22.04 | |
| strategy: | |
| max-parallel: 2 | |
| matrix: | |
| java-version: ['11'] # Ideally also '17', but GH concurrent job limit ... :( | |
| steps: | |
| - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 | |
| - name: Setup runner | |
| uses: ./.github/actions/setup-runner | |
| - name: Setup Java, Gradle | |
| uses: ./.github/actions/dev-tool-java | |
| with: | |
| java-version: ${{ matrix.java-version }} | |
| - name: Prepare Gradle build cache | |
| uses: ./.github/actions/ci-incr-build-cache-prepare | |
| with: | |
| java-version: ${{ matrix.java-version }} | |
| - name: Gradle / intTest versioned/stores | |
| run: | | |
| echo "::group::Collect :nessie-versioned-storage projects" | |
| ./gradlew :listProjectsWithPrefix --prefix :nessie-versioned-storage- --output ../storage-prjs.txt | |
| echo "::endgroup::" | |
| echo "::group::Collect :nessie-versioned-persist projects" | |
| ./gradlew :listProjectsWithPrefix --prefix :nessie-versioned-persist- --output ../persist-prjs.txt | |
| echo "::endgroup::" | |
| ./gradlew $(cat ../persist-prjs.txt) $(cat ../storage-prjs.txt) --scan | |
| - name: Capture Test Reports | |
| uses: actions/upload-artifact@v3 | |
| if: ${{ failure() }} | |
| with: | |
| name: ci-inttest-stores-reports | |
| path: | | |
| **/build/reports/* | |
| **/build/test-results/* | |
| retention-days: 7 | |
| - name: Save partial Gradle build cache | |
| uses: ./.github/actions/ci-incr-build-cache-save | |
| if: github.event_name == 'push' && github.ref == 'refs/heads/main' | |
| with: | |
| job-name: 'int-test-stores' | |
| java-version: ${{ matrix.java-version }} | |
| int-test-integrations: | |
| name: CI intTest integrations | |
| runs-on: ubuntu-22.04 | |
| env: | |
| SPARK_LOCAL_IP: localhost | |
| steps: | |
| - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 | |
| - name: Setup runner | |
| uses: ./.github/actions/setup-runner | |
| - name: Setup Java, Gradle | |
| uses: ./.github/actions/dev-tool-java | |
| - name: Prepare Gradle build cache | |
| uses: ./.github/actions/ci-incr-build-cache-prepare | |
| - name: Gradle / intTest integrations | |
| run: | | |
| echo "::group::Collect :nessie-spark-extensions projects" | |
| ./gradlew :listProjectsWithPrefix --prefix :nessie-spark-ext --output ../spark-prjs.txt | |
| echo "::endgroup::" | |
| ./gradlew $(cat ../spark-prjs.txt) --scan | |
| - name: Capture Test Reports | |
| uses: actions/upload-artifact@v3 | |
| if: ${{ failure() }} | |
| with: | |
| name: ci-inttest-integrations-reports | |
| path: | | |
| **/build/reports/* | |
| **/build/test-results/* | |
| retention-days: 7 | |
| - name: Save partial Gradle build cache | |
| uses: ./.github/actions/ci-incr-build-cache-save | |
| if: github.event_name == 'push' && github.ref == 'refs/heads/main' | |
| with: | |
| job-name: 'int-test-integrations' | |
| int-test-quarkus: | |
| name: CI intTest Quarkus | |
| runs-on: ubuntu-22.04 | |
| strategy: | |
| max-parallel: 2 | |
| matrix: | |
| java-version: ['11'] # Ideally also '17', but GH concurrent job limit ... :( | |
| steps: | |
| - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 | |
| - name: Setup runner | |
| uses: ./.github/actions/setup-runner | |
| - name: Setup Java, Gradle | |
| uses: ./.github/actions/dev-tool-java | |
| with: | |
| java-version: ${{ matrix.java-version }} | |
| - name: Prepare Gradle build cache | |
| uses: ./.github/actions/ci-incr-build-cache-prepare | |
| with: | |
| java-version: ${{ matrix.java-version }} | |
| - name: Gradle / intTest Quarkus | |
| uses: gradle/gradle-build-action@v2 | |
| with: | |
| arguments: | | |
| :nessie-quarkus:intTest | |
| :nessie-quarkus-cli:intTest | |
| :nessie-events-quarkus:intTest | |
| --scan | |
| - name: Dump quarkus.log | |
| if: ${{ failure() }} | |
| run: | | |
| find . -path "**/build/quarkus.log" | while read ql ; do | |
| echo "::group::Quarkus build log $ql" | |
| cat $ql | |
| echo "::endgroup::" | |
| done | |
| - name: Capture Test Reports | |
| uses: actions/upload-artifact@v3 | |
| if: ${{ failure() }} | |
| with: | |
| name: ci-inttest-quarkus-reports | |
| path: | | |
| **/build/quarkus.log | |
| **/build/reports/* | |
| **/build/test-results/* | |
| retention-days: 7 | |
| - name: Save partial Gradle build cache | |
| uses: ./.github/actions/ci-incr-build-cache-save | |
| if: github.event_name == 'push' && github.ref == 'refs/heads/main' | |
| with: | |
| job-name: 'int-test-quarkus' | |
| java-version: ${{ matrix.java-version }} | |
| determine-jobs: | |
| name: CI Determine jobs | |
| runs-on: ubuntu-22.04 | |
| outputs: | |
| # Each "conditional" job has a mapped output here, also a "non-PR" case and a "PR" case | |
| # with label and globs (at the end of the script). | |
| docker: ${{ steps.determine.outputs.docker }} | |
| nesqueit: ${{ steps.determine.outputs.nesqueit }} | |
| helm: ${{ steps.determine.outputs.helm }} | |
| steps: | |
| - name: install minimatch | |
| shell: bash | |
| run: npm install minimatch@"^7.4.3" | |
| - name: Determine conditional jobs to run | |
| id: determine | |
| uses: actions/github-script@v6 | |
| with: | |
| script: | | |
| const {Minimatch} = require("minimatch"); | |
| async function getChangedFiles(pull_request) { | |
| const listFilesOptions = github.rest.pulls.listFiles.endpoint.merge({ | |
| owner: pull_request.base.repo.owner.login, | |
| repo: pull_request.base.repo.name, | |
| pull_number: pull_request.number, | |
| }); | |
| return github.paginate(listFilesOptions).then(resp => resp.map(f => f.filename)) | |
| } | |
| function isMatch(changedFile, matchers) { | |
| for (const matcher of matchers) { | |
| if (matcher.match(changedFile)) { | |
| core.info("Match found for changed file " + changedFile); | |
| return true; | |
| } | |
| } | |
| core.info("No changed file matched the globs"); | |
| return false; | |
| } | |
| function filesMatchAnyGlob(changedFiles, globs) { | |
| const matchers = globs.map(g => new Minimatch(g)); | |
| for (const changedFile of changedFiles) { | |
| if (isMatch(changedFile, matchers)) { | |
| return true; | |
| } | |
| } | |
| return false; | |
| } | |
| function checkLabelOrGlobs(pr, changedFiles, labelName, globs) { | |
| core.info("Checking for " + labelName); | |
| // If the PR has the given label, then let the conditional job run. | |
| for (const label of pr.labels) { | |
| if (label.name === labelName) { | |
| core.info("PR has label " + labelName); | |
| return true; | |
| } | |
| } | |
| // For pull request, check the list of changed files against the given globs. | |
| // Let the conditional job run, if any of the globs matches and of the changed files. | |
| core.info("Checking changed files against globs " + globs); | |
| return filesMatchAnyGlob(changedFiles, globs); | |
| } | |
| /////////////////////////////////////////////////////////////////////////////////////// | |
| // "Non PR" case: | |
| // Behavior of the conditional jobs when the event that triggered the workflow is not | |
| // for a pull request (usually a push-to-main). | |
| // | |
| if (!context.payload || !context.payload.pull_request) { | |
| core.info("Not a pull-request, enabling all jobs."); | |
| // Yield default values for all event payload types, except pull_request | |
| core.setOutput('docker', true); | |
| core.setOutput('nesqueit', false); // handled in a scheduled job | |
| core.setOutput('helm', true); | |
| return true; | |
| } | |
| /////////////////////////////////////////////////////////////////////////////////////// | |
| // "PR" case: | |
| // Determine the whether the conditional jobs shall run, based on a label name, which | |
| // has been manually added, or, if the label is not present, based on some globs to | |
| // check. | |
| const pr = context.payload.pull_request | |
| core.info("pull-request #" + pr.number); | |
| // Retrieves the full list of changed files for the PR. | |
| const changedFiles = await getChangedFiles(pr); | |
| core.startGroup('PR job check / Docker'); | |
| core.setOutput('docker', checkLabelOrGlobs(pr, changedFiles, | |
| 'pr-docker', | |
| ['gradle/wrapper/**', 'tools/dockerbuild/**', '.github/**'])); | |
| core.endGroup(); | |
| core.startGroup('PR job check / NesQuEIT'); | |
| core.setOutput('nesqueit', checkLabelOrGlobs(pr, changedFiles, | |
| 'pr-integrations', | |
| ['gradle/wrapper/**', 'integrations/**', 'api/**', '.github/**'])); | |
| core.endGroup(); | |
| core.startGroup('PR job check / Helm'); | |
| core.setOutput('helm', checkLabelOrGlobs(pr, changedFiles, | |
| 'pr-helm', | |
| ['helm/**', '.github/**'])); | |
| core.endGroup(); | |
| docker-testing: | |
| name: CI Docker build and publishing | |
| runs-on: ubuntu-22.04 | |
| needs: | |
| - determine-jobs | |
| if: needs.determine-jobs.outputs.docker == 'true' | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 | |
| with: | |
| fetch-depth: 0 | |
| - name: Setup runner | |
| uses: ./.github/actions/setup-runner | |
| - name: Setup Java, Gradle | |
| uses: ./.github/actions/dev-tool-java | |
| - name: Setup docker-registry | |
| run: | | |
| sudo apt-get install -y docker-registry apache2-utils | |
| cat <<! > config.yml | |
| version: 0.1 | |
| log: | |
| fields: | |
| service: registry | |
| storage: | |
| cache: | |
| blobdescriptor: inmemory | |
| filesystem: | |
| rootdirectory: /var/lib/docker-registry | |
| delete: | |
| enabled: true | |
| http: | |
| addr: 127.0.0.1:5000 | |
| headers: | |
| X-Content-Type-Options: [nosniff] | |
| auth: | |
| htpasswd: | |
| realm: basic-realm | |
| path: /etc/docker/registry/htpasswd | |
| health: | |
| storagedriver: | |
| enabled: true | |
| interval: 10s | |
| threshold: 3 | |
| ! | |
| sudo mv config.yml /etc/docker/registry/config.yml | |
| sudo htpasswd -cBb /etc/docker/registry/htpasswd micky mouse | |
| sudo service docker-registry restart | |
| echo mouse | docker login -u micky --password-stdin localhost:5000 | |
| VERSION="$(cat version.txt)" | |
| DOCKER_VERSION="${VERSION%-SNAPSHOT}" | |
| echo "DOCKER_VERSION=${DOCKER_VERSION}" >> ${GITHUB_ENV} | |
| echo "DOCKER_IMAGE=localhost:5000/nessie-testing" >> ${GITHUB_ENV} | |
| - name: Prepare Gradle build cache | |
| uses: ./.github/actions/ci-incr-build-cache-prepare | |
| - name: Docker images publishing | |
| env: | |
| ARTIFACTS: ../build-artifacts | |
| run: | | |
| rm -rf "${ARTIFACTS}" | |
| mkdir -p "${ARTIFACTS}" | |
| tools/dockerbuild/build-push-images.sh \ | |
| -a "${ARTIFACTS}" \ | |
| -g ":nessie-quarkus" \ | |
| -p "servers/quarkus-server" \ | |
| ${DOCKER_IMAGE} | |
| rm -rf "${ARTIFACTS}" | |
| - name: Cleanup buildx | |
| run: | | |
| docker buildx use default | |
| docker buildx rm nessiebuild | |
| - name: Docker images exist test | |
| run: | | |
| docker pull ${DOCKER_IMAGE}:latest | |
| docker pull ${DOCKER_IMAGE}:latest-java | |
| docker pull ${DOCKER_IMAGE}:${DOCKER_VERSION} | |
| docker pull ${DOCKER_IMAGE}:${DOCKER_VERSION}-java | |
| cat <<! >> $GITHUB_STEP_SUMMARY | |
| ## Docker images | |
| \`\`\` | |
| $(docker images) | |
| \`\`\` | |
| ! | |
| - name: Check if Docker Java image works | |
| run: | | |
| docker run --detach --name nessie ${DOCKER_IMAGE}:latest-java | |
| echo "Let Nessie Java Docker image run for one minute (to make sure it starts up fine)..." | |
| for i in {1..60}; do | |
| STATUS="$(docker container inspect nessie | jq -r '.[0].State.Status')" | |
| if [[ ${STATUS} != "running" ]] ; then | |
| echo "Nessie Java Docker image stopped on its own ... a bug?" > /dev/stderr | |
| docker logs nessie | |
| cat <<! >> $GITHUB_STEP_SUMMARY | |
| ## Nessie Java Docker image FAILED | |
| \`\`\` | |
| $(docker logs nessie) | |
| \`\`\` | |
| ! | |
| exit 1 | |
| fi | |
| sleep 1 | |
| done | |
| echo "## Nessie Java Docker image smoke test: PASSED" >> $GITHUB_STEP_SUMMARY | |
| echo "Nessie Java Docker image smoke test: PASSED" | |
| docker stop nessie | |
| docker rm nessie | |
| nesqueit: | |
| name: CI NesQuEIT | |
| runs-on: ubuntu-22.04 | |
| needs: | |
| - determine-jobs | |
| # Only run NesQuEIT tests for PRs, if requested. This job can easily run for 30+ minutes. | |
| if: github.event_name == 'pull_request' && needs.determine-jobs.outputs.nesqueit == 'true' | |
| env: | |
| NESSIE_DIR: included-builds/nessie | |
| NESSIE_PATCH_REPOSITORY: '' | |
| NESSIE_PATCH_BRANCH: '' | |
| NESQUEIT_REPOSITORY: projectnessie/query-engine-integration-tests | |
| NESQUEIT_BRANCH: main | |
| ICEBERG_DIR: included-builds/iceberg | |
| ICEBERG_MAIN_REPOSITORY: apache/iceberg | |
| ICEBERG_MAIN_BRANCH: main | |
| ICEBERG_PATCH_REPOSITORY: snazy/iceberg | |
| ICEBERG_PATCH_BRANCH: iceberg-nesqueit | |
| SPARK_LOCAL_IP: localhost | |
| steps: | |
| - name: Prepare Git | |
| run: | | |
| git config --global user.email "[email protected]" | |
| git config --global user.name "Integrations Testing [Bot]" | |
| - name: Checkout NeQuEIT repo | |
| uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 | |
| with: | |
| repository: ${{env.NESQUEIT_REPOSITORY}} | |
| ref: ${{env.NESQUEIT_BRANCH}} | |
| - name: Free disk space | |
| uses: ./.github/actions/free-disk-space | |
| - name: Setup runner | |
| uses: ./.github/actions/setup-runner | |
| with: | |
| more-memory: 'true' | |
| - name: Checkout and patch Nessie PR | |
| uses: ./.github/actions/patch-git | |
| with: | |
| name: Nessie | |
| local-dir: ${{env.NESSIE_DIR}} | |
| main-repository: ${{ env.GITHUB_REPOSITORY }} | |
| patch-repository: ${{env.NESSIE_PATCH_REPOSITORY}} | |
| patch-branch: ${{env.NESSIE_PATCH_BRANCH}} | |
| work-branch: nessie-integration-patched | |
| - name: Checkout and patch Iceberg | |
| uses: ./.github/actions/patch-git | |
| with: | |
| name: Nessie | |
| local-dir: ${{env.ICEBERG_DIR}} | |
| main-repository: ${{env.ICEBERG_MAIN_REPOSITORY}} | |
| main-branch: ${{env.ICEBERG_MAIN_BRANCH}} | |
| patch-repository: ${{env.ICEBERG_PATCH_REPOSITORY}} | |
| patch-branch: ${{env.ICEBERG_PATCH_BRANCH}} | |
| work-branch: iceberg-integration-patched | |
| - name: Set up JDK ${{ matrix.java-version }} | |
| uses: actions/setup-java@v3 | |
| with: | |
| distribution: 'temurin' | |
| java-version: 11 | |
| - name: Iceberg Nessie test | |
| uses: gradle/gradle-build-action@v2 | |
| with: | |
| cache-read-only: true | |
| arguments: :iceberg:iceberg-nessie:test --scan | |
| - name: Nessie Spark 3.2 / 2.12 Extensions test | |
| uses: gradle/gradle-build-action@v2 | |
| with: | |
| arguments: :nessie:nessie-iceberg:nessie-spark-extensions-3.2_2.12:test :nessie:nessie-iceberg:nessie-spark-extensions-3.2_2.12:intTest --scan | |
| - name: Nessie Spark 3.3 / 2.12 Extensions test | |
| uses: gradle/gradle-build-action@v2 | |
| with: | |
| arguments: :nessie:nessie-iceberg:nessie-spark-extensions-3.3_2.12:test :nessie:nessie-iceberg:nessie-spark-extensions-3.3_2.12:intTest --scan | |
| - name: Nessie Spark 3.4 / 2.13 Extensions test | |
| uses: gradle/gradle-build-action@v2 | |
| with: | |
| arguments: :nessie:nessie-iceberg:nessie-spark-extensions-3.4_2.13:test :nessie:nessie-iceberg:nessie-spark-extensions-3.4_2.13:intTest --scan | |
| - name: Nessie Spark 3.5 / 2.13 Extensions test | |
| uses: gradle/gradle-build-action@v2 | |
| with: | |
| arguments: :nessie:nessie-iceberg:nessie-spark-extensions-3.5_2.13:test :nessie:nessie-iceberg:nessie-spark-extensions-3.5_2.13:intTest --scan | |
| #- name: Publish Nessie + Iceberg to local Maven repo | |
| # uses: gradle/gradle-build-action@v2 | |
| # with: | |
| # arguments: publishLocal --scan | |
| # | |
| #- name: Gather locally published versions | |
| # run: | | |
| # NESSIE_VERSION="$(cat included-builds/nessie/version.txt)" | |
| # ICEBERG_VERSION="$(cat included-builds/iceberg/build/iceberg-build.properties | grep '^git.build.version=' | cut -d= -f2)" | |
| # echo "NESSIE_VERSION=${NESSIE_VERSION}" >> ${GITHUB_ENV} | |
| # echo "ICEBERG_VERSION=${ICEBERG_VERSION}" >> ${GITHUB_ENV} | |
| # cat <<! >> $GITHUB_STEP_SUMMARY | |
| # ## Published versions | |
| # | Published Nessie version | Published Iceberg version | | |
| # | ------------------------ | ------------------------- | | |
| # | ${NESSIE_VERSION} | ${ICEBERG_VERSION} | | |
| # ! | |
| - name: Tools & Integrations tests | |
| uses: gradle/gradle-build-action@v2 | |
| with: | |
| arguments: intTest --scan | |
| helm-testing: | |
| name: CI Lint & Test Helm chart | |
| runs-on: ubuntu-22.04 | |
| needs: | |
| - determine-jobs | |
| if: needs.determine-jobs.outputs.helm == 'true' | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 | |
| with: | |
| fetch-depth: 0 | |
| # Skip remaining steps, if current Nessie release version is not available | |
| - name: Precheck | |
| run: | | |
| EXPECT_VERSION="$(cat helm/nessie/Chart.yaml | grep -E '^version: ')" | |
| curl https://charts.projectnessie.org/index.yaml 2>/dev/null | grep -q "${EXPECT_VERSION}" || ( | |
| echo "Skipping Helm checks, because Nessie Helm chart version ${EXPECT_VERSION} is not available" | |
| echo "WF_EXEC=false" >> ${GITHUB_ENV} | |
| ) | |
| - name: Set up Helm | |
| if: env.WF_EXEC != 'false' | |
| uses: azure/setup-helm@v3 | |
| with: | |
| # https://helm.sh/docs/topics/version_skew/ | |
| version: 'v3.11.3' | |
| - name: Setup Python | |
| if: env.WF_EXEC != 'false' | |
| uses: actions/setup-python@v4 | |
| with: | |
| python-version: '3.11' | |
| - name: Set up chart-testing | |
| if: env.WF_EXEC != 'false' | |
| uses: helm/[email protected] | |
| with: | |
| # https://github.com/adrienverge/yamllint/tags | |
| # default is 1.27.1, but leads to Python install issues | |
| yamllint_version: 1.31.0 | |
| # https://github.com/23andMe/Yamale/releases | |
| # default is 3.0.4, just bump to current latest patch version | |
| yamale_version: 3.0.8 | |
| - name: Run chart-testing (list-changed) | |
| if: env.WF_EXEC != 'false' | |
| run: | | |
| ct list-changed --target-branch ${{ github.event.repository.default_branch }} | |
| - name: Run chart-testing (lint) | |
| if: env.WF_EXEC != 'false' | |
| run: ct lint --debug --charts ./helm/nessie | |
| - name: Set up & Start Minikube | |
| if: env.WF_EXEC != 'false' | |
| uses: medyagh/[email protected] | |
| with: | |
| # If required, use the matrix strategy against this option to test against multiple Kubernetes versions: | |
| kubernetes-version: stable | |
| cache: false | |
| - name: Show pods | |
| if: env.WF_EXEC != 'false' | |
| run: kubectl get pods -A | |
| - name: Run chart-testing (install) | |
| if: env.WF_EXEC != 'false' | |
| run: ct install --debug --charts ./helm/nessie | |
| site: | |
| name: CI Website | |
| runs-on: ubuntu-22.04 | |
| steps: | |
| - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 | |
| - name: Setup Python | |
| uses: ./.github/actions/dev-tool-python | |
| with: | |
| python-version: '3.8' | |
| - name: Upgrade pip | |
| run: | | |
| # install pip=>20.1 to use "pip cache dir" | |
| python3 -m pip install --upgrade pip | |
| - name: Install dependencies | |
| run: python3 -m pip install -r ./site/requirements.txt | |
| - name: Check code block indentation | |
| run: python3 check_code_indent.py | |
| working-directory: ./site | |
| - name: Generate Static Site | |
| run: mkdocs build | |
| working-directory: ./site | |
| - name: Deploy Static Site to GitHub | |
| if: github.event_name == 'push' && github.ref == 'refs/heads/main' && github.repository == 'projectnessie/nessie' | |
| uses: peaceiris/actions-gh-pages@v3 | |
| with: | |
| external_repository: projectnessie/projectnessie.github.io | |
| publish_branch: main | |
| deploy_key: ${{ secrets.NESSIE_SITE_DEPLOY_KEY }} | |
| publish_dir: ./site/site | |
| cname: projectnessie.org | |
| store-cache: | |
| # Store the Gradle cache to GH cache as soon as all relevant Nessie/Gradle jobs have finished. | |
| name: CI Store Cache | |
| runs-on: ubuntu-22.04 | |
| if: github.event_name == 'push' && github.ref == 'refs/heads/main' | |
| needs: | |
| # Only include jobs that use Nessie's Gradle cache, especially excluding NesQuEIT, which | |
| # is a "very special" citizen and also not run for "main" CI, which does | |
| # not add anything to the Gradle cache that's not already produced by other jobs. | |
| - code-checks | |
| - test | |
| - test-quarkus | |
| - int-test | |
| - int-test-stores | |
| - int-test-integrations | |
| - int-test-quarkus | |
| strategy: | |
| max-parallel: 2 | |
| matrix: | |
| java-version: ['11'] # Ideally also '17', but GH concurrent job limit ... :( | |
| steps: | |
| - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 | |
| - name: Setup runner | |
| uses: ./.github/actions/setup-runner | |
| - name: Setup Java, Gradle | |
| uses: ./.github/actions/dev-tool-java | |
| with: | |
| java-version: ${{ matrix.java-version }} | |
| - name: Collect partial Gradle build caches | |
| uses: ./.github/actions/ci-incr-build-cache-prepare | |
| with: | |
| java-version: ${{ matrix.java-version }} | |
| cache-read-only: false | |
| - name: Trigger Gradle home cleanup | |
| run: ./gradlew --no-daemon :showVersion | |
| # Note: the "Post Gradle invocation" archives the updated build cache. |