Fix file system on spark push job #6
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # GitHub Actions workflow for running compatibility tests: Avro, Alpini unit, Alpini functional tests, and Pulsar Venice integration tests | |
| name: TestsDeCompatibilite | |
| on: [push, pull_request, workflow_dispatch] | |
| jobs: | |
| AvroCompatibilityTests: | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| jdk: [17] | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 60 | |
| steps: | |
| - uses: actions/checkout@v3 | |
| with: | |
| fetch-depth: 0 | |
| - name: Set up JDK | |
| uses: actions/setup-java@v3 | |
| with: | |
| java-version: ${{ matrix.jdk }} | |
| distribution: 'temurin' | |
| cache: 'gradle' | |
| - shell: bash | |
| run: | | |
| git remote set-head origin --auto | |
| git remote add upstream https://github.com/linkedin/venice | |
| git fetch upstream | |
| - name: Build with Gradle | |
| uses: gradle/gradle-build-action@v2 | |
| with: | |
| arguments: "-DmaxParallelForks=2 --parallel :internal:venice-avro-compatibility-test:test --continue" | |
| - name: Package Build Artifacts | |
| if: success() || failure() | |
| shell: bash | |
| run: | | |
| mkdir ${{ github.job }}-artifacts | |
| find . -path "**/build/reports/*" -or -path "**/build/test-results/*" > artifacts.list | |
| rsync -R --files-from=artifacts.list . ${{ github.job }}-artifacts | |
| tar -zcvf ${{ github.job }}-jdk${{ matrix.jdk }}-logs.tar.gz ${{ github.job }}-artifacts | |
| - name: Upload Build Artifacts | |
| if: success() || failure() | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: ${{ github.job }} | |
| path: ${{ github.job }}-jdk${{ matrix.jdk }}-logs.tar.gz | |
| retention-days: 30 | |
| AlpiniUnitTests: | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| jdk: [8, 11, 17] | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 120 | |
| outputs: | |
| alpini_touched: ${{ steps.check_alpini_files_changed.outputs.alpini }} | |
| steps: | |
| - uses: actions/checkout@v3 | |
| with: | |
| # Checkout as many commits as needed for the diff | |
| fetch-depth: 2 | |
| - name: Check if files have changed | |
| uses: dorny/paths-filter@v2 | |
| id: check_alpini_files_changed | |
| with: | |
| filters: | | |
| alpini: | |
| - 'internal/alpini/**' | |
| - uses: actions/checkout@v3 | |
| if: steps.check_alpini_files_changed.outputs.alpini == 'true' | |
| with: | |
| fetch-depth: 0 | |
| - name: Set up JDK | |
| uses: actions/setup-java@v3 | |
| if: steps.check_alpini_files_changed.outputs.alpini == 'true' | |
| with: | |
| java-version: ${{ matrix.jdk }} | |
| distribution: 'temurin' | |
| cache: 'gradle' | |
| # - name: Allow Deprecated TLS versions for Alpini tests | |
| # run: | | |
| # echo "java.security file before modifications: " | |
| # cat "$JAVA_HOME/conf/security/java.security" | |
| # # This is possibly flaky but | |
| # sed -i 's/TLSv1, //g' "$JAVA_HOME/conf/security/java.security" # Allow TLSv1 | |
| # sed -i 's/TLSv1.1, //g' "$JAVA_HOME/conf/security/java.security" # Allow TLSv1.1 | |
| # echo "java.security file after modifications: " | |
| # cat "$JAVA_HOME/conf/security/java.security" | |
| - shell: bash | |
| if: steps.check_alpini_files_changed.outputs.alpini == 'true' | |
| run: | | |
| git remote set-head origin --auto | |
| git remote add upstream https://github.com/linkedin/venice | |
| git fetch upstream | |
| - name: Build with Gradle | |
| if: steps.check_alpini_files_changed.outputs.alpini == 'true' | |
| uses: gradle/gradle-build-action@v2 | |
| with: | |
| arguments: "--continue --no-daemon -DmaxParallelForks=1 alpiniUnitTest" | |
| - name: Package Build Artifacts | |
| if: steps.check_alpini_files_changed.outputs.alpini == 'true' && (success() || failure()) | |
| shell: bash | |
| run: | | |
| mkdir ${{ github.job }}-artifacts | |
| find . -path "**/build/reports/*" -or -path "**/build/test-results/*" > artifacts.list | |
| rsync -R --files-from=artifacts.list . ${{ github.job }}-artifacts | |
| tar -zcvf ${{ github.job }}-jdk${{ matrix.jdk }}-logs.tar.gz ${{ github.job }}-artifacts | |
| - name: Upload Build Artifacts | |
| if: steps.check_alpini_files_changed.outputs.alpini == 'true' && (success() || failure()) | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: ${{ github.job }} | |
| path: ${{ github.job }}-jdk${{ matrix.jdk }}-logs.tar.gz | |
| retention-days: 30 | |
| AlpiniFunctionalTests: | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| jdk: [17] | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 120 | |
| outputs: | |
| alpini_touched: ${{ steps.check_alpini_files_changed.outputs.alpini }} | |
| steps: | |
| - uses: actions/checkout@v3 | |
| with: | |
| # Checkout as many commits as needed for the diff | |
| fetch-depth: 2 | |
| - name: Check if files have changed | |
| uses: dorny/paths-filter@v2 | |
| id: check_alpini_files_changed | |
| with: | |
| filters: | | |
| alpini: | |
| - 'internal/alpini/**' | |
| - uses: actions/checkout@v3 | |
| if: steps.check_alpini_files_changed.outputs.alpini == 'true' | |
| with: | |
| fetch-depth: 0 | |
| - name: Set up JDK | |
| uses: actions/setup-java@v3 | |
| if: steps.check_alpini_files_changed.outputs.alpini == 'true' | |
| with: | |
| java-version: ${{ matrix.jdk }} | |
| distribution: 'temurin' | |
| cache: 'gradle' | |
| # - name: Allow Deprecated TLS versions for Alpini tests | |
| # run: | | |
| # echo "java.security file before modifications: " | |
| # cat "$JAVA_HOME/conf/security/java.security" | |
| # # This is possibly flaky but | |
| # sed -i 's/TLSv1, //g' "$JAVA_HOME/conf/security/java.security" # Allow TLSv1 | |
| # sed -i 's/TLSv1.1, //g' "$JAVA_HOME/conf/security/java.security" # Allow TLSv1.1 | |
| # echo "java.security file after modifications: " | |
| # cat "$JAVA_HOME/conf/security/java.security" | |
| - shell: bash | |
| if: steps.check_alpini_files_changed.outputs.alpini == 'true' | |
| run: | | |
| git remote set-head origin --auto | |
| git remote add upstream https://github.com/linkedin/venice | |
| git fetch upstream | |
| - name: Build with Gradle | |
| if: steps.check_alpini_files_changed.outputs.alpini == 'true' | |
| uses: gradle/gradle-build-action@v2 | |
| with: | |
| arguments: "--continue --no-daemon -DmaxParallelForks=1 alpiniFunctionalTest" | |
| - name: Package Build Artifacts | |
| if: steps.check_alpini_files_changed.outputs.alpini == 'true' && (success() || failure()) | |
| shell: bash | |
| run: | | |
| mkdir ${{ github.job }}-artifacts | |
| find . -path "**/build/reports/*" -or -path "**/build/test-results/*" > artifacts.list | |
| rsync -R --files-from=artifacts.list . ${{ github.job }}-artifacts | |
| tar -zcvf ${{ github.job }}-jdk${{ matrix.jdk }}-logs.tar.gz ${{ github.job }}-artifacts | |
| - name: Upload Build Artifacts | |
| if: steps.check_alpini_files_changed.outputs.alpini == 'true' && (success() || failure()) | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: ${{ github.job }} | |
| path: ${{ github.job }}-jdk${{ matrix.jdk }}-logs.tar.gz | |
| retention-days: 30 | |
| PulsarVeniceIntegrationTests: | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| jdk: [17] | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 120 | |
| steps: | |
| - uses: actions/checkout@v3 | |
| with: | |
| fetch-depth: 0 | |
| - name: Set up JDK | |
| uses: actions/setup-java@v3 | |
| with: | |
| java-version: ${{ matrix.jdk }} | |
| distribution: 'temurin' | |
| cache: 'gradle' | |
| - shell: bash | |
| run: | | |
| git remote set-head origin --auto | |
| git remote add upstream https://github.com/linkedin/venice | |
| git fetch upstream | |
| - name: Setup Gradle | |
| uses: gradle/gradle-build-action@v2 | |
| - name: Build with gradle | |
| run: ./gradlew assemble --continue --no-daemon -DforkEvery=1 -DmaxParallelForks=1 | |
| - name: Build docker images for Venice (latest-dev tag) | |
| shell: bash | |
| run: | | |
| cd ${{ github.workspace }}/docker | |
| ./build-venice-docker-images.sh | |
| cd ${{ github.workspace }} | |
| - name: Build docker images for Pulsar test (latest-dev tag) | |
| shell: bash | |
| run: | | |
| cd ${{ github.workspace }}/tests/docker-images/pulsar-sink | |
| docker build --tag=pulsar/venice-test:latest-dev ${{ github.workspace }} -f ./Dockerfile | |
| cd ${{ github.workspace }} | |
| - name: Run the test | |
| shell: bash | |
| run: | | |
| ./gradlew :tests:venice-pulsar-test:pulsarIntegrationTest -i | |
| - name: Package Build Artifacts | |
| if: success() || failure() | |
| shell: bash | |
| run: | | |
| mkdir ${{ github.job }}-artifacts | |
| find . -path "**/build/reports/*" -or -path "**/build/test-results/*" > artifacts.list | |
| rsync -R --files-from=artifacts.list . ${{ github.job }}-artifacts | |
| tar -zcvf ${{ github.job }}-artifacts.tar.gz ${{ github.job }}-artifacts | |
| - name: Upload Build Artifacts | |
| if: success() || failure() | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: ${{ github.job }} | |
| path: ${{ github.job }}-artifacts.tar.gz | |
| retention-days: 30 | |
| CompatibilityTestsCompletionCheck: | |
| strategy: | |
| fail-fast: false | |
| runs-on: ubuntu-latest | |
| needs: [AvroCompatibilityTests, AlpiniUnitTests, AlpiniFunctionalTests, PulsarVeniceIntegrationTests] | |
| timeout-minutes: 120 | |
| steps: | |
| - name: AllIsWell | |
| shell: bash | |
| run: | | |
| echo "LGTM!" |