Build and Unit Test #1489
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # Copyright © 2022 Cask Data, Inc. | |
| # Licensed under the Apache License, Version 2.0 (the "License"); you may not | |
| # use this file except in compliance with the License. You may obtain a copy of | |
| # the License at | |
| # http://www.apache.org/licenses/LICENSE-2.0 | |
| # Unless required by applicable law or agreed to in writing, software | |
| # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT | |
| # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the | |
| # License for the specific language governing permissions and limitations under | |
| # the License. | |
| name: Build and Unit Test | |
| on: | |
| schedule: | |
| - cron: '0 8 * * *' | |
| workflow_dispatch: | |
| # workaround to run manual trigger for a particular branch | |
| inputs: | |
| branch: | |
| description: "Branch name on which workflow will be triggered" | |
| required: true | |
| default: "develop" | |
| skip_tests: | |
| description: "Skip running tests for this build" | |
| required: true | |
| type: boolean | |
| default: false | |
| run_tests_with_retries: | |
| description: "Run tests with retries, it will be ignored if tests are skipped." | |
| required: true | |
| type: boolean | |
| default: false | |
| keep_existing_release: | |
| description: "Do not overwrite the GitHub release for versions without snapshot" | |
| required: true | |
| type: boolean | |
| default: true | |
| env: | |
| TAG_NAME: latest | |
| PRE_RELEASE: true | |
| REPLACE_ARTIFACTS: true | |
| REMOVE_ARTIFACTS: true | |
| MAX_ATTEMPTS: 3 | |
| jobs: | |
| set-branch-matrix: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: set-matrix | |
| run: | | |
| if [ ${{ github.event_name }} != "workflow_dispatch" ]; | |
| then | |
| echo 'BRANCH_MATRIX={"include":[{"branch":"develop"}, {"branch":"release/6.9"}, {"branch":"release/6.10"}]}' >> $GITHUB_ENV | |
| else | |
| echo 'BRANCH_MATRIX={"include":[{"branch":"${{ github.event.inputs.branch }}"}]}' >> $GITHUB_ENV | |
| if [ "${{ github.event.inputs.run_tests_with_retries }}" == "false" ] | |
| then | |
| echo 'MAX_ATTEMPTS=1' >> $GITHUB_ENV | |
| fi | |
| fi | |
| outputs: | |
| matrix: ${{ env.BRANCH_MATRIX }} | |
| retries: ${{ env. MAX_ATTEMPTS }} | |
| build: | |
| needs: set-branch-matrix | |
| runs-on: cdapio-hub-k8-runner | |
| strategy: | |
| fail-fast: false | |
| matrix: ${{ fromJson(needs.set-branch-matrix.outputs.matrix) }} | |
| steps: | |
| - name: Get Start Time | |
| run: echo "STARTED_AT=$(date +'%Y-%m-%dT%H:%M:%S')" >> $GITHUB_ENV | |
| - name: Get Secrets from GCP Secret Manager | |
| id: 'secrets' | |
| uses: 'google-github-actions/get-secretmanager-secrets@v0' | |
| if: ${{ matrix.branch == 'develop' || startsWith(matrix.branch, 'release/') }} | |
| with: | |
| secrets: |- | |
| CDAP_OSSRH_USERNAME:cdapio-github-builds/CDAP_OSSRH_USERNAME | |
| CDAP_OSSRH_PASSWORD:cdapio-github-builds/CDAP_OSSRH_PASSWORD | |
| CDAP_GPG_PASSPHRASE:cdapio-github-builds/CDAP_GPG_PASSPHRASE | |
| CDAP_GPG_PRIVATE_KEY:cdapio-github-builds/CDAP_GPG_PRIVATE_KEY | |
| - name: Recursively Checkout Repository | |
| uses: actions/checkout@v3 | |
| with: | |
| fetch-depth: 0 | |
| submodules: recursive | |
| path: cdap-build | |
| ref: ${{ matrix.branch }} | |
| - name: Update Submodules | |
| working-directory: cdap-build | |
| run: | | |
| git submodule update --init --recursive --remote | |
| - name: Cache | |
| uses: actions/cache@v3 | |
| with: | |
| path: ~/.m2/repository | |
| key: ${{ runner.os }}-maven-${{ github.workflow }}-${{ hashFiles('**/pom.xml') }} | |
| restore-keys: | | |
| ${{ runner.os }}-maven-${{ github.workflow }} | |
| - name: Set up Artifact Name # Removes invalid artifact name characters: ",:,<,>,|,*,?,\,/. | |
| if: "${{ github.event.inputs.skip_tests != 'true' }}" | |
| run: | | |
| name=$(echo -n "${{ matrix.branch }}" | sed -e 's/[ \t:\/\\"<>|*?]/-/g' -e 's/--*/-/g') | |
| echo "ARTIFACT_NAME=$name" >> $GITHUB_ENV | |
| - name: Run Tests | |
| if: "${{ github.event.inputs.skip_tests != 'true' }}" | |
| # Pinned version 2.8.2 | |
| uses: nick-fields/retry@3e91a01664abd3c5cd539100d10d33b9c5b68482 | |
| with: | |
| timeout_minutes: 120 | |
| max_attempts: ${{ needs.set-branch-matrix.outputs.retries }} | |
| retry_on: error | |
| on_retry_command: echo "Tests failed in this attempt, retrying ..." | |
| command: | | |
| cd cdap-build | |
| MAVEN_OPTS="-Xmx16G -XX:+UseG1GC -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=/cdap-build/oom.bin" mvn test -fae -T2 -U -V -am -amd -P templates,unit-tests -Dmaven.wagon.http.retryHandler.count=5 -Dmaven.wagon.httpconnectionManager.ttlSeconds=30 | |
| - name: Archive build artifacts | |
| uses: actions/upload-artifact@v3 | |
| if: "${{ github.event.inputs.skip_tests != 'true' || failure() }}" | |
| with: | |
| name: Build debug files - ${{ env.ARTIFACT_NAME }} | |
| path: | | |
| **/target/rat.txt | |
| **/target/surefire-reports/* | |
| /cdap-build/oom.bin | |
| - name: Surefire Report | |
| # Pinned 3.5.2 version | |
| uses: mikepenz/action-junit-report@16a9560bd02f11e7e3bf6b3e2ef6bba6c9d07c32 | |
| if: ${{ always() }} | |
| with: | |
| report_paths: '**/target/surefire-reports/TEST-*.xml' | |
| github_token: ${{ secrets.GITHUB_TOKEN }} | |
| detailed_summary: true | |
| commit: ${{ github.sha }} | |
| check_name: Test Report - ${{ env.ARTIFACT_NAME }} | |
| - name: Build Standalone | |
| # Pinned version 2.8.2 | |
| uses: nick-fields/retry@3e91a01664abd3c5cd539100d10d33b9c5b68482 | |
| with: | |
| timeout_minutes: 60 | |
| max_attempts: 3 | |
| retry_on: error | |
| on_retry_command: echo "Build Standalone failed in this attempt, retrying ..." | |
| command: | | |
| cd cdap-build | |
| MAVEN_OPTS="-Xmx12G" mvn -e -T2 clean package -Dgpg.skip -DskipTests -Ddocker.skip=true -nsu -am -amd -P templates,dist,release -Dadditional.artifacts.dir=$(pwd)/app-artifacts -Dsecurity.extensions.dir=$(pwd)/security-extensions -Dmaven.wagon.http.retryHandler.count=5 -Dmaven.wagon.httpconnectionManager.ttlSeconds=30 | |
| - name: Find Build Version | |
| working-directory: cdap-build/cdap | |
| run: | | |
| export VERSION=$(ls cdap-standalone/target/cdap-sandbox*zip | cut --delimiter=- --fields="-3" --complement | rev | cut --delimiter="." --fields=1 --complement | rev) | |
| echo "CDAP_VERSION=${VERSION}" | |
| echo "CDAP_VERSION=${VERSION}" >> $GITHUB_ENV | |
| if [[ ($VERSION != *-SNAPSHOT) && ("${{ github.event.inputs.keep_existing_release }}" != "false") ]]; | |
| then | |
| echo "PRE_RELEASE=false" >> $GITHUB_ENV | |
| echo "REPLACE_ARTIFACTS=false" >> $GITHUB_ENV | |
| echo "REMOVE_ARTIFACTS=false" >> $GITHUB_ENV | |
| echo "Release will not be overwritten if exists." | |
| else | |
| echo "Release will be overwritten if exists." | |
| fi | |
| - name: Upload CDAP Standalone | |
| if: ${{ matrix.branch == 'develop' || startsWith(matrix.branch, 'release/') }} | |
| uses: actions/upload-artifact@v3 # https://github.com/actions/upload-artifact#zipped-artifact-downloads | |
| with: | |
| name: cdap-sandbox-${{env.CDAP_VERSION}}.zip | |
| path: cdap-build/cdap/cdap-standalone/target/cdap-sandbox-${{env.CDAP_VERSION}}.zip | |
| - name: Set up GPG conf | |
| if: ${{ matrix.branch == 'develop' || startsWith(matrix.branch, 'release/') }} | |
| run: | | |
| echo "pinentry-mode loopback" >> ~/.gnupg/gpg.conf | |
| echo "allow-loopback-pinentry" >> ~/.gnupg/gpg-agent.conf | |
| - name: Import GPG key | |
| if: ${{ matrix.branch == 'develop' || startsWith(matrix.branch, 'release/') }} | |
| run: | | |
| echo "$GPG_PRIVATE_KEY" > private.key | |
| gpg --import --batch private.key | |
| env: | |
| GPG_PRIVATE_KEY: ${{ steps.secrets.outputs.CDAP_GPG_PRIVATE_KEY }} | |
| - name: Maven Deploy | |
| run: | | |
| cd cdap-build | |
| if [[ (${{ matrix.branch }} == "develop") || (${{ matrix.branch }} == release/*) ]]; | |
| then | |
| retry_count=0 | |
| failed_module="" | |
| while [ $retry_count -lt 3 ]; do | |
| if [ -n "$failed_module" ]; | |
| then | |
| build_output=$(mvn deploy -B -V -DskipTests -DskipLocalStaging=true -Ddocker.skip=true -P templates,dist,release,rpm-prepare,rpm,deb-prepare,deb,tgz,unit-tests -Dadditional.artifacts.dir=$(pwd)/app-artifacts -Dsecurity.extensions.dir=$(pwd)/security-extensions -Dmaven.wagon.http.retryHandler.count=5 -Dmaven.wagon.httpconnectionManager.ttlSeconds=30 -Dmaven.wagon.http.retryHandler.requestSentEnabled=true -Dgpg.passphrase=$CDAP_GPG_PASSPHRASE -rf :"$failed_module" 2>&1 || true) | |
| else | |
| build_output=$(mvn deploy -B -V -DskipTests -DskipLocalStaging=true -Ddocker.skip=true -P templates,dist,release,rpm-prepare,rpm,deb-prepare,deb,tgz,unit-tests -Dadditional.artifacts.dir=$(pwd)/app-artifacts -Dsecurity.extensions.dir=$(pwd)/security-extensions -Dmaven.wagon.http.retryHandler.count=5 -Dmaven.wagon.httpconnectionManager.ttlSeconds=30 -Dmaven.wagon.http.retryHandler.requestSentEnabled=true -Dgpg.passphrase=$CDAP_GPG_PASSPHRASE 2>&1 || true) | |
| fi | |
| echo "$build_output" | |
| if [ $(echo "$build_output" | grep -c "BUILD FAILURE") -gt 0 ]; | |
| then | |
| echo "[WARNING] Deployment failed, retrying..." | |
| failed_module=$(echo "$build_output" | grep -e 'mvn <args> -rf' | sed -n 's/.*mvn <args> -rf ://p') | |
| echo "[INFO] FAILED MODULE = $failed_module" | |
| retry_count=$((retry_count + 1)) | |
| else | |
| echo "[INFO] Deployment successful" | |
| break | |
| fi | |
| done | |
| if [ $retry_count -ge 3 ]; | |
| then | |
| echo "[ERROR] Max retries reached..., deployment failed" | |
| exit 1 | |
| fi | |
| else | |
| mvn verify -B -V -T2 -DskipTests -Dgpg.skip -Ddocker.skip=true -P templates,dist,release,rpm-prepare,rpm,deb-prepare,deb,tgz,unit-tests -Dadditional.artifacts.dir=$(pwd)/app-artifacts -Dsecurity.extensions.dir=$(pwd)/security-extensions -Dmaven.wagon.http.retryHandler.count=5 -Dmaven.wagon.httpconnectionManager.ttlSeconds=30 -Dmaven.wagon.http.retryHandler.requestSentEnabled=true | |
| fi | |
| env: | |
| CDAP_OSSRH_USERNAME: ${{ steps.secrets.outputs.CDAP_OSSRH_USERNAME }} | |
| CDAP_OSSRH_PASSWORD: ${{ steps.secrets.outputs.CDAP_OSSRH_PASSWORD }} | |
| CDAP_GPG_PASSPHRASE: ${{ steps.secrets.outputs.CDAP_GPG_PASSPHRASE }} | |
| MAVEN_OPTS: "-Xmx12G" | |
| - name: Build DEB Bundle | |
| working-directory: cdap-build/cdap | |
| run: | | |
| mkdir -p cdap-distributions/target/deb-bundle-tmp | |
| cd cdap-distributions/target/deb-bundle-tmp | |
| cp ../../../*/target/*.deb . | |
| tar zcf ../cdap-distributed-deb-bundle-${{env.CDAP_VERSION}}.tgz *.deb | |
| - name: Set Up Tag | |
| working-directory: cdap-build | |
| run: | | |
| if [[ ${{ matrix.branch }} == release/* ]]; | |
| then | |
| echo "TAG_NAME=v$CDAP_VERSION" >> $GITHUB_ENV | |
| echo "TAG_NAME=v$CDAP_VERSION" | |
| git tag -f v$CDAP_VERSION | |
| git push -f origin refs/tags/v$CDAP_VERSION:refs/tags/v$CDAP_VERSION | |
| elif [[ ${{ matrix.branch }} != "develop" ]]; | |
| then | |
| export TAG=$(git check-ref-format --normalize "tags/${{ matrix.branch }}/tag" | cut -c6- | rev | cut -c5- | rev) | |
| echo "TAG_NAME=$TAG" >> $GITHUB_ENV | |
| echo "TAG_NAME=$TAG" | |
| git tag -f $TAG | |
| git push -f origin refs/tags/$TAG:refs/tags/$TAG | |
| else | |
| git tag -f ${{ env.TAG_NAME }} | |
| git push -f origin refs/tags/${{ env.TAG_NAME }}:refs/tags/${{ env.TAG_NAME }} | |
| fi | |
| - name: Upload CDAP Standalone and CDAP DEB Bundle | |
| # Pinned 1.11.1 version | |
| uses: ncipollo/release-action@4c75f0f2e4ae5f3c807cf0904605408e319dcaac | |
| with: | |
| allowUpdates: true | |
| omitBodyDuringUpdate: true | |
| omitNameDuringUpdate: true | |
| omitPrereleaseDuringUpdate: true | |
| prerelease: ${{ env.PRE_RELEASE }} | |
| removeArtifacts: ${{ env.REMOVE_ARTIFACTS }} | |
| replacesArtifacts: ${{ env.REPLACE_ARTIFACTS }} | |
| token: ${{ secrets.GITHUB_TOKEN }} | |
| tag: ${{ env.TAG_NAME }} | |
| body: Cask Data Appplication Platform - Release ${{ env.CDAP_VERSION }} | |
| artifacts: | | |
| cdap-build/cdap/cdap-standalone/target/cdap-sandbox-${{env.CDAP_VERSION}}.zip,cdap-build/cdap/cdap-distributions/target/cdap-distributed-deb-bundle-${{env.CDAP_VERSION}}.tgz | |
| - name: Alert team if build fails | |
| if: ${{ (matrix.branch == 'develop' || startsWith(matrix.branch, 'release/')) && failure() }} | |
| run: | | |
| gcloud logging write cdapio-github-builds-logs '{ "message": "Build failure", "url": "${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"}' --payload-type=json --severity=ERROR | |
| - name: Get End Time | |
| if: ${{ always() }} | |
| run: echo "ENDED_AT=$(date +'%Y-%m-%dT%H:%M:%S')" >> $GITHUB_ENV | |
| - name: Report Result | |
| if: ${{ (matrix.branch == 'develop' || startsWith(matrix.branch, 'release/')) && github.event.inputs.skip_tests != 'true' && always() }} | |
| run: | | |
| echo '{ "repository": "${{ github.repository }}", "build_name": "${{ github.workflow }}", "branch": "${{ matrix.branch }}", "conclusion": "${{ job.status }}", "started_at": "'$STARTED_AT'", "ended_at": "'$ENDED_AT'", "url": "https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}", "run_id": "${{ github.run_id }}", "run_attempt": "${{ github.run_attempt }}" }' | bq insert ${{ vars.BUILD_HISTORY_TABLE }} |