feat(metadata-jobs): improve consumer logging (#10173) #1
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: spark smoke test | |
| on: | |
| # Allows you to run this workflow manually from the Actions tab | |
| workflow_dispatch: | |
| push: | |
| branches: | |
| - master | |
| paths: | |
| - "metadata_models/**" | |
| - "metadata-integration/java/datahub-client/**" | |
| - "metadata-integration/java/spark-lineage" | |
| - ".github/workflows/spark-smoke-test.yml" | |
| pull_request: | |
| branches: | |
| - "**" | |
| paths: | |
| - "metadata_models/**" | |
| - "metadata-integration/java/datahub-client/**" | |
| - "metadata-integration/java/spark-lineage" | |
| - ".github/workflows/spark-smoke-test.yml" | |
| release: | |
| types: [published] | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} | |
| cancel-in-progress: true | |
| jobs: | |
| spark-smoke-test: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: acryldata/sane-checkout-action@v3 | |
| - name: Set up JDK 17 | |
| uses: actions/setup-java@v3 | |
| with: | |
| distribution: "zulu" | |
| java-version: 17 | |
| - uses: gradle/gradle-build-action@v2 | |
| - uses: actions/setup-python@v4 | |
| with: | |
| python-version: "3.10" | |
| cache: "pip" | |
| - name: Install dependencies | |
| run: ./metadata-ingestion/scripts/install_deps.sh | |
| - name: Disk Check | |
| run: df -h . && docker images | |
| - name: Remove images | |
| run: docker image prune -a -f || true | |
| - name: Disk Check | |
| run: df -h . && docker images | |
| - name: Smoke test | |
| run: | | |
| ./gradlew :metadata-integration:java:spark-lineage:integrationTest \ | |
| -x :datahub-web-react:yarnTest \ | |
| -x :datahub-web-react:yarnLint \ | |
| -x :datahub-web-react:yarnGenerate \ | |
| -x :datahub-web-react:yarnInstall \ | |
| -x :datahub-web-react:yarnBuild \ | |
| -x :datahub-web-react:distZip \ | |
| -x :datahub-web-react:jar | |
| - name: store logs | |
| if: failure() | |
| run: | | |
| docker ps -a | |
| docker logs datahub-gms >& gms-${{ matrix.test_strategy }}.log || true | |
| docker logs datahub-actions >& actions-${{ matrix.test_strategy }}.log || true | |
| docker logs broker >& broker-${{ matrix.test_strategy }}.log || true | |
| docker logs mysql >& mysql-${{ matrix.test_strategy }}.log || true | |
| docker logs elasticsearch >& elasticsearch-${{ matrix.test_strategy }}.log || true | |
| docker logs datahub-frontend-react >& frontend-${{ matrix.test_strategy }}.log || true | |
| - name: Upload logs | |
| uses: actions/upload-artifact@v3 | |
| if: failure() | |
| with: | |
| name: docker logs | |
| path: | | |
| "**/build/container-logs/*.log" | |
| "*.log" | |
| - uses: actions/upload-artifact@v3 | |
| if: always() | |
| with: | |
| name: Test Results (smoke tests) | |
| path: | | |
| **/build/reports/tests/test/** | |
| **/build/test-results/test/** | |
| **/junit.*.xml | |
| !**/binary/** |