datahub-actions-docker.yml

name: datahub-actions docker
on:
  push:
    branches:
      - main
    paths-ignore:
      - 'build/**'
      - '**.md'
  pull_request:
    branches:
      - main
    paths:
      - 'docker/**'
      - '.github/workflows/datahub-actions-docker.yml'
    paths_ignore:
      - 'build/**'
      - '**.md'
  release:
    types: [published, edited]
  workflow_dispatch:

jobs:
  setup:
    runs-on: ubuntu-latest
    outputs:
      tag: ${{ steps.tag.outputs.tag }}
      publish: ${{ steps.publish.outputs.publish }}
      unique_tag: ${{ steps.tag.outputs.unique_tag }}
    steps:
      - name: Checkout
        uses: actions/checkout@v3
      - name: Compute Tag
        id: tag
        run: |
          echo "GITHUB_REF: $GITHUB_REF"
          SHORT_SHA=$(git rev-parse --short "$GITHUB_SHA")
          TAG=$(echo ${GITHUB_REF} | sed -e "s,refs/heads/main,head\,${SHORT_SHA},g" -e 's,refs/tags/,,g' -e 's,refs/pull/\([0-9]*\).*,pr\1,g')
          UNIQUE_TAG=$(echo ${GITHUB_REF} | sed -e "s,refs/heads/main,${SHORT_SHA},g" -e 's,refs/tags/,,g' -e 's,refs/pull/\([0-9]*\).*,pr\1,g')
          echo "tag=$TAG" >> "$GITHUB_OUTPUT"
          echo "unique_tag=$UNIQUE_TAG" >> "$GITHUB_OUTPUT"
      - name: Check whether publishing enabled
        id: publish
        env:
          ENABLE_PUBLISH: ${{ secrets.ACRYL_DOCKER_PASSWORD }}
        run: |
          echo "Enable publish: ${{ env.ENABLE_PUBLISH != '' }}"
          echo "publish=${{ env.ENABLE_PUBLISH != '' }}" >> "$GITHUB_OUTPUT"
  regular_image:
    name: Build & Push Image to DockerHub
    runs-on: ubuntu-latest
    needs: setup
    steps:
      - name: Check out the repo
        uses: actions/checkout@v3
      - name: Docker meta
        id: docker_meta
        uses: docker/metadata-action@v5
        with:
          images: |
            acryldata/datahub-actions
          tags: |
            type=raw,value=${{ needs.setup.outputs.tag }}
      - name: Set up QEMU
        uses: docker/setup-qemu-action@v3
        if: ${{ needs.setup.outputs.publish == 'true' }}
      - name: Set up Docker Buildx
        uses: docker/setup-buildx-action@v3
      - name: Login to DockerHub
        if: ${{ needs.setup.outputs.publish == 'true' }}
        uses: docker/login-action@v2
        with:
          username: ${{ secrets.ACRYL_DOCKER_USERNAME }}
          password: ${{ secrets.ACRYL_DOCKER_PASSWORD }}
      - name: Build & Push Image
        uses: docker/build-push-action@v6
        with:
          file: ./docker/datahub-actions/Dockerfile
          platforms: ${{ needs.setup.outputs.publish == 'true' && 'linux/amd64,linux/arm64' || 'linux/amd64' }}
          tags: ${{ steps.docker_meta.outputs.tags }}
          push: ${{ needs.setup.outputs.publish == 'true' }}
          cache-from: type=registry,ref=${{ steps.docker_meta.outputs.tags }}
          cache-to: type=inline
          target: final
          build-args: 'GEM_FURY_TOKEN=${{ secrets.GEMFURY_PULL_TOKEN }}'
  slim_image:
    name: Build & Push Image to DockerHub (slim)
    runs-on: ubuntu-latest
    needs: setup
    steps:
      - name: Check out the repo (slim)
        uses: actions/checkout@v3
      - name: Docker meta (slim)
        id: docker_meta_slim
        uses: docker/metadata-action@v5
        with:
          images: |
            acryldata/datahub-actions-slim
          tags: |
            type=raw,value=${{ needs.setup.outputs.tag }}
      - name: Set up QEMU (slim)
        if: ${{ needs.setup.outputs.publish == 'true' }}
        uses: docker/setup-qemu-action@v2
      - name: Set up Docker Buildx (slim)
        uses: docker/setup-buildx-action@v2
      - name: Login to DockerHub (slim)
        uses: docker/login-action@v2
        if: ${{ needs.setup.outputs.publish == 'true' }}
        with:
          username: ${{ secrets.ACRYL_DOCKER_USERNAME }}
          password: ${{ secrets.ACRYL_DOCKER_PASSWORD }}
      - name: Build & Push Image (slim)
        uses: docker/build-push-action@v6
        with:
          file: ./docker/datahub-actions/Dockerfile
          platforms: ${{ needs.setup.outputs.publish == 'true' && 'linux/amd64,linux/arm64' || 'linux/amd64' }}
          tags: ${{ steps.docker_meta_slim.outputs.tags }}
          push: ${{ needs.setup.outputs.publish == 'true' }}
          target: final
          load: ${{ needs.setup.outputs.publish != 'true' }}
          build-args: |
            "APP_ENV=prod-slim"
            "GEM_FURY_TOKEN=${{ secrets.GEMFURY_PULL_TOKEN }}"
      - name: Save Docker image
        if: needs.setup.outputs.publish != 'true'
        run: docker save ${{ steps.docker_meta_slim.outputs.tags }} > image.tar
      - name: Upload artifact
        if: needs.setup.outputs.publish != 'true'
        uses: actions/upload-artifact@v3
        with:
          name: docker-image
          path: image.tar
  # image_scan:
  #   permissions:
  #     contents: read # for actions/checkout to fetch code
  #     security-events: write # for github/codeql-action/upload-sarif to upload SARIF results
  #     actions: read # only required for a private repository by github/codeql-action/upload-sarif to get the Action run status
  #   name: "[Monitoring] Scan action images for vulnerabilities"
  #   runs-on: ubuntu-latest
  #   needs: [setup, slim_image]
  #   steps:
  #     - name: Checkout # adding checkout step just to make trivy upload happy
  #       uses: actions/checkout@v3
  #     - name: Download image
  #       uses: ishworkh/docker-image-artifact-download@v1
  #       if: ${{ needs.setup.outputs.publish != 'true' }}
  #       with:
  #         image: acryldata/datahub-actions:${{ needs.setup.outputs.unique_tag }}
  #     - name: Run Trivy vulnerability scanner
  #       uses: aquasecurity/trivy-action@master
  #       env:
  #         TRIVY_OFFLINE_SCAN: true
  #       with:
  #         image-ref: acryldata/datahub-actions:${{ needs.setup.outputs.unique_tag }}
  #         format: 'template'
  #         template: '@/contrib/sarif.tpl'
  #         output: 'trivy-results.sarif'
  #         severity: 'CRITICAL,HIGH'
  #         ignore-unfixed: true
  #         vuln-type: "os,library"
  #     - name: Upload Trivy scan results to GitHub Security tab
  #       uses: github/codeql-action/upload-sarif@v2
  #       with:
  #         sarif_file: 'trivy-results.sarif'
  image_scan_slim:
    permissions:
      contents: read # for actions/checkout to fetch code
      security-events: write # for github/codeql-action/upload-sarif to upload SARIF results
      actions: read # only required for a private repository by github/codeql-action/upload-sarif to get the Action run status
    name: '[Monitoring] Scan slim action images for vulnerabilities'
    runs-on: ubuntu-latest
    needs: [setup, slim_image]
    steps:
      - name: Checkout # adding checkout step just to make trivy upload happy
        uses: actions/checkout@v4
      - name: Download artifact (if not publishing)
        if: needs.setup.outputs.publish != 'true'
        uses: actions/download-artifact@v3
        with:
          name: docker-image
      - name: Load Docker image (if not publishing)
        if: needs.setup.outputs.publish != 'true'
        run: docker load < image.tar
      - name: Download image (if publishing)
        if: needs.setup.outputs.publish == 'true'
        uses: ishworkh/docker-image-artifact-download@v1
        with:
          image: acryldata/datahub-actions-slim:${{ needs.setup.outputs.unique_tag }}
      - name: Run Trivy vulnerability scanner (slim)
        uses: aquasecurity/trivy-action@master
        env:
          TRIVY_OFFLINE_SCAN: true
        with:
          image-ref: acryldata/datahub-actions-slim:${{ needs.setup.outputs.unique_tag }}
          format: 'template'
          template: '@/contrib/sarif.tpl'
          output: 'trivy-results.sarif'
          severity: 'CRITICAL,HIGH'
          ignore-unfixed: true
          vuln-type: 'os,library'
      - name: Upload Trivy scan results to GitHub Security tab (slim)
        uses: github/codeql-action/upload-sarif@v2
        with:
          sarif_file: 'trivy-results.sarif'
  smoke_test:
    name: Run Smoke Tests
    runs-on: ubuntu-latest
    needs: [setup, slim_image]
    steps:
      - name: Free up disk space
        run: |
          sudo apt-get remove 'dotnet-*' azure-cli || true
          sudo rm -rf /usr/local/lib/android/ || true
          sudo docker image prune -a -f || true
      - name: Disk Check
        run: df -h . && docker images
      - name: Checkout the repo
        uses: actions/checkout@v4
      - name: Set up JDK 17
        uses: actions/setup-java@v3
        with:
          distribution: 'zulu'
          java-version: 17
      - uses: gradle/actions/setup-gradle@v3
      - uses: actions/setup-python@v4
        with:
          python-version: '3.10'
          cache: 'pip'
      - name: Download artifact (if not publishing)
        if: needs.setup.outputs.publish != 'true'
        uses: actions/download-artifact@v3
        with:
          name: docker-image
      - name: Load Docker image (if not publishing)
        if: needs.setup.outputs.publish != 'true'
        run: docker load < image.tar
      - name: Download image (if publishing)
        if: needs.setup.outputs.publish == 'true'
        uses: ishworkh/docker-image-artifact-download@v1
        with:
          image: acryldata/datahub-actions-slim:${{ needs.setup.outputs.unique_tag }}
      - name: run quickstart
        env:
          DATAHUB_TELEMETRY_ENABLED: false
          DATAHUB_ACTIONS_IMAGE: acryldata/datahub-actions-slim
          DATAHUB_ACTIONS_VERSION: ${{ needs.setup.outputs.unique_tag }}
          ACTIONS_EXTRA_PACKAGES: 'acryl-datahub-actions[executor]==0.0.13 acryl-datahub-actions==0.0.13 acryl-datahub==0.10.5'
          ACTIONS_CONFIG: 'https://raw.githubusercontent.com/acryldata/datahub-actions/main/docker/config/executor.yaml'
        run: |
          ./smoke-test/run-quickstart.sh
      - name: Disk Check
        run: df -h . && docker images
      - name: Smoke test
        env:
          RUN_QUICKSTART: false
          DATAHUB_ACTIONS_VERSION: ${{ needs.setup.outputs.unique_tag }}
        run: |
          echo "$DATAHUB_ACTIONS_VERSION"
          ./smoke-test/smoke.sh