datahub-actions-docker.yml

name: datahub-actions docker
on:
  push:
    branches:
      - main
    paths-ignore:
      - "build/**"
      - "**.md"
  pull_request:
    branches:
      - main
    paths:
      - "docker/**"
      - ".github/workflows/datahub-actions-docker.yml"
      - "!build/**"
      - "!**.md"
  release:
    types: [published, edited]
  workflow_dispatch:

jobs:
  setup:
    runs-on: ubuntu-latest
    outputs:
      publish: ${{ steps.publish.outputs.publish }}
      # The tracking tag will be "head", "v1.2.3", or "pr1234".
      # The unique tag will be a short SHA.
      unique_tag: ${{ steps.tag.outputs.unique_tag }}
      tracking_tag: ${{ steps.tag.outputs.tracking_tag }}
    steps:
      - name: Checkout
        uses: actions/checkout@v3
      - name: Compute Tag
        id: tag
        run: |
          echo "GITHUB_REF: $GITHUB_REF"
          SHORT_SHA=$(git rev-parse --short "$GITHUB_SHA")
          echo "SHORT_SHA: $SHORT_SHA"
          UNIQUE_TAG=$SHORT_SHA
          echo "UNIQUE_TAG: $UNIQUE_TAG"
          TRACKING_TAG=$(echo ${GITHUB_REF} | sed -e "s,refs/heads/main,head,g" -e 's,refs/tags/,,g' -e 's,refs/pull/\([0-9]*\).*,pr\1,g')
          echo "TRACKING_TAG: $TRACKING_TAG"

          {
            echo "unique_tag=${UNIQUE_TAG}"
            echo "tracking_tag=${TRACKING_TAG}"
          } >> $GITHUB_OUTPUT
      - name: Check whether publishing enabled
        id: publish
        env:
          ENABLE_PUBLISH: ${{ secrets.ACRYL_DOCKER_PASSWORD }}
        run: |
          if [[ -n "$ENABLE_PUBLISH" && "${{ github.repository }}" == "acryldata/datahub-actions" ]]; then
            echo "Publishing is enabled"
            echo "publish=true" >> "$GITHUB_OUTPUT"
          else
            echo "Publishing is not enabled"
            echo "publish=false" >> "$GITHUB_OUTPUT"
          fi
          echo "Enable publish: ${{ env.ENABLE_PUBLISH != '' && github.repository == 'acryldata/datahub-actions' }}"
  regular_image:
    name: Build & Push Image to DockerHub
    if: ${{ needs.setup.outputs.publish == 'true' }} # Only build the regular image if publishing is enabled
    runs-on: ubuntu-latest
    needs: setup
    steps:
      - name: Check out the repo
        uses: actions/checkout@v4
      - name: Setup Outputs
        run: |
          echo "GITHUB_REF: $GITHUB_REF"
          echo "unique_tag=${{ needs.setup.outputs.unique_tag }}"
          echo "tracking_tag=${{ needs.setup.outputs.tracking_tag }}"
      - name: Docker meta
        id: docker_meta
        uses: docker/metadata-action@v5
        with:
          images: |
            acryldata/datahub-actions
          tags: |
            type=raw,value=${{ needs.setup.outputs.unique_tag }}
            type=raw,value=${{ needs.setup.outputs.tracking_tag }},enable=${{ needs.setup.outputs.tracking_tag != '' }}
      - name: Set up QEMU
        uses: docker/setup-qemu-action@v3
        if: ${{ needs.setup.outputs.publish == 'true' }}
      - name: Set up Docker Buildx
        uses: docker/setup-buildx-action@v3
      - name: Login to DockerHub
        if: ${{ needs.setup.outputs.publish == 'true' }}
        uses: docker/login-action@v2
        with:
          username: ${{ secrets.ACRYL_DOCKER_USERNAME }}
          password: ${{ secrets.ACRYL_DOCKER_PASSWORD }}
      - name: Build & Push Image
        uses: docker/build-push-action@v6
        with:
          file: ./docker/datahub-actions/Dockerfile
          platforms: ${{ needs.setup.outputs.publish == 'true' && 'linux/amd64,linux/arm64' || 'linux/amd64' }}
          tags: ${{ steps.docker_meta.outputs.tags }}
          push: ${{ needs.setup.outputs.publish == 'true' }}
          cache-from: type=registry,ref=${{ steps.docker_meta.outputs.tags }}
          cache-to: type=inline
          target: final
  slim_image:
    name: Build & Push Image to DockerHub (slim)
    runs-on: ubuntu-latest
    needs: setup
    steps:
      - name: Check out the repo (slim)
        uses: actions/checkout@v3
      - name: Setup Outputs
        run: |
          echo "GITHUB_REF: $GITHUB_REF"
          echo "unique_tag=${{ needs.setup.outputs.unique_tag }}"
          echo "tracking_tag=${{ needs.setup.outputs.tracking_tag }}"
      - name: Docker meta (slim)
        id: docker_meta_slim
        uses: docker/metadata-action@v5
        with:
          # The slim images use a totally separate image name, so we don't need to
          # suffix the tags with -slim.
          images: |
            acryldata/datahub-actions-slim
          tags: |
            type=raw,value=${{ needs.setup.outputs.unique_tag }}
            type=raw,value=${{ needs.setup.outputs.tracking_tag }},enable=${{ needs.setup.outputs.tracking_tag != '' }}
      - name: Set up QEMU (slim)
        if: ${{ needs.setup.outputs.publish == 'true' }}
        uses: docker/setup-qemu-action@v2
      - name: Set up Docker Buildx (slim)
        uses: docker/setup-buildx-action@v2
      - name: Login to DockerHub (slim)
        uses: docker/login-action@v2
        if: ${{ needs.setup.outputs.publish == 'true' }}
        with:
          username: ${{ secrets.ACRYL_DOCKER_USERNAME }}
          password: ${{ secrets.ACRYL_DOCKER_PASSWORD }}
      - name: Build & Push Image (slim)
        uses: docker/build-push-action@v6
        with:
          file: ./docker/datahub-actions/Dockerfile
          platforms: ${{ needs.setup.outputs.publish == 'true' && 'linux/amd64,linux/arm64' || 'linux/amd64' }}
          tags: ${{ steps.docker_meta_slim.outputs.tags }}
          push: ${{ needs.setup.outputs.publish == 'true' }}
          target: final
          load: ${{ needs.setup.outputs.publish != 'true' }}
          build-args: |
            "APP_ENV=prod-slim"
      - name: Save Docker image
        if: needs.setup.outputs.publish != 'true'
        run: docker save ${{ steps.docker_meta_slim.outputs.tags }} > image.tar
      - name: Upload artifact
        if: needs.setup.outputs.publish != 'true'
        uses: actions/upload-artifact@v3
        with:
          name: docker-image
          path: image.tar
  # image_scan:
  #   permissions:
  #     contents: read # for actions/checkout to fetch code
  #     security-events: write # for github/codeql-action/upload-sarif to upload SARIF results
  #     actions: read # only required for a private repository by github/codeql-action/upload-sarif to get the Action run status
  #   name: "[Monitoring] Scan action images for vulnerabilities"
  #   runs-on: ubuntu-latest
  #   needs: [setup, slim_image]
  #   steps:
  #     - name: Checkout # adding checkout step just to make trivy upload happy
  #       uses: actions/checkout@v3
  #     - name: Download image
  #       uses: ishworkh/docker-image-artifact-download@v1
  #       if: ${{ needs.setup.outputs.publish != 'true' }}
  #       with:
  #         image: acryldata/datahub-actions:${{ needs.setup.outputs.unique_tag }}
  #     - name: Run Trivy vulnerability scanner
  #       uses: aquasecurity/trivy-action@master
  #       env:
  #         TRIVY_OFFLINE_SCAN: true
  #       with:
  #         image-ref: acryldata/datahub-actions:${{ needs.setup.outputs.unique_tag }}
  #         format: 'template'
  #         template: '@/contrib/sarif.tpl'
  #         output: 'trivy-results.sarif'
  #         severity: 'CRITICAL,HIGH'
  #         ignore-unfixed: true
  #         vuln-type: "os,library"
  #     - name: Upload Trivy scan results to GitHub Security tab
  #       uses: github/codeql-action/upload-sarif@v2
  #       with:
  #         sarif_file: 'trivy-results.sarif'
  image_scan_slim:
    permissions:
      contents: read # for actions/checkout to fetch code
      security-events: write # for github/codeql-action/upload-sarif to upload SARIF results
      actions: read # only required for a private repository by github/codeql-action/upload-sarif to get the Action run status
    name: "[Monitoring] Scan slim action images for vulnerabilities"
    runs-on: ubuntu-latest
    needs: [setup, slim_image]
    steps:
      - name: Checkout # adding checkout step just to make trivy upload happy
        uses: actions/checkout@v4
      - name: Download artifact (if not publishing)
        if: needs.setup.outputs.publish != 'true'
        uses: actions/download-artifact@v3
        with:
          name: docker-image
      - name: Load Docker image (if not publishing)
        if: needs.setup.outputs.publish != 'true'
        run: docker load < image.tar
      - name: Pull Docker image (if publishing)
        if: needs.setup.outputs.publish == 'true'
        run: docker pull acryldata/datahub-actions-slim:${{ needs.setup.outputs.unique_tag }}
      - name: Run Trivy vulnerability scanner (slim)
        uses: aquasecurity/trivy-action@master
        env:
          TRIVY_OFFLINE_SCAN: true
        with:
          image-ref: acryldata/datahub-actions-slim:${{ needs.setup.outputs.unique_tag }}
          format: "template"
          template: "@/contrib/sarif.tpl"
          output: "trivy-results.sarif"
          severity: "CRITICAL,HIGH"
          ignore-unfixed: true
          vuln-type: "os,library"
      - name: Upload Trivy scan results to GitHub Security tab (slim)
        uses: github/codeql-action/upload-sarif@v3
        with:
          sarif_file: "trivy-results.sarif"
  smoke_test:
    name: Run Smoke Tests
    runs-on: ubuntu-latest
    needs: [setup, slim_image]
    steps:
      - name: Free up disk space
        run: |
          sudo apt-get remove 'dotnet-*' azure-cli || true
          sudo rm -rf /usr/local/lib/android/ || true
          sudo docker image prune -a -f || true
      - name: Disk Check
        run: df -h . && docker images
      - name: Checkout the repo
        uses: actions/checkout@v4
      - name: Set up JDK 17
        uses: actions/setup-java@v3
        with:
          distribution: "zulu"
          java-version: 17
      - uses: gradle/actions/setup-gradle@v3
      - uses: actions/setup-python@v4
        with:
          python-version: "3.10"
          cache: "pip"
      - name: Download artifact (if not publishing)
        if: needs.setup.outputs.publish != 'true'
        uses: actions/download-artifact@v3
        with:
          name: docker-image
      - name: Load Docker image (if not publishing)
        if: needs.setup.outputs.publish != 'true'
        run: docker load < image.tar
      - name: Pull Docker image (if publishing)
        if: needs.setup.outputs.publish == 'true'
        run: docker pull acryldata/datahub-actions-slim:${{ needs.setup.outputs.unique_tag }}
      - name: run quickstart
        env:
          DATAHUB_TELEMETRY_ENABLED: false
          DATAHUB_ACTIONS_IMAGE: acryldata/datahub-actions-slim
          DATAHUB_ACTIONS_VERSION: ${{ needs.setup.outputs.unique_tag }}
          ACTIONS_EXTRA_PACKAGES: "acryl-datahub-actions[executor]==0.0.13 acryl-datahub-actions==0.0.13 acryl-datahub==0.10.5"
          ACTIONS_CONFIG: "https://raw.githubusercontent.com/acryldata/datahub-actions/main/docker/config/executor.yaml"
        run: |
          ./smoke-test/run-quickstart.sh
      - name: Disk Check
        run: df -h . && docker images
      - name: Smoke test
        env:
          RUN_QUICKSTART: false
          DATAHUB_ACTIONS_VERSION: ${{ needs.setup.outputs.unique_tag }}
        run: |
          echo "$DATAHUB_ACTIONS_VERSION"
          ./smoke-test/smoke.sh