velox_docker_cache.yml

# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements.  See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License.  You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

name: Velox backend Dependency Cache

on:
  push:
    branches:
      - 'main'

concurrency:
  group: ${{ github.repository }}-${{ github.workflow }}
  cancel-in-progress: false

jobs:
  cache-native-lib:
    runs-on: ubuntu-20.04
    container: inteldpo/gluten-centos-packaging:latest # centos7 with dependencies installed
    steps:
      - uses: actions/checkout@v2
      - name: Generate cache key
        run: |
          echo ${{ hashFiles('./ep/build-velox/src/**', './dev/**', './cpp/*', './github/workflows/*') }} > cache-key
      - name: Check existing caches
        id: check-cache
        uses: actions/cache/restore@v3
        with:
          lookup-only: true
          path: ./cpp/build/releases/
          key: cache-velox-build-${{ hashFiles('./cache-key') }}
      - name: Build Gluten Velox third party
        if: steps.check-cache.outputs.cache-hit != 'true'
        run: |
          source dev/ci-velox-buildstatic.sh
      - name: Cache
        if: steps.check-cache.outputs.cache-hit != 'true'
        id: cache
        uses: actions/cache/save@v3
        with:
          path: ./cpp/build/releases/
          key: cache-velox-build-${{ hashFiles('./cache-key') }}

  ccache-native-lib-ubuntu-velox-ut:
    runs-on: ubuntu-20.04
    env:
      CCACHE_DIR: "${{ github.workspace }}/.ccache"
    container: ghcr.io/facebookincubator/velox-dev:amd64-ubuntu-22.04-avx
    steps:
      - uses: actions/checkout@v2
      - name: Get Ccache
        uses: actions/cache/restore@v3
        with:
          path: '${{ env.CCACHE_DIR }}'
          key: ccache-ubuntu-release-default
      - name: Ensure Cache Dirs Exists
        working-directory: ${{ github.workspace }}
        run: |
          mkdir -p '${{ env.CCACHE_DIR }}'
      - name: Build Gluten velox third party
        run: |
          rm -rf /opt/miniconda-for-velox/
          cd ep/build-velox/src && \
          ./get_velox.sh
          cd ../build/velox_ep/
          make EXTRA_CMAKE_FLAGS="-DVELOX_ENABLE_PARQUET=ON -DVELOX_BUILD_TESTING=ON -DVELOX_BUILD_TEST_UTILS=ON" 

      - name: CCache after
        run: |
          ccache -vs

      - uses: actions/cache/save@v3
        with:
          path: '${{ env.CCACHE_DIR }}'
          key: ccache-ubuntu-release-default
  ccache-native-lib-centos-velox-ut:
    runs-on: ubuntu-20.04
    env:
      CCACHE_DIR: "${{ github.workspace }}/.ccache"
    container: ghcr.io/facebookincubator/velox-dev:circleci-avx
    steps:
      - uses: actions/checkout@v2
      - name: Setup java and maven
        run: |
          yum install sudo patch java-1.8.0-openjdk-devel wget -y && \
          wget https://downloads.apache.org/maven/maven-3/3.8.8/binaries/apache-maven-3.8.8-bin.tar.gz
          tar -xvf apache-maven-3.8.8-bin.tar.gz
          mv apache-maven-3.8.8 /usr/lib/maven
      - name: Get Ccache
        uses: actions/cache/restore@v3
        with:
          path: '${{ env.CCACHE_DIR }}'
          key: ccache-centos-release-default
      - name: Ensure Cache Dirs Exists
        working-directory: ${{ github.workspace }}
        run: |
          mkdir -p '${{ env.CCACHE_DIR }}'
      - name: Build Gluten velox third party
        run: |
          rm -rf /opt/miniconda-for-velox/
          cd ep/build-velox/src && \
          ./get_velox.sh
          cd ../build/velox_ep/
          source /opt/rh/gcc-toolset-9/enable
          make EXTRA_CMAKE_FLAGS="-DVELOX_ENABLE_PARQUET=ON -DVELOX_BUILD_TESTING=ON -DVELOX_BUILD_TEST_UTILS=ON" 

      - name: CCache after
        run: |
          ccache -s

      - uses: actions/cache/save@v3
        with:
          path: '${{ env.CCACHE_DIR }}'
          key: ccache-centos-release-default