Skip to content

Build bundle package #1

Build bundle package

Build bundle package #1

# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
name: Build bundle package
concurrency:
group: ${{ github.repository }}-${{ github.head_ref || github.sha }}-${{ github.workflow }}
cancel-in-progress: true
on:
workflow_dispatch:
inputs:
os:
description: 'OS version: ubuntu:20.04, ubuntu:22.04, centos:7 or centos:8'
required: true
default: 'ubuntu:20.04'
spark:
description: 'Spark version: spark-3.2, spark-3.3, spark-3.4 or spark-3.5'
required: true
default: 'spark-3.5'
hadoop:
description: 'Hadoop version: 2.7.4, 3.2.0, 3.2.2, 3.3.1, 3.3.3 or 3.3.6'
required: true
default: '3.3.3'
jobs:
build-native-lib:
runs-on: ubuntu-20.04
container: inteldpo/gluten-centos-packaging:latest
steps:
- uses: actions/checkout@v2
- name: Build Gluten velox third party
run: |
yum install sudo patch java-1.8.0-openjdk-devel -y && \
cd $GITHUB_WORKSPACE/ep/build-velox/src && \
./get_velox.sh && \
source /opt/rh/devtoolset-9/enable && \
source /opt/gluten//dev/vcpkg/env.sh && \
cd $GITHUB_WORKSPACE/ && \
sed -i '/^headers/d' ep/build-velox/build/velox_ep/CMakeLists.txt && \
export NUM_THREADS=4
./dev/builddeps-veloxbe.sh --build_tests=OFF --build_benchmarks=OFF --enable_s3=OFF \
--enable_gcs=OFF --enable_hdfs=ON --enable_abfs=OFF
- uses: actions/upload-artifact@v2
with:
path: ./cpp/build/releases/
name: velox-native-lib-${{github.sha}}
retention-days: 1
build-bundle-package-ubuntu:
if: startsWith(github.event.inputs.os, 'ubuntu')
needs: build-native-lib
runs-on: ubuntu-20.04
container: ${{ github.event.inputs.os }}
steps:
- uses: actions/checkout@v2
- name: Download All Artifacts
uses: actions/download-artifact@v2
with:
name: velox-native-lib-${{github.sha}}
path: ./cpp/build/releases
- name: Setup java and maven
run: |
apt-get update && \
apt-get install -y openjdk-8-jdk maven && \
apt remove openjdk-11* -y
- name: Build for Spark ${{ github.event.inputs.spark }}
run: |
cd $GITHUB_WORKSPACE/ && \
mvn clean install -P${{ github.event.inputs.spark }} -Dhadoop.version=${{ github.event.inputs.hadoop }} -Pbackends-velox -Prss -DskipTests -Dmaven.source.skip
- name: Upload bundle package
uses: actions/upload-artifact@v2
with:
name: gluten-velox-bundle-package
path: package/target/gluten-velox-bundle-*.jar
retention-days: 7
build-bundle-package-centos7:
if: ${{ github.event.inputs.os == 'centos:7' }}
needs: build-native-lib
runs-on: ubuntu-20.04
container: ${{ github.event.inputs.os }}
steps:
- uses: actions/checkout@v2
- name: Download All Artifacts
uses: actions/download-artifact@v2
with:
name: velox-native-lib-${{github.sha}}
path: ./cpp/build/releases
- name: Setup java and maven
run: |
yum update -y && yum install -y java-1.8.0-openjdk-devel wget && \
wget https://downloads.apache.org/maven/maven-3/3.8.8/binaries/apache-maven-3.8.8-bin.tar.gz && \
tar -xvf apache-maven-3.8.8-bin.tar.gz && \
mv apache-maven-3.8.8 /usr/lib/maven
- name: Build for Spark ${{ github.event.inputs.spark }}
run: |
cd $GITHUB_WORKSPACE/ && \
export MAVEN_HOME=/usr/lib/maven && \
export PATH=${PATH}:${MAVEN_HOME}/bin && \
mvn clean install -P${{ github.event.inputs.spark }} -Dhadoop.version=${{ github.event.inputs.hadoop }} -Pbackends-velox -Prss -DskipTests -Dmaven.source.skip
- name: Upload bundle package
uses: actions/upload-artifact@v2
with:
name: gluten-velox-bundle-package
path: package/target/gluten-velox-bundle-*.jar
retention-days: 7
build-bundle-package-centos8:
if: ${{ github.event.inputs.os == 'centos:8' }}
needs: build-native-lib
runs-on: ubuntu-20.04
container: ${{ github.event.inputs.os }}
steps:
- uses: actions/checkout@v2
- name: Download All Artifacts
uses: actions/download-artifact@v2
with:
name: velox-native-lib-${{github.sha}}
path: ./cpp/build/releases
- name: Update mirror list
run: |
sed -i -e "s|mirrorlist=|#mirrorlist=|g" /etc/yum.repos.d/CentOS-* || true && \
sed -i -e "s|#baseurl=http://mirror.centos.org|baseurl=http://vault.centos.org|g" /etc/yum.repos.d/CentOS-* || true
- name: Setup java and maven
run: |
yum update -y && yum install -y java-1.8.0-openjdk-devel wget && \
wget https://downloads.apache.org/maven/maven-3/3.8.8/binaries/apache-maven-3.8.8-bin.tar.gz && \
tar -xvf apache-maven-3.8.8-bin.tar.gz && \
mv apache-maven-3.8.8 /usr/lib/maven
- name: Build for Spark ${{ github.event.inputs.spark }}
run: |
cd $GITHUB_WORKSPACE/ && \
export MAVEN_HOME=/usr/lib/maven && \
export PATH=${PATH}:${MAVEN_HOME}/bin && \
mvn clean install -P${{ github.event.inputs.spark }} -Dhadoop.version=${{ github.event.inputs.hadoop }} -Pbackends-velox -Prss -DskipTests -Dmaven.source.skip
- name: Upload bundle package
uses: actions/upload-artifact@v2
with:
name: gluten-velox-bundle-package
path: package/target/gluten-velox-bundle-*.jar
retention-days: 7