release_build #72
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # FIXME: update this workflow | |
| name: release_build | |
| on: # yamllint disable-line rule:truthy | |
| workflow_dispatch: | |
| jobs: | |
| Build_Linux_X86_64: | |
| uses: timeplus-io/proton/.github/workflows/run_command.yml@develop | |
| with: | |
| ec2-instance-type: c5.4xlarge | |
| ec2-image-id: ami-042a37e33a285c22b | |
| submodules: 'recursive' | |
| run_mode: 'start' # start ec2 on demand instance | |
| upload_files: proton-*-Linux-x86_64 | |
| command: | | |
| cd $GITHUB_WORKSPACE | |
| # git config | |
| git config user.name "proton-robot" | |
| git config user.email "[email protected]" | |
| # update version | |
| ./release --version patch | |
| # prepare build cache | |
| aws s3 cp --no-progress s3://tp-internal/proton/cache.tar.gz . | |
| mkdir $GITHUB_WORKSPACE/ccache | |
| tar -zxf ./cache.tar.gz -C $GITHUB_WORKSPACE/ccache | |
| rm cache.tar.gz | |
| # compiling | |
| ./docker/packager/packager --package-type binary --docker-image-version clang-16 --build-type release --proton-build --enable-proton-local --cache ccache --ccache_dir $GITHUB_WORKSPACE/ccache --output-dir $GITHUB_WORKSPACE/output | |
| if [ ! -f "$GITHUB_WORKSPACE/output/proton" ]; then | |
| echo "Compiling proton Failed" | |
| exit 127 | |
| fi | |
| # get proton tag | |
| PROTON_TAG=`grep "SET(VERSION_DESCRIBE" $GITHUB_WORKSPACE/cmake/autogenerated_versions.txt | sed 's/^.*VERSION_DESCRIBE \(.*\)$/\1/' | sed 's/[) ].*//'` | |
| echo "Proton tag: $PROTON_TAG" | |
| echo "tag_name=$PROTON_TAG" >> $GITHUB_OUTPUT | |
| PROTON_BINARY=proton-$PROTON_TAG-Linux-x86_64 | |
| echo "Proton Binary Name: $PROTON_BINARY" | |
| docker run --name static-server -p 8080:80 -v $GITHUB_WORKSPACE/output:/usr/share/nginx/html:ro -d nginx | |
| cd $GITHUB_WORKSPACE/docker/server | |
| # build docker image | |
| docker build . --network host --build-arg single_binary_location_url=http://localhost:8080/proton -t ghcr.io/timeplus-io/proton:$GITHUB_SHA-amd64 | |
| # push docker image | |
| docker push ghcr.io/timeplus-io/proton:$GITHUB_SHA-amd64 | |
| # upload build cache | |
| tar -zcf ./cache.tar.gz -C $GITHUB_WORKSPACE/ccache . | |
| aws s3 cp --no-progress ./cache.tar.gz s3://tp-internal/proton/ | |
| mv $GITHUB_WORKSPACE/output/proton $GITHUB_WORKSPACE/$PROTON_BINARY | |
| secrets: | |
| AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} | |
| AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} | |
| AWS_REGION: ${{ secrets.AWS_REGION }} | |
| AWS_S3_BUCKET: ${{ secrets.AWS_S3_BUCKET }} | |
| GH_PERSONAL_ACCESS_TOKEN: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }} | |
| DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} | |
| DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} | |
| Build_Linux_Arm64: | |
| uses: timeplus-io/proton/.github/workflows/run_command.yml@develop | |
| with: | |
| ec2-instance-type: c6g.4xlarge | |
| ec2-image-id: ami-0f3dbc4cc9994fdee | |
| submodules: 'recursive' | |
| run_mode: 'start' # start ec2 on demand instance | |
| upload_files: proton-*-Linux-aarch64 | |
| command: | | |
| cd $GITHUB_WORKSPACE | |
| # git config | |
| git config user.name "proton-robot" | |
| git config user.email "[email protected]" | |
| # update version | |
| ./release --version patch | |
| # prepare build cache | |
| aws s3 cp --no-progress s3://tp-internal/proton/cache-arm.tar.gz . | |
| mkdir $GITHUB_WORKSPACE/ccache | |
| tar -zxf ./cache-arm.tar.gz -C $GITHUB_WORKSPACE/ccache | |
| rm cache-arm.tar.gz | |
| # compiling | |
| ./docker/packager/packager --package-type binary --docker-image-version clang-16 --build-type release --proton-build --enable-proton-local --cache ccache --ccache_dir $GITHUB_WORKSPACE/ccache --output-dir $GITHUB_WORKSPACE/output | |
| # get proton tag | |
| PROTON_TAG=`grep "SET(VERSION_DESCRIBE" $GITHUB_WORKSPACE/cmake/autogenerated_versions.txt | sed 's/^.*VERSION_DESCRIBE \(.*\)$/\1/' | sed 's/[) ].*//'` | |
| echo "Proton tag: $PROTON_TAG" | |
| echo "tag_name=$PROTON_TAG" >> $GITHUB_OUTPUT | |
| PROTON_BINARY=proton-$PROTON_TAG-Linux-aarch64 | |
| echo "Proton Binary Name: $PROTON_BINARY" | |
| docker run --name static-server -p 8080:80 -v $GITHUB_WORKSPACE/output:/usr/share/nginx/html:ro -d nginx | |
| cd $GITHUB_WORKSPACE/docker/server | |
| # build docker image | |
| docker build . --network host --build-arg single_binary_location_url=http://localhost:8080/proton -t ghcr.io/timeplus-io/proton:$GITHUB_SHA-arm64v8 | |
| # push docker image | |
| docker push ghcr.io/timeplus-io/proton:$GITHUB_SHA-arm64v8 | |
| # upload build cache | |
| tar -zcf ./cache-arm.tar.gz -C $GITHUB_WORKSPACE/ccache . | |
| aws s3 cp --no-progress ./cache-arm.tar.gz s3://tp-internal/proton/ | |
| mv $GITHUB_WORKSPACE/output/proton $GITHUB_WORKSPACE/$PROTON_BINARY | |
| secrets: | |
| AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} | |
| AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} | |
| AWS_REGION: ${{ secrets.AWS_REGION }} | |
| AWS_S3_BUCKET: ${{ secrets.AWS_S3_BUCKET }} | |
| GH_PERSONAL_ACCESS_TOKEN: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }} | |
| DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} | |
| DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} | |
| CommitTag: | |
| needs: [Build_Linux_X86_64, Build_Linux_Arm64] | |
| uses: timeplus-io/proton/.github/workflows/run_command.yml@develop | |
| with: | |
| ec2-instance-type: c5.large | |
| ec2-image-id: ami-042a37e33a285c22b | |
| submodules: 'false' | |
| command: | | |
| cd $GITHUB_WORKSPACE | |
| # git config | |
| git config user.name "proton-robot" | |
| git config user.email "[email protected]" | |
| docker pull ghcr.io/timeplus-io/proton:$GITHUB_SHA-amd64 | |
| docker pull ghcr.io/timeplus-io/proton:$GITHUB_SHA-arm64v8 | |
| docker manifest create ghcr.io/timeplus-io/proton:latest \ | |
| ghcr.io/timeplus-io/proton:$GITHUB_SHA-amd64 \ | |
| ghcr.io/timeplus-io/proton:$GITHUB_SHA-arm64v8 | |
| docker manifest push ghcr.io/timeplus-io/proton:latest | |
| PROTON_TAG="$(docker history --no-trunc ghcr.io/timeplus-io/proton:$GITHUB_SHA-amd64 | grep -Eo 'ARG version=[0-9]+.[0-9]+.[0-9]+' -o | grep -Eo '[0-9]+.[0-9]+.[0-9]+')" | |
| docker manifest create ghcr.io/timeplus-io/proton:$PROTON_TAG-rc \ | |
| ghcr.io/timeplus-io/proton:$GITHUB_SHA-amd64 \ | |
| ghcr.io/timeplus-io/proton:$GITHUB_SHA-arm64v8 | |
| docker manifest push ghcr.io/timeplus-io/proton:$PROTON_TAG-rc | |
| curl -L \ | |
| -X POST \ | |
| -H "Accept: application/vnd.github+json" \ | |
| -H "Authorization: token $GH_PERSONAL_ACCESS_TOKEN"\ | |
| -H "X-GitHub-Api-Version: 2022-11-28" \ | |
| https://api.github.com/repos/timeplus-io/proton/actions/workflows/manual_trigger_test.yml/dispatches \ | |
| -d "{\"ref\":\"develop\",\"inputs\":{\"arch\": \"x64\", \"tag\":\"$PROTON_TAG-rc\"}}\"" | |
| curl -L \ | |
| -X POST \ | |
| -H "Accept: application/vnd.github+json" \ | |
| -H "Authorization: token $GH_PERSONAL_ACCESS_TOKEN"\ | |
| -H "X-GitHub-Api-Version: 2022-11-28" \ | |
| https://api.github.com/repos/timeplus-io/proton/actions/workflows/manual_trigger_test.yml/dispatches \ | |
| -d "{\"ref\":\"develop\",\"inputs\":{\"arch\": \"arm\", \"tag\":\"$PROTON_TAG-rc\"}}\"" | |
| secrets: | |
| AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} | |
| AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} | |
| AWS_REGION: ${{ secrets.AWS_REGION }} | |
| AWS_S3_BUCKET: ${{ secrets.AWS_S3_BUCKET }} | |
| GH_PERSONAL_ACCESS_TOKEN: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }} | |
| DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} | |
| DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} | |
| Build_Darwin_X86_64: | |
| uses: timeplus-io/proton/.github/workflows/run_command.yml@develop | |
| with: | |
| ec2-instance-type: c5.4xlarge | |
| ec2-image-id: ami-042a37e33a285c22b | |
| submodules: 'recursive' | |
| run_mode: 'start' # start ec2 on demand instance | |
| upload_files: proton-*-Darwin-x86_64 | |
| command: | | |
| cd $GITHUB_WORKSPACE | |
| # git config | |
| git config user.name "proton-robot" | |
| git config user.email "[email protected]" | |
| # update version | |
| ./release --version patch | |
| # download the pre-built binary of v8 (this is only for cross-compile) | |
| aws s3 cp --no-progress s3://tp-internal/proton/cross-compile-prebuilt-binary/v8-cmake-x64.tar.gz $GITHUB_WORKSPACE/contrib/v8-cmake/ | |
| tar -zxf $GITHUB_WORKSPACE/contrib/v8-cmake/v8-cmake-x64.tar.gz -C $GITHUB_WORKSPACE/contrib/v8-cmake/ | |
| rm $GITHUB_WORKSPACE/contrib/v8-cmake/v8-cmake-x64.tar.gz | |
| chmod a+x $GITHUB_WORKSPACE/contrib/v8-cmake/bytecode_builtins_list_generator | |
| chmod a+x $GITHUB_WORKSPACE/contrib/v8-cmake/mksnapshot | |
| chmod a+x $GITHUB_WORKSPACE/contrib/v8-cmake/torque | |
| # prepare build cache | |
| mkdir $GITHUB_WORKSPACE/ccache | |
| echo "max_size = 100.0G" > $GITHUB_WORKSPACE/ccache/ccache.conf | |
| # compiling | |
| ./docker/packager/packager --package-type binary --docker-image-version clang-16 --build-type release --proton-build --enable-proton-local --cache ccache --ccache_dir $GITHUB_WORKSPACE/ccache --output-dir $GITHUB_WORKSPACE/output --compiler clang-16-darwin | |
| if [ ! -f "$GITHUB_WORKSPACE/output/proton" ]; then | |
| echo "Compiling proton Failed" | |
| exit 127 | |
| fi | |
| # clean ccache and build footprint | |
| rm -rf $GITHUB_WORKSPACE/ccache | |
| rm -rf $GITHUB_WORKSPACE/build_docker | |
| # get proton tag | |
| PROTON_TAG=`grep "SET(VERSION_DESCRIBE" $GITHUB_WORKSPACE/cmake/autogenerated_versions.txt | sed 's/^.*VERSION_DESCRIBE \(.*\)$/\1/' | sed 's/[) ].*//'` | |
| echo "Proton tag: $PROTON_TAG" | |
| echo "tag_name=$PROTON_TAG" >> $GITHUB_OUTPUT | |
| PROTON_BINARY=proton-$PROTON_TAG-Darwin-x86_64 | |
| echo "Proton Binary Name: $PROTON_BINARY" | |
| sha256sum $GITHUB_WORKSPACE/output/proton | |
| mv $GITHUB_WORKSPACE/output/proton $GITHUB_WORKSPACE/$PROTON_BINARY | |
| secrets: | |
| AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} | |
| AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} | |
| AWS_REGION: ${{ secrets.AWS_REGION }} | |
| AWS_S3_BUCKET: ${{ secrets.AWS_S3_BUCKET }} | |
| GH_PERSONAL_ACCESS_TOKEN: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }} | |
| DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} | |
| DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} | |
| Build_Darwin_Arm64: | |
| if: false | |
| uses: timeplus-io/proton/.github/workflows/run_command.yml@develop | |
| with: | |
| ec2-instance-type: c6g.4xlarge | |
| ec2-image-id: ami-0f3dbc4cc9994fdee | |
| submodules: 'recursive' | |
| run_mode: 'start' # start ec2 on demand instance | |
| upload_files: proton-*-Darwin-arm64 | |
| command: | | |
| cd $GITHUB_WORKSPACE | |
| # git config | |
| git config user.name "proton-robot" | |
| git config user.email "[email protected]" | |
| # update version | |
| ./release --version patch | |
| # download the pre-built binary of v8 (this is only for cross-compile) | |
| aws s3 cp --no-progress s3://tp-internal/proton/cross-compile-prebuilt-binary/v8-cmake-arm64.tar.gz $GITHUB_WORKSPACE/contrib/v8-cmake/ | |
| tar -zxf $GITHUB_WORKSPACE/contrib/v8-cmake/v8-cmake-arm64.tar.gz -C $GITHUB_WORKSPACE/contrib/v8-cmake/ | |
| rm $GITHUB_WORKSPACE/contrib/v8-cmake/v8-cmake-arm64.tar.gz | |
| chmod a+x $GITHUB_WORKSPACE/contrib/v8-cmake/bytecode_builtins_list_generator | |
| chmod a+x $GITHUB_WORKSPACE/contrib/v8-cmake/mksnapshot | |
| chmod a+x $GITHUB_WORKSPACE/contrib/v8-cmake/torque | |
| # prepare build cache | |
| mkdir $GITHUB_WORKSPACE/ccache | |
| echo "max_size = 100.0G" > $GITHUB_WORKSPACE/ccache/ccache.conf | |
| # compiling | |
| ./docker/packager/packager --package-type binary --docker-image-version clang-16 --build-type release --proton-build --enable-proton-local --cache ccache --ccache_dir $GITHUB_WORKSPACE/ccache --output-dir $GITHUB_WORKSPACE/output --compiler clang-16-darwin-aarch64 | |
| if [ ! -f "$GITHUB_WORKSPACE/output/proton" ]; then | |
| echo "Compiling proton Failed" | |
| exit 127 | |
| fi | |
| # clean ccache and build footprint | |
| rm -rf $GITHUB_WORKSPACE/ccache | |
| rm -rf $GITHUB_WORKSPACE/build_docker | |
| # get proton tag | |
| PROTON_TAG=`grep "SET(VERSION_DESCRIBE" $GITHUB_WORKSPACE/cmake/autogenerated_versions.txt | sed 's/^.*VERSION_DESCRIBE \(.*\)$/\1/' | sed 's/[) ].*//'` | |
| echo "Proton tag: $PROTON_TAG" | |
| echo "tag_name=$PROTON_TAG" >> $GITHUB_OUTPUT | |
| PROTON_BINARY=proton-$PROTON_TAG-Darwin-arm64 | |
| echo "Proton Binary Name: $PROTON_BINARY" | |
| sha256sum $GITHUB_WORKSPACE/output/proton | |
| mv $GITHUB_WORKSPACE/output/proton $GITHUB_WORKSPACE/$PROTON_BINARY | |
| secrets: | |
| AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} | |
| AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} | |
| AWS_REGION: ${{ secrets.AWS_REGION }} | |
| AWS_S3_BUCKET: ${{ secrets.AWS_S3_BUCKET }} | |
| GH_PERSONAL_ACCESS_TOKEN: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }} | |
| DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} | |
| DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} | |
| Build_Native_Darwin_arm64: | |
| needs: Build_Linux_X86_64 | |
| runs-on: [self-hosted, macOS, ARM64] | |
| env: | |
| build_directory: ${{ github.workspace }}/build | |
| build_type: Release | |
| steps: | |
| - name: Checkout | |
| uses: actions/[email protected] | |
| with: | |
| token: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }} | |
| fetch-depth: 0 # Fetch all history including all tags | |
| submodules: true | |
| # Checkout the latest tag | |
| - name: Checkout latest tag | |
| run: | | |
| git fetch --all --tags | |
| LATEST_TAG=$(git describe --tags `git rev-list --tags --max-count=1`) | |
| echo "Checking out tag $LATEST_TAG" | |
| git checkout $LATEST_TAG | |
| echo "LATEST_TAG=$LATEST_TAG" >> $GITHUB_ENV | |
| # Setup build environment and configure CMake | |
| - name: Create and Configure Build | |
| run: | | |
| mkdir -p ${{ env.build_directory }} | |
| export CC=$(brew --prefix llvm@17)/bin/clang | |
| export CXX=$(brew --prefix llvm@17)/bin/clang++ | |
| export PATH=$(brew --prefix llvm@17)/bin:$PATH | |
| cmake -B ${{ env.build_directory }} -G "Ninja" -DCMAKE_BUILD_TYPE=${{ env.build_type }} -DENABLE_TESTS=OFF -DENABLE_UTILS=OFF -DENABLE_EXAMPLES=OFF | |
| # Compile the project using Ninja | |
| - name: Build with Ninja | |
| run: cmake --build ${{ env.build_directory }} | |
| # Strip the binary to reduce its size and rename it | |
| - name: Strip and Rename binary | |
| run: | | |
| ORIGINAL_BINARY=${{ env.build_directory }}/programs/proton | |
| STRIPPED_BINARY=${{ env.build_directory }}/programs/proton-${{ env.LATEST_TAG }}-Darwin-arm64 | |
| /opt/homebrew/opt/llvm@17/bin/llvm-strip $ORIGINAL_BINARY -o $STRIPPED_BINARY | |
| # Set up AWS credentials for S3 upload | |
| - name: Configure AWS credentials | |
| uses: aws-actions/[email protected] | |
| with: | |
| aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} | |
| aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} | |
| aws-region: ${{ secrets.AWS_REGION }} | |
| # Upload the stripped binary to an AWS S3 bucket | |
| - name: Upload Artifact To S3 | |
| run: | | |
| STRIPPED_BINARY=${{ env.build_directory }}/programs/proton-${{ env.LATEST_TAG }}-Darwin-arm64 | |
| aws s3 cp --no-progress $STRIPPED_BINARY s3://tp-internal/proton/native_build_macOS/ | |
| # Print SHA256 Checksum | |
| - name: Print SHA256 Checksum | |
| run: | | |
| STRIPPED_BINARY=${{ env.build_directory }}/programs/proton-${{ env.LATEST_TAG }}-Darwin-arm64 | |
| shasum -a 256 $STRIPPED_BINARY | |
| # Release the binary on GitHub Releases with automatic release notes | |
| - name: Release binary | |
| uses: softprops/[email protected] | |
| with: | |
| files: ${{ env.build_directory }}/programs/proton-${{ env.LATEST_TAG }}-Darwin-arm64 | |
| token: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }} | |
| generate_release_notes: true | |
| tag_name: ${{ env.LATEST_TAG }} |