release_build #62
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # FIXME: update this workflow | |
| name: release_build | |
| on: # yamllint disable-line rule:truthy | |
| workflow_dispatch: | |
| jobs: | |
| Build_Linux_X86_64: | |
| uses: timeplus-io/proton/.github/workflows/run_command.yml@develop | |
| with: | |
| ec2-instance-type: c5.4xlarge | |
| ec2-image-id: ami-042a37e33a285c22b | |
| submodules: 'recursive' | |
| run_mode: 'start' # start ec2 on demand instance | |
| upload_files: proton-*-Linux-x86_64 | |
| command: | | |
| cd $GITHUB_WORKSPACE | |
| # git config | |
| git config user.name "proton-robot" | |
| git config user.email "[email protected]" | |
| # update version | |
| ./release --version patch | |
| # prepare build cache | |
| aws s3 cp --no-progress s3://tp-internal/proton/cache.tar.gz . | |
| mkdir $GITHUB_WORKSPACE/ccache | |
| tar -zxf ./cache.tar.gz -C $GITHUB_WORKSPACE/ccache | |
| rm cache.tar.gz | |
| # compiling | |
| ./docker/packager/packager --package-type binary --docker-image-version clang-16 --build-type release --proton-build --enable-proton-local --cache ccache --ccache_dir $GITHUB_WORKSPACE/ccache --output-dir $GITHUB_WORKSPACE/output | |
| if [ ! -f "$GITHUB_WORKSPACE/output/proton" ]; then | |
| echo "Compiling proton Failed" | |
| exit 127 | |
| fi | |
| # get proton tag | |
| PROTON_TAG=`grep "SET(VERSION_DESCRIBE" $GITHUB_WORKSPACE/cmake/autogenerated_versions.txt | sed 's/^.*VERSION_DESCRIBE \(.*\)$/\1/' | sed 's/[) ].*//'` | |
| echo "Proton tag: $PROTON_TAG" | |
| echo "tag_name=$PROTON_TAG" >> $GITHUB_OUTPUT | |
| PROTON_BINARY=proton-$PROTON_TAG-Linux-x86_64 | |
| echo "Proton Binary Name: $PROTON_BINARY" | |
| docker run --name static-server -p 8080:80 -v $GITHUB_WORKSPACE/output:/usr/share/nginx/html:ro -d nginx | |
| cd $GITHUB_WORKSPACE/docker/server | |
| # build docker image | |
| docker build . --network host --build-arg single_binary_location_url=http://localhost:8080/proton -t ghcr.io/timeplus-io/proton:$GITHUB_SHA-amd64 | |
| # push docker image | |
| docker push ghcr.io/timeplus-io/proton:$GITHUB_SHA-amd64 | |
| # upload build cache | |
| tar -zcf ./cache.tar.gz -C $GITHUB_WORKSPACE/ccache . | |
| aws s3 cp --no-progress ./cache.tar.gz s3://tp-internal/proton/ | |
| mv $GITHUB_WORKSPACE/output/proton $GITHUB_WORKSPACE/$PROTON_BINARY | |
| secrets: | |
| AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} | |
| AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} | |
| AWS_REGION: ${{ secrets.AWS_REGION }} | |
| AWS_S3_BUCKET: ${{ secrets.AWS_S3_BUCKET }} | |
| GH_PERSONAL_ACCESS_TOKEN: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }} | |
| DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} | |
| DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} | |
| Build_Linux_Arm64: | |
| uses: timeplus-io/proton/.github/workflows/run_command.yml@develop | |
| with: | |
| ec2-instance-type: c6g.4xlarge | |
| ec2-image-id: ami-0f3dbc4cc9994fdee | |
| submodules: 'recursive' | |
| run_mode: 'start' # start ec2 on demand instance | |
| upload_files: proton-*-Linux-aarch64 | |
| command: | | |
| cd $GITHUB_WORKSPACE | |
| # git config | |
| git config user.name "proton-robot" | |
| git config user.email "[email protected]" | |
| # update version | |
| ./release --version patch | |
| # prepare build cache | |
| aws s3 cp --no-progress s3://tp-internal/proton/cache-arm.tar.gz . | |
| mkdir $GITHUB_WORKSPACE/ccache | |
| tar -zxf ./cache-arm.tar.gz -C $GITHUB_WORKSPACE/ccache | |
| rm cache-arm.tar.gz | |
| # compiling | |
| ./docker/packager/packager --package-type binary --docker-image-version clang-16 --build-type release --proton-build --enable-proton-local --cache ccache --ccache_dir $GITHUB_WORKSPACE/ccache --output-dir $GITHUB_WORKSPACE/output | |
| # get proton tag | |
| PROTON_TAG=`grep "SET(VERSION_DESCRIBE" $GITHUB_WORKSPACE/cmake/autogenerated_versions.txt | sed 's/^.*VERSION_DESCRIBE \(.*\)$/\1/' | sed 's/[) ].*//'` | |
| echo "Proton tag: $PROTON_TAG" | |
| echo "tag_name=$PROTON_TAG" >> $GITHUB_OUTPUT | |
| PROTON_BINARY=proton-$PROTON_TAG-Linux-aarch64 | |
| echo "Proton Binary Name: $PROTON_BINARY" | |
| docker run --name static-server -p 8080:80 -v $GITHUB_WORKSPACE/output:/usr/share/nginx/html:ro -d nginx | |
| cd $GITHUB_WORKSPACE/docker/server | |
| # build docker image | |
| docker build . --network host --build-arg single_binary_location_url=http://localhost:8080/proton -t ghcr.io/timeplus-io/proton:$GITHUB_SHA-arm64v8 | |
| # push docker image | |
| docker push ghcr.io/timeplus-io/proton:$GITHUB_SHA-arm64v8 | |
| # upload build cache | |
| tar -zcf ./cache-arm.tar.gz -C $GITHUB_WORKSPACE/ccache . | |
| aws s3 cp --no-progress ./cache-arm.tar.gz s3://tp-internal/proton/ | |
| mv $GITHUB_WORKSPACE/output/proton $GITHUB_WORKSPACE/$PROTON_BINARY | |
| secrets: | |
| AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} | |
| AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} | |
| AWS_REGION: ${{ secrets.AWS_REGION }} | |
| AWS_S3_BUCKET: ${{ secrets.AWS_S3_BUCKET }} | |
| GH_PERSONAL_ACCESS_TOKEN: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }} | |
| DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} | |
| DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} | |
| CommitTag: | |
| needs: [Build_Linux_X86_64, Build_Linux_Arm64] | |
| uses: timeplus-io/proton/.github/workflows/run_command.yml@develop | |
| with: | |
| ec2-instance-type: c5.4xlarge | |
| ec2-image-id: ami-042a37e33a285c22b | |
| submodules: 'recursive' | |
| command: | | |
| cd $GITHUB_WORKSPACE | |
| # git config | |
| git config user.name "proton-robot" | |
| git config user.email "[email protected]" | |
| docker pull ghcr.io/timeplus-io/proton:$GITHUB_SHA-amd64 | |
| docker pull ghcr.io/timeplus-io/proton:$GITHUB_SHA-arm64v8 | |
| docker manifest create ghcr.io/timeplus-io/proton:latest \ | |
| ghcr.io/timeplus-io/proton:$GITHUB_SHA-amd64 \ | |
| ghcr.io/timeplus-io/proton:$GITHUB_SHA-arm64v8 | |
| docker manifest push ghcr.io/timeplus-io/proton:latest | |
| PROTON_TAG="$(docker history --no-trunc ghcr.io/timeplus-io/proton:$GITHUB_SHA-amd64 | grep -Eo 'ARG version=[0-9]+.[0-9]+.[0-9]+' -o | grep -Eo '[0-9]+.[0-9]+.[0-9]+')" | |
| docker manifest create ghcr.io/timeplus-io/proton:$PROTON_TAG-rc \ | |
| ghcr.io/timeplus-io/proton:$GITHUB_SHA-amd64 \ | |
| ghcr.io/timeplus-io/proton:$GITHUB_SHA-arm64v8 | |
| docker manifest push ghcr.io/timeplus-io/proton:$PROTON_TAG-rc | |
| curl -L \ | |
| -X POST \ | |
| -H "Accept: application/vnd.github+json" \ | |
| -H "Authorization: token $GH_PERSONAL_ACCESS_TOKEN"\ | |
| -H "X-GitHub-Api-Version: 2022-11-28" \ | |
| https://api.github.com/repos/timeplus-io/proton/actions/workflows/manual_trigger_test.yml/dispatches \ | |
| -d "{\"ref\":\"develop\",\"inputs\":{\"arch\": \"x64\", \"tag\":\"$PROTON_TAG-rc\"}}\"" | |
| curl -L \ | |
| -X POST \ | |
| -H "Accept: application/vnd.github+json" \ | |
| -H "Authorization: token $GH_PERSONAL_ACCESS_TOKEN"\ | |
| -H "X-GitHub-Api-Version: 2022-11-28" \ | |
| https://api.github.com/repos/timeplus-io/proton/actions/workflows/manual_trigger_test.yml/dispatches \ | |
| -d "{\"ref\":\"develop\",\"inputs\":{\"arch\": \"arm\", \"tag\":\"$PROTON_TAG-rc\"}}\"" | |
| secrets: | |
| AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} | |
| AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} | |
| AWS_REGION: ${{ secrets.AWS_REGION }} | |
| AWS_S3_BUCKET: ${{ secrets.AWS_S3_BUCKET }} | |
| GH_PERSONAL_ACCESS_TOKEN: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }} | |
| DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} | |
| DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} | |
| Build_Darwin_X86_64: | |
| uses: timeplus-io/proton/.github/workflows/run_command.yml@develop | |
| with: | |
| ec2-instance-type: c5.4xlarge | |
| ec2-image-id: ami-042a37e33a285c22b | |
| submodules: 'recursive' | |
| run_mode: 'start' # start ec2 on demand instance | |
| upload_files: proton-*-Darwin-x86_64 | |
| command: | | |
| cd $GITHUB_WORKSPACE | |
| # git config | |
| git config user.name "proton-robot" | |
| git config user.email "[email protected]" | |
| # update version | |
| ./release --version patch | |
| # download the pre-built binary of v8 (this is only for cross-compile) | |
| aws s3 cp --no-progress s3://tp-internal/proton/cross-compile-prebuilt-binary/v8-cmake-x64.tar.gz $GITHUB_WORKSPACE/contrib/v8-cmake/ | |
| tar -zxf $GITHUB_WORKSPACE/contrib/v8-cmake/v8-cmake-x64.tar.gz -C $GITHUB_WORKSPACE/contrib/v8-cmake/ | |
| rm $GITHUB_WORKSPACE/contrib/v8-cmake/v8-cmake-x64.tar.gz | |
| chmod a+x $GITHUB_WORKSPACE/contrib/v8-cmake/bytecode_builtins_list_generator | |
| chmod a+x $GITHUB_WORKSPACE/contrib/v8-cmake/mksnapshot | |
| chmod a+x $GITHUB_WORKSPACE/contrib/v8-cmake/torque | |
| # prepare build cache | |
| mkdir $GITHUB_WORKSPACE/ccache | |
| echo "max_size = 100.0G" > $GITHUB_WORKSPACE/ccache/ccache.conf | |
| # compiling | |
| ./docker/packager/packager --package-type binary --docker-image-version clang-16 --build-type release --proton-build --enable-proton-local --cache ccache --ccache_dir $GITHUB_WORKSPACE/ccache --output-dir $GITHUB_WORKSPACE/output --compiler clang-16-darwin | |
| if [ ! -f "$GITHUB_WORKSPACE/output/proton" ]; then | |
| echo "Compiling proton Failed" | |
| exit 127 | |
| fi | |
| # clean ccache and build footprint | |
| rm -rf $GITHUB_WORKSPACE/ccache | |
| rm -rf $GITHUB_WORKSPACE/build_docker | |
| # get proton tag | |
| PROTON_TAG=`grep "SET(VERSION_DESCRIBE" $GITHUB_WORKSPACE/cmake/autogenerated_versions.txt | sed 's/^.*VERSION_DESCRIBE \(.*\)$/\1/' | sed 's/[) ].*//'` | |
| echo "Proton tag: $PROTON_TAG" | |
| echo "tag_name=$PROTON_TAG" >> $GITHUB_OUTPUT | |
| PROTON_BINARY=proton-$PROTON_TAG-Darwin-x86_64 | |
| echo "Proton Binary Name: $PROTON_BINARY" | |
| sha256sum $GITHUB_WORKSPACE/output/proton | |
| mv $GITHUB_WORKSPACE/output/proton $GITHUB_WORKSPACE/$PROTON_BINARY | |
| secrets: | |
| AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} | |
| AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} | |
| AWS_REGION: ${{ secrets.AWS_REGION }} | |
| AWS_S3_BUCKET: ${{ secrets.AWS_S3_BUCKET }} | |
| GH_PERSONAL_ACCESS_TOKEN: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }} | |
| DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} | |
| DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} | |
| Build_Darwin_Arm64: | |
| uses: timeplus-io/proton/.github/workflows/run_command.yml@develop | |
| with: | |
| ec2-instance-type: c6g.4xlarge | |
| ec2-image-id: ami-0f3dbc4cc9994fdee | |
| submodules: 'recursive' | |
| run_mode: 'start' # start ec2 on demand instance | |
| upload_files: proton-*-Darwin-arm64 | |
| command: | | |
| cd $GITHUB_WORKSPACE | |
| # git config | |
| git config user.name "proton-robot" | |
| git config user.email "[email protected]" | |
| # update version | |
| ./release --version patch | |
| # download the pre-built binary of v8 (this is only for cross-compile) | |
| aws s3 cp --no-progress s3://tp-internal/proton/cross-compile-prebuilt-binary/v8-cmake-arm64.tar.gz $GITHUB_WORKSPACE/contrib/v8-cmake/ | |
| tar -zxf $GITHUB_WORKSPACE/contrib/v8-cmake/v8-cmake-arm64.tar.gz -C $GITHUB_WORKSPACE/contrib/v8-cmake/ | |
| rm $GITHUB_WORKSPACE/contrib/v8-cmake/v8-cmake-arm64.tar.gz | |
| chmod a+x $GITHUB_WORKSPACE/contrib/v8-cmake/bytecode_builtins_list_generator | |
| chmod a+x $GITHUB_WORKSPACE/contrib/v8-cmake/mksnapshot | |
| chmod a+x $GITHUB_WORKSPACE/contrib/v8-cmake/torque | |
| # prepare build cache | |
| mkdir $GITHUB_WORKSPACE/ccache | |
| echo "max_size = 100.0G" > $GITHUB_WORKSPACE/ccache/ccache.conf | |
| # compiling | |
| ./docker/packager/packager --package-type binary --docker-image-version clang-16 --build-type release --proton-build --enable-proton-local --cache ccache --ccache_dir $GITHUB_WORKSPACE/ccache --output-dir $GITHUB_WORKSPACE/output --compiler clang-16-darwin-aarch64 | |
| if [ ! -f "$GITHUB_WORKSPACE/output/proton" ]; then | |
| echo "Compiling proton Failed" | |
| exit 127 | |
| fi | |
| # clean ccache and build footprint | |
| rm -rf $GITHUB_WORKSPACE/ccache | |
| rm -rf $GITHUB_WORKSPACE/build_docker | |
| # get proton tag | |
| PROTON_TAG=`grep "SET(VERSION_DESCRIBE" $GITHUB_WORKSPACE/cmake/autogenerated_versions.txt | sed 's/^.*VERSION_DESCRIBE \(.*\)$/\1/' | sed 's/[) ].*//'` | |
| echo "Proton tag: $PROTON_TAG" | |
| echo "tag_name=$PROTON_TAG" >> $GITHUB_OUTPUT | |
| PROTON_BINARY=proton-$PROTON_TAG-Darwin-arm64 | |
| echo "Proton Binary Name: $PROTON_BINARY" | |
| sha256sum $GITHUB_WORKSPACE/output/proton | |
| mv $GITHUB_WORKSPACE/output/proton $GITHUB_WORKSPACE/$PROTON_BINARY | |
| secrets: | |
| AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} | |
| AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} | |
| AWS_REGION: ${{ secrets.AWS_REGION }} | |
| AWS_S3_BUCKET: ${{ secrets.AWS_S3_BUCKET }} | |
| GH_PERSONAL_ACCESS_TOKEN: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }} | |
| DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} | |
| DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} |