manual_trigger_test [x64] [1.5.15-rc] #193
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
name: manual_trigger_test | |
run-name: manual_trigger_test [${{ github.event.inputs.arch }}] [${{ github.event.inputs.tag }}] | |
on: # yamllint disable-line rule:truthy | |
workflow_dispatch: | |
inputs: | |
arch: | |
description: 'Hardware architecture' | |
required: true | |
default: x64 | |
type: choice | |
options: | |
- x64 | |
- arm | |
tag: | |
description: 'Docker image tag' | |
required: true | |
type: string | |
default: 'sanitizer' | |
has_smoke_test: | |
description: 'Run smoke test' | |
required: true | |
type: boolean | |
default: true | |
has_unit_test: | |
description: 'Run unit test' | |
required: true | |
type: boolean | |
default: false | |
has_stateless_test: | |
description: 'Run stateless test' | |
required: true | |
type: boolean | |
default: true | |
has_stateful_test: | |
description: 'Run stateful test' | |
required: true | |
type: boolean | |
default: true | |
has_query_compatibility_test: | |
description: 'Run query compatibility test' | |
required: true | |
type: boolean | |
default: true | |
jobs: | |
prepare_smoke_test: | |
runs-on: ubuntu-latest | |
if: ${{ github.event.inputs.has_smoke_test == 'true' }} | |
outputs: | |
command: | | |
# pull docker image | |
docker pull ghcr.io/timeplus-io/proton:$TAG | |
echo "run stream test" | |
cd $GITHUB_WORKSPACE/tests/stream | |
export PROTON_VERSION=$TAG | |
env | |
# make virtualenv | |
ln -s /usr/bin/python3 /usr/bin/python | |
apt-get update | |
systemctl stop unattended-upgrades | |
apt install python3-venv -y | |
python -m venv env | |
source env/bin/activate | |
pip install --upgrade pip | |
# FIXME: remove this line after pyyaml community fixed install bug | |
pip install pyyaml==5.3.1 | |
# FIXME(yokofly): docker 7.0.0 introduce a breaking change | |
# https://github.com/docker/docker-py/issues/3194 | |
pip install docker==6.1.3 | |
pip install -r helpers/requirements.txt | |
python ci_runner.py --settings=redp,nativelog | |
echo "check test result" | |
cat $GITHUB_WORKSPACE/tests/stream/.status | |
cat $GITHUB_WORKSPACE/tests/stream/.status | grep nativelog:ExitCode.OK | grep redp:ExitCode.OK | |
steps: | |
- name: display command | |
run: | | |
echo 'command: ${{ steps.set_command.outputs.command }}' | |
smoke_test_on_x64: | |
needs: prepare_smoke_test | |
uses: timeplus-io/proton/.github/workflows/run_command.yml@develop | |
if: github.event.inputs.arch == 'x64' | |
with: | |
ec2-instance-type: ${{ vars.X64_INSTANCE_TYPE }} | |
ec2-image-id: ${{ vars.X64_TEST_AMI }} | |
ec2-volume-size: '60' | |
submodules: false | |
sanitizer: '' | |
arch: ${{ vars.X64_ARCH }} | |
tag: ${{ github.event.inputs.tag }} | |
command: | | |
${{ needs.prepare_smoke_test.outputs.command }} | |
secrets: | |
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} | |
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} | |
AWS_REGION: ${{ secrets.AWS_REGION }} | |
AWS_S3_BUCKET: ${{ secrets.AWS_S3_BUCKET }} | |
GH_PERSONAL_ACCESS_TOKEN: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }} | |
DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} | |
DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} | |
TIMEPLUS_ADDRESS: ${{ secrets.TIMEPLUS_ADDRESS }} | |
TIMEPLUS_API_KEY: ${{ secrets.TIMEPLUS_API_KEY }} | |
TIMEPLUS_WORKSPACE: ${{ secrets.TIMEPLUS_WORKSPACE }} | |
smoke_test_on_arm: | |
needs: prepare_smoke_test | |
uses: timeplus-io/proton/.github/workflows/run_command.yml@develop | |
if: github.event.inputs.arch == 'arm' | |
with: | |
ec2-instance-type: ${{ vars.ARM_INSTANCE_TYPE }} | |
ec2-image-id: ${{ vars.ARM_TEST_AMI }} | |
ec2-volume-size: '50' | |
submodules: false | |
arch: ${{ vars.ARM_ARCH }} | |
sanitizer: '' | |
tag: ${{ github.event.inputs.tag }} | |
command: | | |
${{ needs.prepare_smoke_test.outputs.command }} | |
secrets: | |
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} | |
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} | |
AWS_REGION: ${{ secrets.AWS_REGION }} | |
AWS_S3_BUCKET: ${{ secrets.AWS_S3_BUCKET }} | |
GH_PERSONAL_ACCESS_TOKEN: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }} | |
DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} | |
DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} | |
TIMEPLUS_ADDRESS: ${{ secrets.TIMEPLUS_ADDRESS }} | |
TIMEPLUS_API_KEY: ${{ secrets.TIMEPLUS_API_KEY }} | |
TIMEPLUS_WORKSPACE: ${{ secrets.TIMEPLUS_WORKSPACE }} | |
prepare_query_compability_test: | |
runs-on: ubuntu-latest | |
if: ${{ github.event.inputs.has_query_compatibility_test == 'true' }} | |
outputs: | |
command: | | |
# pull docker image | |
docker pull ghcr.io/timeplus-io/proton:$TAG | |
echo "run quary compatibility test" | |
# cd $GITHUB_WORKSPACE/tests/stream | |
mkdir -p output | |
cd quark/api | |
export PROTON_VERSION=$TAG | |
env | |
# make virtualenv | |
ln -s /usr/bin/python3 /usr/bin/python | |
apt-get update | |
systemctl stop unattended-upgrades | |
apt install python3-venv -y | |
python -m venv env | |
source env/bin/activate | |
pip install --upgrade pip | |
# FIXME: remove this line after pyyaml community fixed install bug | |
pip install pyyaml==5.3.1 | |
# FIXME(yokofly): docker 7.0.0 introduce a breaking change | |
# https://github.com/docker/docker-py/issues/3194 | |
pip install docker==6.1.3 | |
pip install -r helpers/requirements.txt | |
aws s3 cp s3://tp-internal/proton/proton-python-driver/clickhouse-driver-0.2.4.tar.gz ./ | |
pip install ./clickhouse-driver-0.2.4.tar.gz | |
#python ci_runner.py --debug --settings=default --test_folders=test_production_compatibility | |
docker-compose -f test_production_compatibility/configs/docker/docker-compose.yaml up -d | |
sleep 10 | |
bash ./api_test_run_proton.sh | |
#echo "check test result" | |
#cat $GITHUB_WORKSPACE/tests/stream/.status | |
#cat $GITHUB_WORKSPACE/tests/stream/.status | grep default:ExitCode.OK | |
steps: | |
- name: display command | |
run: | | |
echo 'command: ${{ steps.set_command.outputs.command }}' | |
query_compability_test_on_x64: | |
needs: prepare_query_compability_test | |
uses: timeplus-io/proton/.github/workflows/run_command.yml@develop | |
if: github.event.inputs.arch == 'x64' | |
with: | |
ec2-instance-type: ${{ vars.X64_TEST_ONLY_INSTANCE_TYPE_SMALL }} | |
ec2-image-id: ${{ vars.X64_TEST_AMI }} | |
ec2-volume-size: '30' | |
submodules: false | |
sanitizer: '' | |
arch: ${{ vars.X64_ARCH }} | |
tag: ${{ github.event.inputs.tag }} | |
command: | | |
${{ needs.prepare_query_compability_test.outputs.command }} | |
secrets: | |
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} | |
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} | |
AWS_REGION: ${{ secrets.AWS_REGION }} | |
AWS_S3_BUCKET: ${{ secrets.AWS_S3_BUCKET }} | |
GH_PERSONAL_ACCESS_TOKEN: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }} | |
DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} | |
DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} | |
TIMEPLUS_ADDRESS: ${{ secrets.TIMEPLUS_ADDRESS }} | |
TIMEPLUS_API_KEY: ${{ secrets.TIMEPLUS_API_KEY }} | |
TIMEPLUS_WORKSPACE: ${{ secrets.TIMEPLUS_WORKSPACE }} | |
TP_DEMO_API_KEY: ${{ secrets.TP_DEMO_API_KEY }} | |
TP_DEMO_SITE_API_KEY: ${{ secrets.TP_DEMO_SITE_API_KEY }} | |
query_compability_on_arm: | |
needs: prepare_query_compability_test | |
uses: timeplus-io/proton/.github/workflows/run_command.yml@develop | |
if: github.event.inputs.arch == 'arm' | |
with: | |
ec2-instance-type: ${{ vars.ARM_TEST_ONLY_INSTANCE_TYPE_SMALL }} | |
ec2-image-id: ${{ vars.ARM_TEST_AMI }} | |
ec2-volume-size: '30' | |
submodules: false | |
arch: ${{ vars.ARM_ARCH }} | |
sanitizer: '' | |
tag: ${{ github.event.inputs.tag }} | |
command: | | |
${{ needs.prepare_query_compability_test.outputs.command }} | |
secrets: | |
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} | |
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} | |
AWS_REGION: ${{ secrets.AWS_REGION }} | |
AWS_S3_BUCKET: ${{ secrets.AWS_S3_BUCKET }} | |
GH_PERSONAL_ACCESS_TOKEN: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }} | |
DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} | |
DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} | |
TIMEPLUS_ADDRESS: ${{ secrets.TIMEPLUS_ADDRESS }} | |
TIMEPLUS_API_KEY: ${{ secrets.TIMEPLUS_API_KEY }} | |
TIMEPLUS_WORKSPACE: ${{ secrets.TIMEPLUS_WORKSPACE }} | |
TP_DEMO_API_KEY: ${{ secrets.TP_DEMO_API_KEY }} | |
TP_DEMO_SITE_API_KEY: ${{ secrets.TP_DEMO_SITE_API_KEY }} | |
prepare_unit_test: | |
runs-on: ubuntu-latest | |
if: ${{ github.event.inputs.has_unit_test == 'true' }} | |
outputs: | |
command: | | |
cd $GITHUB_WORKSPACE/tests/proton_ci | |
# make virtualenv | |
ln -s /usr/bin/python3 /usr/bin/python | |
apt-get update | |
systemctl stop unattended-upgrades | |
apt install python3-venv -y | |
python -m venv env | |
source env/bin/activate | |
pip install -r requirements.txt | |
export PROTON_VERSION=$TAG | |
python unit_tests_check.py | |
steps: | |
- name: display command | |
run: | | |
echo 'command: ${{ steps.set_command.outputs.command }}' | |
unit_test_on_x64: | |
needs: prepare_unit_test | |
uses: timeplus-io/proton/.github/workflows/run_command.yml@develop | |
if: github.event.inputs.arch == 'x64' | |
with: | |
ec2-instance-type: ${{ vars.X64_INSTANCE_TYPE }} | |
ec2-image-id: ${{ vars.X64_TEST_AMI }} | |
ec2-volume-size: '30' | |
submodules: false | |
sanitizer: '' | |
arch: ${{ vars.X64_ARCH }} | |
tag: ${{ github.event.inputs.tag }} | |
command: | | |
${{ needs.prepare_unit_test.outputs.command }} | |
secrets: | |
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} | |
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} | |
AWS_REGION: ${{ secrets.AWS_REGION }} | |
AWS_S3_BUCKET: ${{ secrets.AWS_S3_BUCKET }} | |
GH_PERSONAL_ACCESS_TOKEN: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }} | |
DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} | |
DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} | |
TIMEPLUS_ADDRESS: ${{ secrets.TIMEPLUS_ADDRESS }} | |
TIMEPLUS_API_KEY: ${{ secrets.TIMEPLUS_API_KEY }} | |
TIMEPLUS_WORKSPACE: ${{ secrets.TIMEPLUS_WORKSPACE }} | |
unit_test_on_arm: | |
needs: prepare_unit_test | |
uses: timeplus-io/proton/.github/workflows/run_command.yml@develop | |
if: github.event.inputs.arch == 'arm' | |
with: | |
ec2-instance-type: ${{ vars.ARM_INSTANCE_TYPE }} | |
ec2-image-id: ${{ vars.ARM_TEST_AMI }} | |
ec2-volume-size: '30' | |
submodules: false | |
arch: ${{ vars.ARM_ARCH }} | |
sanitizer: '' | |
tag: ${{ github.event.inputs.tag }} | |
command: | | |
${{ needs.prepare_unit_test.outputs.command }} | |
secrets: | |
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} | |
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} | |
AWS_REGION: ${{ secrets.AWS_REGION }} | |
AWS_S3_BUCKET: ${{ secrets.AWS_S3_BUCKET }} | |
GH_PERSONAL_ACCESS_TOKEN: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }} | |
DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} | |
DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} | |
TIMEPLUS_ADDRESS: ${{ secrets.TIMEPLUS_ADDRESS }} | |
TIMEPLUS_API_KEY: ${{ secrets.TIMEPLUS_API_KEY }} | |
TIMEPLUS_WORKSPACE: ${{ secrets.TIMEPLUS_WORKSPACE }} | |
prepare_stateless_test: | |
runs-on: ubuntu-latest | |
if: ${{ github.event.inputs.has_stateless_test == 'true' }} | |
outputs: | |
command: | | |
cd $GITHUB_WORKSPACE/tests/proton_ci | |
# make virtualenv | |
ln -s /usr/bin/python3 /usr/bin/python | |
apt-get update | |
systemctl stop unattended-upgrades | |
apt install python3-venv -y | |
python -m venv env | |
source env/bin/activate | |
pip install -r requirements.txt | |
export PROTON_VERSION=$TAG | |
python functional_tests_check.py stateless | |
steps: | |
- name: display command | |
run: | | |
echo 'command: ${{ steps.set_command.outputs.command }}' | |
stateless_test_on_x64: | |
needs: prepare_stateless_test | |
uses: timeplus-io/proton/.github/workflows/run_command.yml@develop | |
if: github.event.inputs.arch == 'x64' | |
with: | |
ec2-instance-type: ${{ vars.X64_INSTANCE_TYPE }} | |
ec2-image-id: ${{ vars.X64_TEST_AMI }} | |
ec2-volume-size: '40' | |
submodules: false | |
sanitizer: '' | |
arch: ${{ vars.X64_ARCH }} | |
tag: ${{ github.event.inputs.tag }} | |
command: | | |
${{ needs.prepare_stateless_test.outputs.command }} | |
secrets: | |
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} | |
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} | |
AWS_REGION: ${{ secrets.AWS_REGION }} | |
AWS_S3_BUCKET: ${{ secrets.AWS_S3_BUCKET }} | |
GH_PERSONAL_ACCESS_TOKEN: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }} | |
DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} | |
DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} | |
TIMEPLUS_ADDRESS: ${{ secrets.TIMEPLUS_ADDRESS }} | |
TIMEPLUS_API_KEY: ${{ secrets.TIMEPLUS_API_KEY }} | |
TIMEPLUS_WORKSPACE: ${{ secrets.TIMEPLUS_WORKSPACE }} | |
stateless_test_on_arm: | |
needs: prepare_stateless_test | |
uses: timeplus-io/proton/.github/workflows/run_command.yml@develop | |
if: github.event.inputs.arch == 'arm' | |
with: | |
ec2-instance-type: ${{ vars.ARM_INSTANCE_TYPE }} | |
ec2-image-id: ${{ vars.ARM_TEST_AMI }} | |
ec2-volume-size: '40' | |
submodules: false | |
arch: ${{ vars.ARM_ARCH }} | |
sanitizer: '' | |
tag: ${{ github.event.inputs.tag }} | |
command: | | |
${{ needs.prepare_stateless_test.outputs.command }} | |
secrets: | |
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} | |
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} | |
AWS_REGION: ${{ secrets.AWS_REGION }} | |
AWS_S3_BUCKET: ${{ secrets.AWS_S3_BUCKET }} | |
GH_PERSONAL_ACCESS_TOKEN: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }} | |
DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} | |
DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} | |
TIMEPLUS_ADDRESS: ${{ secrets.TIMEPLUS_ADDRESS }} | |
TIMEPLUS_API_KEY: ${{ secrets.TIMEPLUS_API_KEY }} | |
TIMEPLUS_WORKSPACE: ${{ secrets.TIMEPLUS_WORKSPACE }} | |
prepare_stateful_test: | |
runs-on: ubuntu-latest | |
if: ${{ github.event.inputs.has_stateful_test == 'true' }} | |
outputs: | |
command: | | |
cd $GITHUB_WORKSPACE/tests/proton_ci | |
# make virtualenv | |
ln -s /usr/bin/python3 /usr/bin/python | |
apt-get update | |
systemctl stop unattended-upgrades | |
apt install python3-venv -y | |
python -m venv env | |
source env/bin/activate | |
pip install -r requirements.txt | |
export PROTON_VERSION=$TAG | |
python functional_tests_check.py stateful | |
steps: | |
- name: display command | |
run: | | |
echo 'command: ${{ steps.set_command.outputs.command }}' | |
stateful_test_on_x64: | |
needs: prepare_stateful_test | |
uses: timeplus-io/proton/.github/workflows/run_command.yml@develop | |
if: github.event.inputs.arch == 'x64' | |
with: | |
ec2-instance-type: ${{ vars.X64_INSTANCE_TYPE }} | |
ec2-image-id: ${{ vars.X64_TEST_AMI }} | |
ec2-volume-size: '40' | |
submodules: false | |
sanitizer: '' | |
arch: ${{ vars.X64_ARCH }} | |
tag: ${{ github.event.inputs.tag }} | |
command: | | |
${{ needs.prepare_stateful_test.outputs.command }} | |
secrets: | |
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} | |
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} | |
AWS_REGION: ${{ secrets.AWS_REGION }} | |
AWS_S3_BUCKET: ${{ secrets.AWS_S3_BUCKET }} | |
GH_PERSONAL_ACCESS_TOKEN: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }} | |
DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} | |
DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} | |
TIMEPLUS_ADDRESS: ${{ secrets.TIMEPLUS_ADDRESS }} | |
TIMEPLUS_API_KEY: ${{ secrets.TIMEPLUS_API_KEY }} | |
TIMEPLUS_WORKSPACE: ${{ secrets.TIMEPLUS_WORKSPACE }} | |
stateful_test_on_arm: | |
needs: prepare_stateful_test | |
uses: timeplus-io/proton/.github/workflows/run_command.yml@develop | |
if: github.event.inputs.arch == 'arm' | |
with: | |
ec2-instance-type: ${{ vars.ARM_INSTANCE_TYPE }} | |
ec2-image-id: ${{ vars.ARM_TEST_AMI }} | |
ec2-volume-size: '40' | |
submodules: false | |
arch: ${{ vars.ARM_ARCH }} | |
sanitizer: '' | |
tag: ${{ github.event.inputs.tag }} | |
command: | | |
${{ needs.prepare_stateful_test.outputs.command }} | |
secrets: | |
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} | |
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} | |
AWS_REGION: ${{ secrets.AWS_REGION }} | |
AWS_S3_BUCKET: ${{ secrets.AWS_S3_BUCKET }} | |
GH_PERSONAL_ACCESS_TOKEN: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }} | |
DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} | |
DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} | |
TIMEPLUS_ADDRESS: ${{ secrets.TIMEPLUS_ADDRESS }} | |
TIMEPLUS_API_KEY: ${{ secrets.TIMEPLUS_API_KEY }} | |
TIMEPLUS_WORKSPACE: ${{ secrets.TIMEPLUS_WORKSPACE }} |