Skip to content

manual_trigger_test [x64] [1.5.15-rc] #193

manual_trigger_test [x64] [1.5.15-rc]

manual_trigger_test [x64] [1.5.15-rc] #193

name: manual_trigger_test
run-name: manual_trigger_test [${{ github.event.inputs.arch }}] [${{ github.event.inputs.tag }}]
on: # yamllint disable-line rule:truthy
workflow_dispatch:
inputs:
arch:
description: 'Hardware architecture'
required: true
default: x64
type: choice
options:
- x64
- arm
tag:
description: 'Docker image tag'
required: true
type: string
default: 'sanitizer'
has_smoke_test:
description: 'Run smoke test'
required: true
type: boolean
default: true
has_unit_test:
description: 'Run unit test'
required: true
type: boolean
default: false
has_stateless_test:
description: 'Run stateless test'
required: true
type: boolean
default: true
has_stateful_test:
description: 'Run stateful test'
required: true
type: boolean
default: true
has_query_compatibility_test:
description: 'Run query compatibility test'
required: true
type: boolean
default: true
jobs:
prepare_smoke_test:
runs-on: ubuntu-latest
if: ${{ github.event.inputs.has_smoke_test == 'true' }}
outputs:
command: |
# pull docker image
docker pull ghcr.io/timeplus-io/proton:$TAG
echo "run stream test"
cd $GITHUB_WORKSPACE/tests/stream
export PROTON_VERSION=$TAG
env
# make virtualenv
ln -s /usr/bin/python3 /usr/bin/python
apt-get update
systemctl stop unattended-upgrades
apt install python3-venv -y
python -m venv env
source env/bin/activate
pip install --upgrade pip
# FIXME: remove this line after pyyaml community fixed install bug
pip install pyyaml==5.3.1
# FIXME(yokofly): docker 7.0.0 introduce a breaking change
# https://github.com/docker/docker-py/issues/3194
pip install docker==6.1.3
pip install -r helpers/requirements.txt
python ci_runner.py --settings=redp,nativelog
echo "check test result"
cat $GITHUB_WORKSPACE/tests/stream/.status
cat $GITHUB_WORKSPACE/tests/stream/.status | grep nativelog:ExitCode.OK | grep redp:ExitCode.OK
steps:
- name: display command
run: |
echo 'command: ${{ steps.set_command.outputs.command }}'
smoke_test_on_x64:
needs: prepare_smoke_test
uses: timeplus-io/proton/.github/workflows/run_command.yml@develop
if: github.event.inputs.arch == 'x64'
with:
ec2-instance-type: ${{ vars.X64_INSTANCE_TYPE }}
ec2-image-id: ${{ vars.X64_TEST_AMI }}
ec2-volume-size: '60'
submodules: false
sanitizer: ''
arch: ${{ vars.X64_ARCH }}
tag: ${{ github.event.inputs.tag }}
command: |
${{ needs.prepare_smoke_test.outputs.command }}
secrets:
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
AWS_REGION: ${{ secrets.AWS_REGION }}
AWS_S3_BUCKET: ${{ secrets.AWS_S3_BUCKET }}
GH_PERSONAL_ACCESS_TOKEN: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }}
DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }}
DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }}
TIMEPLUS_ADDRESS: ${{ secrets.TIMEPLUS_ADDRESS }}
TIMEPLUS_API_KEY: ${{ secrets.TIMEPLUS_API_KEY }}
TIMEPLUS_WORKSPACE: ${{ secrets.TIMEPLUS_WORKSPACE }}
smoke_test_on_arm:
needs: prepare_smoke_test
uses: timeplus-io/proton/.github/workflows/run_command.yml@develop
if: github.event.inputs.arch == 'arm'
with:
ec2-instance-type: ${{ vars.ARM_INSTANCE_TYPE }}
ec2-image-id: ${{ vars.ARM_TEST_AMI }}
ec2-volume-size: '50'
submodules: false
arch: ${{ vars.ARM_ARCH }}
sanitizer: ''
tag: ${{ github.event.inputs.tag }}
command: |
${{ needs.prepare_smoke_test.outputs.command }}
secrets:
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
AWS_REGION: ${{ secrets.AWS_REGION }}
AWS_S3_BUCKET: ${{ secrets.AWS_S3_BUCKET }}
GH_PERSONAL_ACCESS_TOKEN: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }}
DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }}
DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }}
TIMEPLUS_ADDRESS: ${{ secrets.TIMEPLUS_ADDRESS }}
TIMEPLUS_API_KEY: ${{ secrets.TIMEPLUS_API_KEY }}
TIMEPLUS_WORKSPACE: ${{ secrets.TIMEPLUS_WORKSPACE }}
prepare_query_compability_test:
runs-on: ubuntu-latest
if: ${{ github.event.inputs.has_query_compatibility_test == 'true' }}
outputs:
command: |
# pull docker image
docker pull ghcr.io/timeplus-io/proton:$TAG
echo "run quary compatibility test"
# cd $GITHUB_WORKSPACE/tests/stream
mkdir -p output
cd quark/api
export PROTON_VERSION=$TAG
env
# make virtualenv
ln -s /usr/bin/python3 /usr/bin/python
apt-get update
systemctl stop unattended-upgrades
apt install python3-venv -y
python -m venv env
source env/bin/activate
pip install --upgrade pip
# FIXME: remove this line after pyyaml community fixed install bug
pip install pyyaml==5.3.1
# FIXME(yokofly): docker 7.0.0 introduce a breaking change
# https://github.com/docker/docker-py/issues/3194
pip install docker==6.1.3
pip install -r helpers/requirements.txt
aws s3 cp s3://tp-internal/proton/proton-python-driver/clickhouse-driver-0.2.4.tar.gz ./
pip install ./clickhouse-driver-0.2.4.tar.gz
#python ci_runner.py --debug --settings=default --test_folders=test_production_compatibility
docker-compose -f test_production_compatibility/configs/docker/docker-compose.yaml up -d
sleep 10
bash ./api_test_run_proton.sh
#echo "check test result"
#cat $GITHUB_WORKSPACE/tests/stream/.status
#cat $GITHUB_WORKSPACE/tests/stream/.status | grep default:ExitCode.OK
steps:
- name: display command
run: |
echo 'command: ${{ steps.set_command.outputs.command }}'
query_compability_test_on_x64:
needs: prepare_query_compability_test
uses: timeplus-io/proton/.github/workflows/run_command.yml@develop
if: github.event.inputs.arch == 'x64'
with:
ec2-instance-type: ${{ vars.X64_TEST_ONLY_INSTANCE_TYPE_SMALL }}
ec2-image-id: ${{ vars.X64_TEST_AMI }}
ec2-volume-size: '30'
submodules: false
sanitizer: ''
arch: ${{ vars.X64_ARCH }}
tag: ${{ github.event.inputs.tag }}
command: |
${{ needs.prepare_query_compability_test.outputs.command }}
secrets:
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
AWS_REGION: ${{ secrets.AWS_REGION }}
AWS_S3_BUCKET: ${{ secrets.AWS_S3_BUCKET }}
GH_PERSONAL_ACCESS_TOKEN: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }}
DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }}
DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }}
TIMEPLUS_ADDRESS: ${{ secrets.TIMEPLUS_ADDRESS }}
TIMEPLUS_API_KEY: ${{ secrets.TIMEPLUS_API_KEY }}
TIMEPLUS_WORKSPACE: ${{ secrets.TIMEPLUS_WORKSPACE }}
TP_DEMO_API_KEY: ${{ secrets.TP_DEMO_API_KEY }}
TP_DEMO_SITE_API_KEY: ${{ secrets.TP_DEMO_SITE_API_KEY }}
query_compability_on_arm:
needs: prepare_query_compability_test
uses: timeplus-io/proton/.github/workflows/run_command.yml@develop
if: github.event.inputs.arch == 'arm'
with:
ec2-instance-type: ${{ vars.ARM_TEST_ONLY_INSTANCE_TYPE_SMALL }}
ec2-image-id: ${{ vars.ARM_TEST_AMI }}
ec2-volume-size: '30'
submodules: false
arch: ${{ vars.ARM_ARCH }}
sanitizer: ''
tag: ${{ github.event.inputs.tag }}
command: |
${{ needs.prepare_query_compability_test.outputs.command }}
secrets:
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
AWS_REGION: ${{ secrets.AWS_REGION }}
AWS_S3_BUCKET: ${{ secrets.AWS_S3_BUCKET }}
GH_PERSONAL_ACCESS_TOKEN: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }}
DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }}
DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }}
TIMEPLUS_ADDRESS: ${{ secrets.TIMEPLUS_ADDRESS }}
TIMEPLUS_API_KEY: ${{ secrets.TIMEPLUS_API_KEY }}
TIMEPLUS_WORKSPACE: ${{ secrets.TIMEPLUS_WORKSPACE }}
TP_DEMO_API_KEY: ${{ secrets.TP_DEMO_API_KEY }}
TP_DEMO_SITE_API_KEY: ${{ secrets.TP_DEMO_SITE_API_KEY }}
prepare_unit_test:
runs-on: ubuntu-latest
if: ${{ github.event.inputs.has_unit_test == 'true' }}
outputs:
command: |
cd $GITHUB_WORKSPACE/tests/proton_ci
# make virtualenv
ln -s /usr/bin/python3 /usr/bin/python
apt-get update
systemctl stop unattended-upgrades
apt install python3-venv -y
python -m venv env
source env/bin/activate
pip install -r requirements.txt
export PROTON_VERSION=$TAG
python unit_tests_check.py
steps:
- name: display command
run: |
echo 'command: ${{ steps.set_command.outputs.command }}'
unit_test_on_x64:
needs: prepare_unit_test
uses: timeplus-io/proton/.github/workflows/run_command.yml@develop
if: github.event.inputs.arch == 'x64'
with:
ec2-instance-type: ${{ vars.X64_INSTANCE_TYPE }}
ec2-image-id: ${{ vars.X64_TEST_AMI }}
ec2-volume-size: '30'
submodules: false
sanitizer: ''
arch: ${{ vars.X64_ARCH }}
tag: ${{ github.event.inputs.tag }}
command: |
${{ needs.prepare_unit_test.outputs.command }}
secrets:
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
AWS_REGION: ${{ secrets.AWS_REGION }}
AWS_S3_BUCKET: ${{ secrets.AWS_S3_BUCKET }}
GH_PERSONAL_ACCESS_TOKEN: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }}
DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }}
DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }}
TIMEPLUS_ADDRESS: ${{ secrets.TIMEPLUS_ADDRESS }}
TIMEPLUS_API_KEY: ${{ secrets.TIMEPLUS_API_KEY }}
TIMEPLUS_WORKSPACE: ${{ secrets.TIMEPLUS_WORKSPACE }}
unit_test_on_arm:
needs: prepare_unit_test
uses: timeplus-io/proton/.github/workflows/run_command.yml@develop
if: github.event.inputs.arch == 'arm'
with:
ec2-instance-type: ${{ vars.ARM_INSTANCE_TYPE }}
ec2-image-id: ${{ vars.ARM_TEST_AMI }}
ec2-volume-size: '30'
submodules: false
arch: ${{ vars.ARM_ARCH }}
sanitizer: ''
tag: ${{ github.event.inputs.tag }}
command: |
${{ needs.prepare_unit_test.outputs.command }}
secrets:
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
AWS_REGION: ${{ secrets.AWS_REGION }}
AWS_S3_BUCKET: ${{ secrets.AWS_S3_BUCKET }}
GH_PERSONAL_ACCESS_TOKEN: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }}
DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }}
DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }}
TIMEPLUS_ADDRESS: ${{ secrets.TIMEPLUS_ADDRESS }}
TIMEPLUS_API_KEY: ${{ secrets.TIMEPLUS_API_KEY }}
TIMEPLUS_WORKSPACE: ${{ secrets.TIMEPLUS_WORKSPACE }}
prepare_stateless_test:
runs-on: ubuntu-latest
if: ${{ github.event.inputs.has_stateless_test == 'true' }}
outputs:
command: |
cd $GITHUB_WORKSPACE/tests/proton_ci
# make virtualenv
ln -s /usr/bin/python3 /usr/bin/python
apt-get update
systemctl stop unattended-upgrades
apt install python3-venv -y
python -m venv env
source env/bin/activate
pip install -r requirements.txt
export PROTON_VERSION=$TAG
python functional_tests_check.py stateless
steps:
- name: display command
run: |
echo 'command: ${{ steps.set_command.outputs.command }}'
stateless_test_on_x64:
needs: prepare_stateless_test
uses: timeplus-io/proton/.github/workflows/run_command.yml@develop
if: github.event.inputs.arch == 'x64'
with:
ec2-instance-type: ${{ vars.X64_INSTANCE_TYPE }}
ec2-image-id: ${{ vars.X64_TEST_AMI }}
ec2-volume-size: '40'
submodules: false
sanitizer: ''
arch: ${{ vars.X64_ARCH }}
tag: ${{ github.event.inputs.tag }}
command: |
${{ needs.prepare_stateless_test.outputs.command }}
secrets:
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
AWS_REGION: ${{ secrets.AWS_REGION }}
AWS_S3_BUCKET: ${{ secrets.AWS_S3_BUCKET }}
GH_PERSONAL_ACCESS_TOKEN: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }}
DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }}
DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }}
TIMEPLUS_ADDRESS: ${{ secrets.TIMEPLUS_ADDRESS }}
TIMEPLUS_API_KEY: ${{ secrets.TIMEPLUS_API_KEY }}
TIMEPLUS_WORKSPACE: ${{ secrets.TIMEPLUS_WORKSPACE }}
stateless_test_on_arm:
needs: prepare_stateless_test
uses: timeplus-io/proton/.github/workflows/run_command.yml@develop
if: github.event.inputs.arch == 'arm'
with:
ec2-instance-type: ${{ vars.ARM_INSTANCE_TYPE }}
ec2-image-id: ${{ vars.ARM_TEST_AMI }}
ec2-volume-size: '40'
submodules: false
arch: ${{ vars.ARM_ARCH }}
sanitizer: ''
tag: ${{ github.event.inputs.tag }}
command: |
${{ needs.prepare_stateless_test.outputs.command }}
secrets:
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
AWS_REGION: ${{ secrets.AWS_REGION }}
AWS_S3_BUCKET: ${{ secrets.AWS_S3_BUCKET }}
GH_PERSONAL_ACCESS_TOKEN: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }}
DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }}
DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }}
TIMEPLUS_ADDRESS: ${{ secrets.TIMEPLUS_ADDRESS }}
TIMEPLUS_API_KEY: ${{ secrets.TIMEPLUS_API_KEY }}
TIMEPLUS_WORKSPACE: ${{ secrets.TIMEPLUS_WORKSPACE }}
prepare_stateful_test:
runs-on: ubuntu-latest
if: ${{ github.event.inputs.has_stateful_test == 'true' }}
outputs:
command: |
cd $GITHUB_WORKSPACE/tests/proton_ci
# make virtualenv
ln -s /usr/bin/python3 /usr/bin/python
apt-get update
systemctl stop unattended-upgrades
apt install python3-venv -y
python -m venv env
source env/bin/activate
pip install -r requirements.txt
export PROTON_VERSION=$TAG
python functional_tests_check.py stateful
steps:
- name: display command
run: |
echo 'command: ${{ steps.set_command.outputs.command }}'
stateful_test_on_x64:
needs: prepare_stateful_test
uses: timeplus-io/proton/.github/workflows/run_command.yml@develop
if: github.event.inputs.arch == 'x64'
with:
ec2-instance-type: ${{ vars.X64_INSTANCE_TYPE }}
ec2-image-id: ${{ vars.X64_TEST_AMI }}
ec2-volume-size: '40'
submodules: false
sanitizer: ''
arch: ${{ vars.X64_ARCH }}
tag: ${{ github.event.inputs.tag }}
command: |
${{ needs.prepare_stateful_test.outputs.command }}
secrets:
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
AWS_REGION: ${{ secrets.AWS_REGION }}
AWS_S3_BUCKET: ${{ secrets.AWS_S3_BUCKET }}
GH_PERSONAL_ACCESS_TOKEN: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }}
DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }}
DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }}
TIMEPLUS_ADDRESS: ${{ secrets.TIMEPLUS_ADDRESS }}
TIMEPLUS_API_KEY: ${{ secrets.TIMEPLUS_API_KEY }}
TIMEPLUS_WORKSPACE: ${{ secrets.TIMEPLUS_WORKSPACE }}
stateful_test_on_arm:
needs: prepare_stateful_test
uses: timeplus-io/proton/.github/workflows/run_command.yml@develop
if: github.event.inputs.arch == 'arm'
with:
ec2-instance-type: ${{ vars.ARM_INSTANCE_TYPE }}
ec2-image-id: ${{ vars.ARM_TEST_AMI }}
ec2-volume-size: '40'
submodules: false
arch: ${{ vars.ARM_ARCH }}
sanitizer: ''
tag: ${{ github.event.inputs.tag }}
command: |
${{ needs.prepare_stateful_test.outputs.command }}
secrets:
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
AWS_REGION: ${{ secrets.AWS_REGION }}
AWS_S3_BUCKET: ${{ secrets.AWS_S3_BUCKET }}
GH_PERSONAL_ACCESS_TOKEN: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }}
DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }}
DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }}
TIMEPLUS_ADDRESS: ${{ secrets.TIMEPLUS_ADDRESS }}
TIMEPLUS_API_KEY: ${{ secrets.TIMEPLUS_API_KEY }}
TIMEPLUS_WORKSPACE: ${{ secrets.TIMEPLUS_WORKSPACE }}