Skip to content
Performance Test Suite

Performance Test #299

Sign in to view logs

Performance Test

Performance Test #299

Workflow file for this run

.github/workflows/performance_test.yaml at 86853ac
name: Performance Test Suite
on:
push:
branches:
- 'perf/**'
schedule:
- cron: '0 1 * * MON-FRI'
workflow_dispatch:
inputs:
ref:
required: false
env:
TF_AWS_REGION: eu-west-1
TF_VAR_s3_bucket_name: tf-emqx-performance-test2
TF_VAR_test_duration: 1800
TF_VAR_prometheus_remote_write_region: eu-west-1
TF_VAR_prometheus_remote_write_url: ${{ secrets.TF_EMQX_PERF_TEST_PROMETHEUS_REMOTE_WRITE_URL }}
SLACK_WEBHOOK_URL: ${{ secrets.TF_EMQX_PERF_TEST_SLACK_URL }}
permissions:
contents: read
jobs:
prepare:
runs-on: ubuntu-latest
if: github.repository_owner == 'emqx'
container: ghcr.io/emqx/emqx-builder/5.3-9:1.15.7-26.2.5-3-ubuntu20.04
outputs:
BENCH_ID: ${{ steps.prepare.outputs.BENCH_ID }}
PACKAGE_FILE: ${{ steps.package_file.outputs.PACKAGE_FILE }}
steps:
- uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
with:
fetch-depth: 0
ref: ${{ github.event.inputs.ref }}
- name: Work around https://github.com/actions/checkout/issues/766
run: |
git config --global --add safe.directory "$GITHUB_WORKSPACE"
- id: prepare
run: |
echo "EMQX_NAME=emqx" >> $GITHUB_ENV
echo "CODE_PATH=$GITHUB_WORKSPACE" >> $GITHUB_ENV
echo "BENCH_ID=$(date --utc +%F)/emqx-$(./pkg-vsn.sh emqx)" >> $GITHUB_OUTPUT
- name: Build deb package
run: |
make ${EMQX_NAME}-pkg
./scripts/pkg-tests.sh ${EMQX_NAME}-pkg
- name: Get package file name
id: package_file
run: |
echo "PACKAGE_FILE=$(find _packages/emqx -name 'emqx-*.deb' | head -n 1 | xargs basename)" >> $GITHUB_OUTPUT
- uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808 # v4.3.3
with:
name: emqx-ubuntu20.04
path: _packages/emqx/${{ steps.package_file.outputs.PACKAGE_FILE }}
scenario_1on1:
runs-on: ubuntu-latest
needs:
- prepare
env:
TF_VAR_package_file: ${{ needs.prepare.outputs.PACKAGE_FILE }}
steps:
- name: Configure AWS Credentials
uses: aws-actions/configure-aws-credentials@e3dd6a429d7300a6a4c196c26e071d42e0343502 # v4.0.2
with:
aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_PERF_TEST }}
aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY_PERF_TEST }}
aws-region: eu-west-1
- name: Checkout tf-emqx-performance-test
uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
with:
repository: emqx/tf-emqx-performance-test
path: tf-emqx-performance-test
ref: v0.2.3
- uses: actions/download-artifact@65a9edc5881444af0b9093a5e628f2fe47ea3b2e # v4.1.7
with:
name: emqx-ubuntu20.04
path: tf-emqx-performance-test/
- name: Setup Terraform
uses: hashicorp/setup-terraform@651471c36a6092792c552e8b1bef71e592b462d8 # v3.1.1
with:
terraform_wrapper: false
- name: run scenario
working-directory: ./tf-emqx-performance-test
timeout-minutes: 60
env:
TF_VAR_bench_id: "${{ needs.prepare.outputs.BENCH_ID }}/1on1"
TF_VAR_use_emqttb: 1
TF_VAR_use_emqtt_bench: 0
TF_VAR_emqttb_instance_count: 1
TF_VAR_emqttb_instance_type: c5.2xlarge
TF_VAR_emqttb_scenario: '@pubsub_fwd -n 50_000 --pub-qos 1 --sub-qos 1'
TF_VAR_emqx_instance_type: c5.2xlarge
TF_VAR_emqx_instance_count: 3
run: |
terraform init
terraform apply -auto-approve
./wait-emqttb.sh
./fetch-metrics.sh
terraform destroy -auto-approve
aws s3 sync --exclude '*' --include '*.tar.gz' s3://$TF_VAR_s3_bucket_name/$TF_VAR_bench_id .
- name: Send notification to Slack
uses: slackapi/slack-github-action@70cd7be8e40a46e8b0eced40b0de447bdb42f68e # v1.26.0
with:
payload-file-path: "./tf-emqx-performance-test/slack-payload.json"
- name: terraform destroy
if: always()
working-directory: ./tf-emqx-performance-test
run: |
terraform destroy -auto-approve
- uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808 # v4.3.3
if: success()
with:
name: metrics
path: |
"./tf-emqx-performance-test/*.tar.gz"
- uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808 # v4.3.3
if: failure()
with:
name: terraform
path: |
./tf-emqx-performance-test/.terraform
./tf-emqx-performance-test/*.tfstate
scenario_fanout:
runs-on: ubuntu-latest
needs:
- prepare
- scenario_1on1
env:
TF_VAR_package_file: ${{ needs.prepare.outputs.PACKAGE_FILE }}
steps:
- name: Configure AWS Credentials
uses: aws-actions/configure-aws-credentials@e3dd6a429d7300a6a4c196c26e071d42e0343502 # v4.0.2
with:
aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_PERF_TEST }}
aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY_PERF_TEST }}
aws-region: eu-west-1
- name: Checkout tf-emqx-performance-test
uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
with:
repository: emqx/tf-emqx-performance-test
path: tf-emqx-performance-test
ref: v0.2.3
- uses: actions/download-artifact@65a9edc5881444af0b9093a5e628f2fe47ea3b2e # v4.1.7
with:
name: emqx-ubuntu20.04
path: tf-emqx-performance-test/
- name: Setup Terraform
uses: hashicorp/setup-terraform@651471c36a6092792c552e8b1bef71e592b462d8 # v3.1.1
with:
terraform_wrapper: false
- name: run scenario
working-directory: ./tf-emqx-performance-test
timeout-minutes: 60
env:
TF_VAR_bench_id: "${{ needs.prepare.outputs.BENCH_ID }}/fan-out"
TF_VAR_use_emqttb: 1
TF_VAR_use_emqtt_bench: 0
TF_VAR_emqttb_instance_count: 1
TF_VAR_emqttb_instance_type: c5.2xlarge
TF_VAR_emqttb_scenario: '@pub --topic "t/%n" --conninterval 10ms --pubinterval 20ms --num-clients 5 --size 16 @sub --topic "t/#" --conninterval 10ms --num-clients 1000'
TF_VAR_emqx_instance_type: c5.large
TF_VAR_emqx_instance_count: 3
run: |
terraform init
terraform apply -auto-approve
./wait-emqttb.sh
./fetch-metrics.sh
terraform destroy -auto-approve
aws s3 sync --exclude '*' --include '*.tar.gz' s3://$TF_VAR_s3_bucket_name/$TF_VAR_bench_id .
- name: Send notification to Slack
uses: slackapi/slack-github-action@70cd7be8e40a46e8b0eced40b0de447bdb42f68e # v1.26.0
with:
payload-file-path: "./tf-emqx-performance-test/slack-payload.json"
- name: terraform destroy
if: always()
working-directory: ./tf-emqx-performance-test
run: |
terraform destroy -auto-approve
- uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808 # v4.3.3
if: success()
with:
name: metrics
path: |
"./tf-emqx-performance-test/*.tar.gz"
- uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808 # v4.3.3
if: failure()
with:
name: terraform
path: |
./tf-emqx-performance-test/.terraform
./tf-emqx-performance-test/*.tfstate
scenario_fanin:
runs-on: ubuntu-latest
needs:
- prepare
- scenario_1on1
- scenario_fanout
env:
TF_VAR_package_file: ${{ needs.prepare.outputs.PACKAGE_FILE }}
steps:
- name: Configure AWS Credentials
uses: aws-actions/configure-aws-credentials@e3dd6a429d7300a6a4c196c26e071d42e0343502 # v4.0.2
with:
aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_PERF_TEST }}
aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY_PERF_TEST }}
aws-region: eu-west-1
- name: Checkout tf-emqx-performance-test
uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
with:
repository: emqx/tf-emqx-performance-test
path: tf-emqx-performance-test
ref: v0.2.3
- uses: actions/download-artifact@65a9edc5881444af0b9093a5e628f2fe47ea3b2e # v4.1.7
with:
name: emqx-ubuntu20.04
path: tf-emqx-performance-test/
- name: Setup Terraform
uses: hashicorp/setup-terraform@651471c36a6092792c552e8b1bef71e592b462d8 # v3.1.1
with:
terraform_wrapper: false
- name: run scenario
working-directory: ./tf-emqx-performance-test
timeout-minutes: 60
env:
TF_VAR_bench_id: "${{ needs.prepare.outputs.BENCH_ID }}/fan-in"
TF_VAR_use_emqttb: 1
TF_VAR_use_emqtt_bench: 0
TF_VAR_emqttb_instance_count: 2
TF_VAR_emqttb_start_n_multiplier: 25000
TF_VAR_emqttb_instance_type: c5.xlarge
TF_VAR_emqttb_scenario: '@pub --topic t/%n --conninterval 10ms --pubinterval 1s --num-clients 25_000 --start-n $START_N --size 16 @sub --topic \$share/perf/t/# --conninterval 10ms --num-clients 250'
TF_VAR_emqx_instance_type: c5.2xlarge
TF_VAR_emqx_instance_count: 3
run: |
terraform init
terraform apply -auto-approve
./wait-emqttb.sh
./fetch-metrics.sh
terraform destroy -auto-approve
aws s3 sync --exclude '*' --include '*.tar.gz' s3://$TF_VAR_s3_bucket_name/$TF_VAR_bench_id .
- name: Send notification to Slack
uses: slackapi/slack-github-action@70cd7be8e40a46e8b0eced40b0de447bdb42f68e # v1.26.0
with:
payload-file-path: "./tf-emqx-performance-test/slack-payload.json"
- name: terraform destroy
if: always()
working-directory: ./tf-emqx-performance-test
run: |
terraform destroy -auto-approve
- uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808 # v4.3.3
if: success()
with:
name: metrics
path: |
"./tf-emqx-performance-test/*.tar.gz"
- uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808 # v4.3.3
if: failure()
with:
name: terraform
path: |
./tf-emqx-performance-test/.terraform
./tf-emqx-performance-test/*.tfstate
scenario_1m_conns:
runs-on: ubuntu-latest
needs:
- prepare
- scenario_fanin
- scenario_fanout
- scenario_1on1
env:
TF_VAR_package_file: ${{ needs.prepare.outputs.PACKAGE_FILE }}
steps:
- name: Configure AWS Credentials
uses: aws-actions/configure-aws-credentials@e3dd6a429d7300a6a4c196c26e071d42e0343502 # v4.0.2
with:
aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_PERF_TEST }}
aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY_PERF_TEST }}
aws-region: eu-west-1
- name: Checkout tf-emqx-performance-test
uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
with:
repository: emqx/tf-emqx-performance-test
path: tf-emqx-performance-test
ref: v0.2.3
- uses: actions/download-artifact@65a9edc5881444af0b9093a5e628f2fe47ea3b2e # v4.1.7
with:
name: emqx-ubuntu20.04
path: tf-emqx-performance-test/
- name: Setup Terraform
uses: hashicorp/setup-terraform@651471c36a6092792c552e8b1bef71e592b462d8 # v3.1.1
with:
terraform_wrapper: false
- name: run scenario
working-directory: ./tf-emqx-performance-test
timeout-minutes: 60
env:
TF_VAR_bench_id: "${{ needs.prepare.outputs.BENCH_ID }}/1m-connections"
TF_VAR_use_emqttb: 1
TF_VAR_use_emqtt_bench: 0
TF_VAR_emqttb_instance_count: 5
TF_VAR_emqttb_instance_type: c5.2xlarge
TF_VAR_emqttb_scenario: '@conn -N 200_000 --conninterval 1ms'
TF_VAR_emqx_instance_type: c5.2xlarge
TF_VAR_emqx_instance_count: 3
run: |
terraform init
terraform apply -auto-approve
./wait-emqttb.sh
./fetch-metrics.sh
terraform destroy -auto-approve
aws s3 sync --exclude '*' --include '*.tar.gz' s3://$TF_VAR_s3_bucket_name/$TF_VAR_bench_id .
- name: Send notification to Slack
uses: slackapi/slack-github-action@70cd7be8e40a46e8b0eced40b0de447bdb42f68e # v1.26.0
with:
payload-file-path: "./tf-emqx-performance-test/slack-payload.json"
- name: terraform destroy
if: always()
working-directory: ./tf-emqx-performance-test
run: |
terraform destroy -auto-approve
- uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808 # v4.3.3
if: success()
with:
name: metrics
path: |
"./tf-emqx-performance-test/*.tar.gz"
- uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808 # v4.3.3
if: failure()
with:
name: terraform
path: |
./tf-emqx-performance-test/.terraform
./tf-emqx-performance-test/*.tfstate