ci: add benchmarking #10
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Benchmark | |
| on: | |
| push: | |
| branches: | |
| - main | |
| - test-benchmark | |
| paths-ignore: | |
| - '**.md' | |
| - 'contrib/**' | |
| permissions: | |
| # deployments permission to deploy GitHub pages website | |
| deployments: write | |
| # contents permission to update benchmark contents in gh-pages branch | |
| contents: write | |
| jobs: | |
| benchmark-for-awslogs: | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| os: [ ubuntu-latest ] # TODO: Add Windows e2e tests: https://github.com/aws/shim-loggers-for-containerd/issues/68 | |
| name: Benchmark / awslogs / ${{ matrix.os }} | |
| runs-on: ${{ matrix.os }} | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - uses: actions/setup-go@v4 | |
| with: | |
| go-version: '1.21' | |
| cache: false | |
| - name: Start LocalStack # Copy from https://docs.localstack.cloud/user-guide/ci/github-actions/ | |
| run: | | |
| LOCALSTACK_VERSION=2.2.0 | |
| pip install localstack==${LOCALSTACK_VERSION} awscli-local[ver1] # install LocalStack cli and awslocal | |
| docker pull localstack/localstack:${LOCALSTACK_VERSION} # Make sure to pull the latest version of the image | |
| localstack start -d # Start LocalStack in the background | |
| echo "Waiting for LocalStack startup..." # Wait 30 seconds for the LocalStack container | |
| localstack wait -t 30 # to become ready before timing out | |
| echo "Startup complete" | |
| - name: install and start containerd | |
| shell: bash | |
| run: sudo scripts/install-containerd | |
| - name: start ecs local endpoint | |
| shell: bash | |
| run: scripts/start-ecs-local-endpoint | |
| - name: ip forwarding # awslogs driver hardcodes "169.254.170.2" as the aws credential endpoint ip so need to forward to local endpoint | |
| shell: bash | |
| run: sudo scripts/ip-forwarding | |
| - name: build | |
| run: sudo make build | |
| - name: Run benchmark | |
| run: sudo -E make test-benchmark-for-awslogs | tee benchmark-for-awslogs.txt | |
| shell: bash | |
| - name: Store benchmark result | |
| uses: benchmark-action/github-action-benchmark@v1 | |
| with: | |
| name: Benchmark for awslogs | |
| tool: 'go' | |
| benchmark-data-dir-path: "dev/bench/awslogs/ubuntu" | |
| output-file-path: benchmark-for-awslogs.txt | |
| - name: Push benchmark result | |
| run: git push 'https://github.com/aws/shim-loggers-for-containerd.git' gh-pages:gh-pages | |
| benchmark-for-fluentd: | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| os: [ ubuntu-latest ] # TODO: Add Windows e2e tests: https://github.com/aws/shim-loggers-for-containerd/issues/68 | |
| name: Benchmark / fluentd / ${{ matrix.os }} | |
| runs-on: ${{ matrix.os }} | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - uses: actions/setup-go@v4 | |
| with: | |
| go-version: '1.21' | |
| cache: false | |
| - name: install and start containerd | |
| shell: bash | |
| run: sudo scripts/install-containerd | |
| - name: start fluentd local endpoint | |
| shell: bash | |
| run: | | |
| : # not using github action env because env map cannot be defined in terms of other vars in the map. public.ecr.aws/docker/library/fluentd:v1.16-debian-1 | |
| FLUENTD_LOG_DIR=${GITHUB_WORKSPACE}/fluentd-logs | |
| FLUENTD_PORT=24224 | |
| FLUENTD_IMAGE=public.ecr.aws/docker/library/fluentd:v1.16-debian-1 | |
| : # ECR Public anonymous pull rate quota is 1 per sec. https://docs.aws.amazon.com/AmazonECR/latest/public/public-service-quotas.html. | |
| : # Running multiple jobs at the same time may exceed rate limit. Retry 3 times to prevent flaky behavior. | |
| count=0 | |
| until [ $count -ge 3 ] | |
| do | |
| docker pull $FLUENTD_IMAGE && break | |
| count=$((count+1)) | |
| [ $count -eq 3 ] && echo "Pull command failed after 3 attempts" && exit 1 | |
| sleep 3 | |
| done | |
| : # Fluentd container is not using root user so need 777 to make it writable. https://docs.github.com/en/actions/using-github-hosted-runners/about-github-hosted-runners#docker-container-filesystem | |
| sudo mkdir -m 777 $FLUENTD_LOG_DIR | |
| docker run -d -p $FLUENTD_PORT:24224 -p $FLUENTD_PORT:24224/udp -v $FLUENTD_LOG_DIR:/fluentd/log $FLUENTD_IMAGE | |
| - name: build | |
| run: sudo make build | |
| - name: Run benchmark | |
| run: sudo make test-benchmark-for-fluentd | tee benchmark-for-fluentd.txt | |
| shell: bash | |
| - name: Store benchmark result | |
| uses: benchmark-action/github-action-benchmark@v1 | |
| with: | |
| name: Benchmark for awslogs | |
| tool: 'go' | |
| benchmark-data-dir-path: "dev/bench/fluentd/ubuntu" | |
| output-file-path: benchmark-for-fluentd.txt | |
| - name: Push benchmark result | |
| run: git push 'https://github.com/aws/shim-loggers-for-containerd.git' gh-pages:gh-pages | |
| benchmark-for-splunk: | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| os: [ ubuntu-latest ] # TODO: Add Windows e2e tests: https://github.com/aws/shim-loggers-for-containerd/issues/68 | |
| name: Benchmark / splunk / ${{ matrix.os }} | |
| runs-on: ${{ matrix.os }} | |
| permissions: | |
| id-token: write | |
| contents: write | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - uses: actions/setup-go@v4 | |
| with: | |
| go-version: '1.21' | |
| cache: false | |
| - name: install and start containerd | |
| shell: bash | |
| run: sudo scripts/install-containerd | |
| - name: start splunk local endpoint | |
| id: splunk-local-endpoint | |
| shell: bash | |
| run: | | |
| SPLUNK_PASSWORD=1234567Aa! | |
| SPLUNK_PORT=8089 | |
| SPLUNK_IMAGE=splunk/splunk | |
| CONTAINER_ID=$(docker run -d -p $SPLUNK_PORT:8089 -p 8088:8088 \ | |
| -e "SPLUNK_START_ARGS=--accept-license" \ | |
| -e "SPLUNK_PASSWORD=$SPLUNK_PASSWORD" \ | |
| -e "SPLUNK_HEC_TOKEN=abcd1234" \ | |
| $SPLUNK_IMAGE) | |
| : # Splunk can only receives requests after becoming healthy | |
| until [ $(docker inspect $CONTAINER_ID --format ‘{{.State.Health.Status}}’) == ‘healthy’ ]; do sleep 10s; done; | |
| curl -L -k -u admin:$SPLUNK_PASSWORD \ | |
| -X POST https://localhost:$SPLUNK_PORT/services/admin/token-auth/tokens_auth \ | |
| -d disabled=false | |
| : # Wait for token to be enabled | |
| sleep 5 | |
| TOKEN_OUTPUT=$(curl -L -k -u admin:$SPLUNK_PASSWORD \ | |
| -X POST https://localhost:$SPLUNK_PORT/services/authorization/tokens?output_mode=json \ | |
| -d name=admin -d audience=tests) | |
| SPLUNK_TOKEN=$(echo $TOKEN_OUTPUT | jq -r '.entry[0].content.token') | |
| echo "SPLUNK_TOKEN=$SPLUNK_TOKEN" >> $GITHUB_OUTPUT | |
| - name: build | |
| run: sudo make build | |
| - name: Run benchmark | |
| run: sudo SPLUNK_TOKEN=${{ steps.splunk-local-endpoint.outputs.SPLUNK_TOKEN }} make test-benchmark-for-splunk | tee benchmark-for-splunk.txt | |
| shell: bash | |
| - name: Store benchmark result | |
| uses: benchmark-action/github-action-benchmark@v1 | |
| with: | |
| name: Benchmark for splunk | |
| tool: 'go' | |
| benchmark-data-dir-path: "dev/bench/splunk/ubuntu" | |
| output-file-path: benchmark-for-splunk.txt | |
| - name: Push benchmark result | |
| run: git push 'https://github.com/aws/shim-loggers-for-containerd.git' gh-pages:gh-pages |