benchmark.yaml

name: Benchmark
on:
  push:
    branches:
      - main
      - test-benchmark

permissions:
  # deployments permission to deploy GitHub pages website
  deployments: write
  # contents permission to update benchmark contents in gh-pages branch
  contents: write

jobs:
  benchmark-for-awslogs:
    strategy:
      fail-fast: false
      matrix:
        os: [ ubuntu-latest ] # TODO: Add Windows e2e tests: https://github.com/aws/shim-loggers-for-containerd/issues/68
    name: Benchmark / awslogs / ${{ matrix.os }}
    runs-on: ${{ matrix.os }}
    steps:
      - uses: actions/checkout@v3
      - uses: actions/setup-go@v4
        with:
          go-version: '1.21'
          cache: false
      - name: Start LocalStack
        shell: bash
        run: scripts/start-localstack
      - name: install and start containerd
        shell: bash
        run: sudo scripts/install-containerd
      - name: start ecs local endpoint
        shell: bash
        run: scripts/start-ecs-local-endpoint
      - name: ip forwarding # awslogs driver hardcodes "169.254.170.2" as the aws credential endpoint ip so need to forward to local endpoint
        shell: bash
        run: sudo scripts/ip-forwarding
      - name: build
        run: sudo make build
      - name: Run benchmark
        run: sudo -E make test-benchmark-for-awslogs | tee benchmark-for-awslogs.txt
        shell: bash
      - name: Store benchmark result
        uses: benchmark-action/github-action-benchmark@v1
        with:
          name: Benchmark for awslogs
          tool: 'go'
          benchmark-data-dir-path: "dev/bench/awslogs/ubuntu"
          output-file-path: benchmark-for-awslogs.txt
      - name: Push benchmark result
        run: git push 'https://github.com/aws/shim-loggers-for-containerd.git' gh-pages:gh-pages
  benchmark-for-fluentd:
    strategy:
      fail-fast: false
      matrix:
        os: [ ubuntu-latest ] # TODO: Add Windows e2e tests: https://github.com/aws/shim-loggers-for-containerd/issues/68
    name: Benchmark / fluentd / ${{ matrix.os }}
    runs-on: ${{ matrix.os }}
    steps:
      - uses: actions/checkout@v3
      - uses: actions/setup-go@v4
        with:
          go-version: '1.21'
          cache: false
      - name: install and start containerd
        shell: bash
        run: sudo scripts/install-containerd
      - name: start fluentd local endpoint
        shell: bash
        run: sudo scripts/start-fluentd-local-endpoint
      - name: build
        run: sudo make build
      - name: Run benchmark
        run: sudo make test-benchmark-for-fluentd | tee benchmark-for-fluentd.txt
        shell: bash
      - name: Store benchmark result
        uses: benchmark-action/github-action-benchmark@v1
        with:
          name: Benchmark for awslogs
          tool: 'go'
          benchmark-data-dir-path: "dev/bench/fluentd/ubuntu"
          output-file-path: benchmark-for-fluentd.txt
      - name: Push benchmark result
        run: git push 'https://github.com/aws/shim-loggers-for-containerd.git' gh-pages:gh-pages
  benchmark-for-splunk:
    strategy:
      fail-fast: false
      matrix:
        os: [ ubuntu-latest ] # TODO: Add Windows e2e tests: https://github.com/aws/shim-loggers-for-containerd/issues/68
    name: Benchmark / splunk / ${{ matrix.os }}
    runs-on: ${{ matrix.os }}
    permissions:
      id-token: write
      contents: write
    steps:
      - uses: actions/checkout@v3
      - uses: actions/setup-go@v4
        with:
          go-version: '1.21'
          cache: false
      - name: install and start containerd
        shell: bash
        run: sudo scripts/install-containerd
      - name: start splunk local endpoint
        id: splunk-local-endpoint
        shell: bash
        run: |
          SPLUNK_PASSWORD=1234567Aa!
          SPLUNK_PORT=8089
          SPLUNK_IMAGE=splunk/splunk
          CONTAINER_ID=$(docker run -d -p $SPLUNK_PORT:8089 -p 8088:8088 \
          -e "SPLUNK_START_ARGS=--accept-license" \
          -e "SPLUNK_PASSWORD=$SPLUNK_PASSWORD" \
          -e "SPLUNK_HEC_TOKEN=abcd1234" \
          $SPLUNK_IMAGE)
          
          : # Splunk can only receives requests after becoming healthy
          until [ $(docker inspect $CONTAINER_ID --format ‘{{.State.Health.Status}}’) == ‘healthy’ ]; do sleep 10s; done;
          
          curl -L -k -u admin:$SPLUNK_PASSWORD \
          -X POST https://localhost:$SPLUNK_PORT/services/admin/token-auth/tokens_auth \
          -d disabled=false
          : # Wait for token to be enabled
          sleep 5
          TOKEN_OUTPUT=$(curl -L -k -u admin:$SPLUNK_PASSWORD \
          -X POST https://localhost:$SPLUNK_PORT/services/authorization/tokens?output_mode=json \
          -d name=admin -d audience=tests)
          SPLUNK_TOKEN=$(echo $TOKEN_OUTPUT | jq -r '.entry[0].content.token')
          echo "SPLUNK_TOKEN=$SPLUNK_TOKEN" >> $GITHUB_OUTPUT
      - name: build
        run: sudo make build
      - name: Run benchmark
        run: sudo SPLUNK_TOKEN=${{ steps.splunk-local-endpoint.outputs.SPLUNK_TOKEN }} make test-benchmark-for-splunk | tee benchmark-for-splunk.txt
        shell: bash
      - name: Store benchmark result
        uses: benchmark-action/github-action-benchmark@v1
        with:
          name: Benchmark for splunk
          tool: 'go'
          benchmark-data-dir-path: "dev/bench/splunk/ubuntu"
          output-file-path: benchmark-for-splunk.txt
      - name: Push benchmark result
        run: git push 'https://github.com/aws/shim-loggers-for-containerd.git' gh-pages:gh-pages