ci: add benchmarking

Signed-off-by: Ziwen Ning <[email protected]>

.github/workflows/benchmark.yaml

@@ -0,0 +1,172 @@
+name: Benchmark
+on:
+  push:
+    branches:
+      - main
+      - test-benchmark
+    paths-ignore:
+      - '**.md'
+      - 'contrib/**'
+
+permissions:
+  # deployments permission to deploy GitHub pages website
+  deployments: write
+  # contents permission to update benchmark contents in gh-pages branch
+  contents: write
+
+jobs:
+  benchmark-for-awslogs:
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ ubuntu-latest ] # TODO: Add Windows e2e tests: https://github.com/aws/shim-loggers-for-containerd/issues/68
+    name: Benchmark / awslogs / ${{ matrix.os }}
+    runs-on: ${{ matrix.os }}
+    steps:
+      - uses: actions/checkout@v3
+      - uses: actions/setup-go@v4
+        with:
+          go-version: '1.21'
+          cache: false
+      - name: Start LocalStack # Copy from https://docs.localstack.cloud/user-guide/ci/github-actions/
+        run: |
+          LOCALSTACK_VERSION=2.2.0
+          pip install localstack==${LOCALSTACK_VERSION} awscli-local[ver1] # install LocalStack cli and awslocal
+          docker pull localstack/localstack:${LOCALSTACK_VERSION}         # Make sure to pull the latest version of the image
+          localstack start -d                       # Start LocalStack in the background
+          
+          echo "Waiting for LocalStack startup..."  # Wait 30 seconds for the LocalStack container
+          localstack wait -t 30                     # to become ready before timing out 
+          echo "Startup complete"
+      - name: install and start containerd
+        shell: bash
+        run: sudo scripts/install-containerd
+      - name: start ecs local endpoint
+        shell: bash
+        run: scripts/start-ecs-local-endpoint
+      - name: ip forwarding # awslogs driver hardcodes "169.254.170.2" as the aws credential endpoint ip so need to forward to local endpoint
+        shell: bash
+        run: sudo scripts/ip-forwarding
+      - name: build
+        run: sudo make build
+      - name: Run benchmark
+        run: sudo -E make test-benchmark-for-awslogs | tee benchmark-for-awslogs.txt
+        shell: bash
+      - name: Store benchmark result
+        uses: benchmark-action/github-action-benchmark@v1
+        with:
+          name: Benchmark for awslogs
+          tool: 'go'
+          benchmark-data-dir-path: "dev/bench/awslogs/ubuntu"
+          output-file-path: benchmark-for-awslogs.txt
+      - name: Push benchmark result
+        run: git push 'https://github.com/aws/shim-loggers-for-containerd.git' gh-pages:gh-pages
+  benchmark-for-fluentd:
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ ubuntu-latest ] # TODO: Add Windows e2e tests: https://github.com/aws/shim-loggers-for-containerd/issues/68
+    name: Benchmark / fluentd / ${{ matrix.os }}
+    runs-on: ${{ matrix.os }}
+    steps:
+      - uses: actions/checkout@v3
+      - uses: actions/setup-go@v4
+        with:
+          go-version: '1.21'
+          cache: false
+      - name: install and start containerd
+        shell: bash
+        run: sudo scripts/install-containerd
+      - name: start fluentd local endpoint
+        shell: bash
+        run: |
+          : # not using github action env because env map cannot be defined in terms of other vars in the map. public.ecr.aws/docker/library/fluentd:v1.16-debian-1
+          FLUENTD_LOG_DIR=${GITHUB_WORKSPACE}/fluentd-logs
+          FLUENTD_PORT=24224
+          FLUENTD_IMAGE=public.ecr.aws/docker/library/fluentd:v1.16-debian-1
+          : # ECR Public anonymous pull rate quota is 1 per sec. https://docs.aws.amazon.com/AmazonECR/latest/public/public-service-quotas.html.
+          : # Running multiple jobs at the same time may exceed rate limit. Retry 3 times to prevent flaky behavior.
+          count=0
+          until [ $count -ge 3 ]
+          do
+            docker pull $FLUENTD_IMAGE && break
+            count=$((count+1))
+            [ $count -eq 3 ] && echo "Pull command failed after 3 attempts" && exit 1
+            sleep 3
+          done
+          : # Fluentd container is not using root user so need 777 to make it writable. https://docs.github.com/en/actions/using-github-hosted-runners/about-github-hosted-runners#docker-container-filesystem
+          sudo mkdir -m 777 $FLUENTD_LOG_DIR
+          docker run -d -p $FLUENTD_PORT:24224 -p $FLUENTD_PORT:24224/udp -v $FLUENTD_LOG_DIR:/fluentd/log $FLUENTD_IMAGE
+      - name: build
+        run: sudo make build
+      - name: Run benchmark
+        run: sudo make test-benchmark-for-fluentd | tee benchmark-for-fluentd.txt
+        shell: bash
+      - name: Store benchmark result
+        uses: benchmark-action/github-action-benchmark@v1
+        with:
+          name: Benchmark for awslogs
+          tool: 'go'
+          benchmark-data-dir-path: "dev/bench/fluentd/ubuntu"
+          output-file-path: benchmark-for-fluentd.txt
+      - name: Push benchmark result
+        run: git push 'https://github.com/aws/shim-loggers-for-containerd.git' gh-pages:gh-pages
+  benchmark-for-splunk:
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ ubuntu-latest ] # TODO: Add Windows e2e tests: https://github.com/aws/shim-loggers-for-containerd/issues/68
+    name: Benchmark / splunk / ${{ matrix.os }}
+    runs-on: ${{ matrix.os }}
+    permissions:
+      id-token: write
+      contents: write
+    steps:
+      - uses: actions/checkout@v3
+      - uses: actions/setup-go@v4
+        with:
+          go-version: '1.21'
+          cache: false
+      - name: install and start containerd
+        shell: bash
+        run: sudo scripts/install-containerd
+      - name: start splunk local endpoint
+        id: splunk-local-endpoint
+        shell: bash
+        run: |
+          SPLUNK_PASSWORD=1234567Aa!
+          SPLUNK_PORT=8089
+          SPLUNK_IMAGE=splunk/splunk
+          CONTAINER_ID=$(docker run -d -p $SPLUNK_PORT:8089 -p 8088:8088 \
+          -e "SPLUNK_START_ARGS=--accept-license" \
+          -e "SPLUNK_PASSWORD=$SPLUNK_PASSWORD" \
+          -e "SPLUNK_HEC_TOKEN=abcd1234" \
+          $SPLUNK_IMAGE)
+          
+          : # Splunk can only receives requests after becoming healthy
+          until [ $(docker inspect $CONTAINER_ID --format ‘{{.State.Health.Status}}’) == ‘healthy’ ]; do sleep 10s; done;
+          
+          curl -L -k -u admin:$SPLUNK_PASSWORD \
+          -X POST https://localhost:$SPLUNK_PORT/services/admin/token-auth/tokens_auth \
+          -d disabled=false
+          : # Wait for token to be enabled
+          sleep 5
+          TOKEN_OUTPUT=$(curl -L -k -u admin:$SPLUNK_PASSWORD \
+          -X POST https://localhost:$SPLUNK_PORT/services/authorization/tokens?output_mode=json \
+          -d name=admin -d audience=tests)
+          SPLUNK_TOKEN=$(echo $TOKEN_OUTPUT | jq -r '.entry[0].content.token')
+          echo "SPLUNK_TOKEN=$SPLUNK_TOKEN" >> $GITHUB_OUTPUT
+      - name: build
+        run: sudo make build
+      - name: Run benchmark
+        run: sudo SPLUNK_TOKEN=${{ steps.splunk-local-endpoint.outputs.SPLUNK_TOKEN }} make test-benchmark-for-splunk | tee benchmark-for-splunk.txt
+        shell: bash
+      - name: Store benchmark result
+        uses: benchmark-action/github-action-benchmark@v1
+        with:
+          name: Benchmark for splunk
+          tool: 'go'
+          benchmark-data-dir-path: "dev/bench/splunk/ubuntu"
+          output-file-path: benchmark-for-splunk.txt
+      - name: Push benchmark result
+        run: git push 'https://github.com/aws/shim-loggers-for-containerd.git' gh-pages:gh-pages

Makefile

@@ -43,6 +43,18 @@ test-e2e-for-fluentd:
 test-e2e-for-splunk:
 	go test -timeout 30m ./e2e -test.v -ginkgo.v --binary "$(AWS_CONTAINERD_LOGGERS_BINARY)" --log-driver "splunk" --splunk-token ${SPLUNK_TOKEN}
 
+.PHONY: test-benchmark-for-awslogs
+test-benchmark-for-awslogs:
+	cd benchmark/awslogs && go test -bench=Awslogs -benchmem --binary "$(AWS_CONTAINERD_LOGGERS_BINARY)"
+
+.PHONY: test-benchmark-for-fluentd
+test-benchmark-for-fluentd:
+	cd benchmark/fluentd && go test -bench=Fluentd -benchmem --binary "$(AWS_CONTAINERD_LOGGERS_BINARY)"
+
+.PHONY: test-benchmark-for-splunk
+test-benchmark-for-splunk:
+	cd benchmark/splunk && go test -bench=Splunk -benchmem --binary "$(AWS_CONTAINERD_LOGGERS_BINARY)" --splunk-token ${SPLUNK_TOKEN}
+
 .PHONY: coverage
 coverage:
 	go test -tags unit $(shell go list ./... | grep -v e2e) -coverprofile=test-coverage.out

benchmark/awslogs/awslogs_test.go

@@ -0,0 +1,84 @@
+// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+package awslogs
+
+import (
+	"context"
+	"flag"
+	"strings"
+	"testing"
+
+	"github.com/aws/aws-sdk-go-v2/aws"
+	"github.com/aws/aws-sdk-go-v2/config"
+	"github.com/aws/aws-sdk-go-v2/service/cloudwatchlogs"
+	"github.com/aws/shim-loggers-for-containerd/e2e"
+	"github.com/containerd/containerd/cio"
+)
+
+const (
+	awslogsCredentialsEndpointKey = "--awslogs-credentials-endpoint" //nolint:gosec // not credentials
+	awslogsRegionKey              = "--awslogs-region"
+	awslogsStreamKey              = "--awslogs-stream"
+	awslogsGroupKey               = "--awslogs-group"
+	awslogsEndpointKey            = "--awslogs-endpoint"
+	testEcsLocalEndpointPort      = "51679"
+	testAwslogsCredentialEndpoint = ":" + testEcsLocalEndpointPort + "/creds"
+	testAwslogsRegion             = "us-east-1"
+	testAwslogsStream             = "test-stream"
+	nonExistentAwslogsStream      = "non-existent-stream"
+	testAwslogsGroup              = "test-shim-logger"
+	testAwslogsEndpoint           = "http://localhost.localstack.cloud:4566" // Recommended endpoint:
+)
+
+var (
+	// Binary is the path the binary of the shim loggers for containerd.
+	Binary = flag.String("binary", "", "the binary of shim loggers for containerd")
+)
+
+func BenchmarkAwslogs(b *testing.B) {
+	var cwClient *cloudwatchlogs.Client
+	// Reference to set up Go client for aws local stack: https://docs.localstack.cloud/user-guide/integrations/sdks/go/.
+	customResolver := aws.EndpointResolverWithOptionsFunc(func(service, region string, options ...interface{}) (aws.Endpoint, error) {
+		return aws.Endpoint{
+			PartitionID:   "aws",
+			URL:           testAwslogsEndpoint,
+			SigningRegion: testAwslogsRegion,
+		}, nil
+	})
+	cfg, err := config.LoadDefaultConfig(context.TODO(), config.WithRegion(testAwslogsRegion),
+		config.WithEndpointResolverWithOptions(customResolver))
+	if err != nil {
+		b.Fatal(err)
+	}
+	cwClient = cloudwatchlogs.NewFromConfig(cfg)
+	_, err = cwClient.CreateLogGroup(context.TODO(), &cloudwatchlogs.CreateLogGroupInput{
+		LogGroupName: aws.String(testAwslogsGroup),
+	})
+	if err != nil {
+		b.Fatal(err)
+	}
+	_, err = cwClient.CreateLogStream(context.TODO(), &cloudwatchlogs.CreateLogStreamInput{
+		LogGroupName:  aws.String(testAwslogsGroup),
+		LogStreamName: aws.String(testAwslogsStream),
+	})
+	if err != nil {
+		b.Fatal(err)
+	}
+	testLog := strings.Repeat("a", 1024)
+	args := map[string]string{
+		e2e.LogDriverTypeKey:          e2e.AwslogsDriverName,
+		e2e.ContainerIDKey:            e2e.TestContainerID,
+		e2e.ContainerNameKey:          e2e.TestContainerName,
+		awslogsCredentialsEndpointKey: testAwslogsCredentialEndpoint,
+		awslogsRegionKey:              testAwslogsRegion,
+		awslogsGroupKey:               testAwslogsGroup,
+		awslogsStreamKey:              nonExistentAwslogsStream,
+		awslogsEndpointKey:            testAwslogsEndpoint,
+	}
+	creator := cio.BinaryIO(*Binary, args)
+	err = e2e.SendTestLogByContainerd(creator, testLog)
+	if err != nil {
+		b.Fatal(err)
+	}
+}

benchmark/fluentd/fluentd_test.go

@@ -0,0 +1,32 @@
+// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+package fluentd
+
+import (
+	"flag"
+	"strings"
+	"testing"
+
+	"github.com/aws/shim-loggers-for-containerd/e2e"
+	"github.com/containerd/containerd/cio"
+)
+
+var (
+	// Binary is the path the binary of the shim loggers for containerd.
+	Binary = flag.String("binary", "", "the binary of shim loggers for containerd")
+)
+
+func BenchmarkFluentd(b *testing.B) {
+	testLog := strings.Repeat("a", 1024)
+	args := map[string]string{
+		e2e.LogDriverTypeKey: e2e.FluentdDriverName,
+		e2e.ContainerIDKey:   e2e.TestContainerID,
+		e2e.ContainerNameKey: e2e.TestContainerName,
+	}
+	creator := cio.BinaryIO(*Binary, args)
+	err := e2e.SendTestLogByContainerd(creator, testLog)
+	if err != nil {
+		b.Fatal(err)
+	}
+}

benchmark/splunk/splunk_test.go

@@ -0,0 +1,43 @@
+// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+package splunk
+
+import (
+	"flag"
+	"strings"
+	"testing"
+
+	"github.com/aws/shim-loggers-for-containerd/e2e"
+	"github.com/containerd/containerd/cio"
+)
+
+const (
+	splunkTokenKey              = "--splunk-token" //nolint:gosec // no real credential
+	splunkURLkey                = "--splunk-url"
+	splunkInsecureskipverifyKey = "--splunk-insecureskipverify"
+	testSplunkURL               = "https://localhost:8089"
+)
+
+var (
+	// Binary is the path the binary of the shim loggers for containerd.
+	Binary      = flag.String("binary", "", "the binary of shim loggers for containerd")
+	SplunkToken = flag.String("splunk-token", "", "the token to access Splunk")
+)
+
+func BenchmarkSplunk(b *testing.B) {
+	testLog := strings.Repeat("a", 1024)
+	args := map[string]string{
+		e2e.LogDriverTypeKey:        e2e.SplunkDriverName,
+		e2e.ContainerIDKey:          e2e.TestContainerID,
+		e2e.ContainerNameKey:        e2e.TestContainerName,
+		splunkTokenKey:              *SplunkToken,
+		splunkURLkey:                testSplunkURL,
+		splunkInsecureskipverifyKey: "true",
+	}
+	creator := cio.BinaryIO(*Binary, args)
+	err := e2e.SendTestLogByContainerd(creator, testLog)
+	if err != nil {
+		b.Fatal(err)
+	}
+}