-
Notifications
You must be signed in to change notification settings - Fork 29
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
*Description of changes:* add benchmarking for time and memory of sending a 1MB log Result page: https://aws.github.io/shim-loggers-for-containerd/dev/bench/ UX improvements will come later in the gh-pages branch. By submitting this pull request, I confirm that you can use, modify, copy, and redistribute this contribution, under the terms of your choice. Signed-off-by: Ziwen Ning <[email protected]>
- Loading branch information
Showing
13 changed files
with
527 additions
and
171 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,144 @@ | ||
| name: Benchmark | ||
| on: | ||
| push: | ||
| branches: | ||
| - main | ||
|
|
||
| permissions: | ||
| # deployments permission to deploy GitHub pages website | ||
| deployments: write | ||
| # contents permission to update benchmark contents in gh-pages branch | ||
| contents: write | ||
|
|
||
| jobs: | ||
| benchmark-for-awslogs: | ||
| strategy: | ||
| fail-fast: false | ||
| matrix: | ||
| os: [ ubuntu-latest ] # TODO: Add Windows e2e tests: https://github.com/aws/shim-loggers-for-containerd/issues/68 | ||
| name: Benchmark / awslogs / ${{ matrix.os }} | ||
| runs-on: ${{ matrix.os }} | ||
| steps: | ||
| - uses: actions/checkout@v3 | ||
| - uses: actions/setup-go@v4 | ||
| with: | ||
| go-version: '1.21' | ||
| cache: false | ||
| - name: Start LocalStack | ||
| shell: bash | ||
| run: scripts/start-localstack | ||
| - name: install and start containerd | ||
| shell: bash | ||
| run: sudo scripts/install-containerd | ||
| - name: start ecs local endpoint | ||
| shell: bash | ||
| run: scripts/start-ecs-local-endpoint | ||
| - name: ip forwarding # awslogs driver hardcodes "169.254.170.2" as the aws credential endpoint ip so need to forward to local endpoint | ||
| shell: bash | ||
| run: sudo scripts/ip-forwarding | ||
| - name: build | ||
| run: sudo make build | ||
| - name: Run benchmark | ||
| run: sudo -E make test-benchmark-for-awslogs | tee benchmark-for-awslogs.txt | ||
| shell: bash | ||
| - name: Store benchmark result | ||
| uses: benchmark-action/github-action-benchmark@v1 | ||
| with: | ||
| name: Benchmark for awslogs | ||
| tool: 'go' | ||
| benchmark-data-dir-path: "dev/bench/awslogs/ubuntu" | ||
| output-file-path: benchmark-for-awslogs.txt | ||
| - name: Push benchmark result | ||
| run: git push 'https://github.com/aws/shim-loggers-for-containerd.git' gh-pages:gh-pages | ||
| benchmark-for-fluentd: | ||
| strategy: | ||
| fail-fast: false | ||
| matrix: | ||
| os: [ ubuntu-latest ] # TODO: Add Windows e2e tests: https://github.com/aws/shim-loggers-for-containerd/issues/68 | ||
| name: Benchmark / fluentd / ${{ matrix.os }} | ||
| runs-on: ${{ matrix.os }} | ||
| steps: | ||
| - uses: actions/checkout@v3 | ||
| - uses: actions/setup-go@v4 | ||
| with: | ||
| go-version: '1.21' | ||
| cache: false | ||
| - name: install and start containerd | ||
| shell: bash | ||
| run: sudo scripts/install-containerd | ||
| - name: start fluentd local endpoint | ||
| shell: bash | ||
| run: scripts/start-fluentd-local-endpoint | ||
| - name: build | ||
| run: sudo make build | ||
| - name: Run benchmark | ||
| run: sudo make test-benchmark-for-fluentd | tee benchmark-for-fluentd.txt | ||
| shell: bash | ||
| - name: Store benchmark result | ||
| uses: benchmark-action/github-action-benchmark@v1 | ||
| with: | ||
| name: Benchmark for awslogs | ||
| tool: 'go' | ||
| benchmark-data-dir-path: "dev/bench/fluentd/ubuntu" | ||
| output-file-path: benchmark-for-fluentd.txt | ||
| - name: Push benchmark result | ||
| run: git push 'https://github.com/aws/shim-loggers-for-containerd.git' gh-pages:gh-pages | ||
| benchmark-for-splunk: | ||
| strategy: | ||
| fail-fast: false | ||
| matrix: | ||
| os: [ ubuntu-latest ] # TODO: Add Windows e2e tests: https://github.com/aws/shim-loggers-for-containerd/issues/68 | ||
| name: Benchmark / splunk / ${{ matrix.os }} | ||
| runs-on: ${{ matrix.os }} | ||
| permissions: | ||
| id-token: write | ||
| contents: write | ||
| steps: | ||
| - uses: actions/checkout@v3 | ||
| - uses: actions/setup-go@v4 | ||
| with: | ||
| go-version: '1.21' | ||
| cache: false | ||
| - name: install and start containerd | ||
| shell: bash | ||
| run: sudo scripts/install-containerd | ||
| - name: start splunk local endpoint | ||
| id: splunk-local-endpoint | ||
| shell: bash | ||
| run: | | ||
| SPLUNK_PASSWORD=1234567Aa! | ||
| SPLUNK_PORT=8089 | ||
| SPLUNK_IMAGE=splunk/splunk | ||
| CONTAINER_ID=$(docker run -d -p $SPLUNK_PORT:8089 -p 8088:8088 \ | ||
| -e "SPLUNK_START_ARGS=--accept-license" \ | ||
| -e "SPLUNK_PASSWORD=$SPLUNK_PASSWORD" \ | ||
| -e "SPLUNK_HEC_TOKEN=abcd1234" \ | ||
| $SPLUNK_IMAGE) | ||
| : # Splunk can only receives requests after becoming healthy | ||
| until [ $(docker inspect $CONTAINER_ID --format ‘{{.State.Health.Status}}’) == ‘healthy’ ]; do sleep 10s; done; | ||
| curl -L -k -u admin:$SPLUNK_PASSWORD \ | ||
| -X POST https://localhost:$SPLUNK_PORT/services/admin/token-auth/tokens_auth \ | ||
| -d disabled=false | ||
| : # Wait for token to be enabled | ||
| sleep 5 | ||
| TOKEN_OUTPUT=$(curl -L -k -u admin:$SPLUNK_PASSWORD \ | ||
| -X POST https://localhost:$SPLUNK_PORT/services/authorization/tokens?output_mode=json \ | ||
| -d name=admin -d audience=tests) | ||
| SPLUNK_TOKEN=$(echo $TOKEN_OUTPUT | jq -r '.entry[0].content.token') | ||
| echo "SPLUNK_TOKEN=$SPLUNK_TOKEN" >> $GITHUB_OUTPUT | ||
| - name: build | ||
| run: sudo make build | ||
| - name: Run benchmark | ||
| run: sudo SPLUNK_TOKEN=${{ steps.splunk-local-endpoint.outputs.SPLUNK_TOKEN }} make test-benchmark-for-splunk | tee benchmark-for-splunk.txt | ||
| shell: bash | ||
| - name: Store benchmark result | ||
| uses: benchmark-action/github-action-benchmark@v1 | ||
| with: | ||
| name: Benchmark for splunk | ||
| tool: 'go' | ||
| benchmark-data-dir-path: "dev/bench/splunk/ubuntu" | ||
| output-file-path: benchmark-for-splunk.txt | ||
| - name: Push benchmark result | ||
| run: git push 'https://github.com/aws/shim-loggers-for-containerd.git' gh-pages:gh-pages |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,86 @@ | ||
| // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. | ||
| // SPDX-License-Identifier: Apache-2.0 | ||
|
|
||
| //go:build benchmark | ||
|
|
||
| package awslogs | ||
|
|
||
| import ( | ||
| "context" | ||
| "flag" | ||
| "strings" | ||
| "testing" | ||
|
|
||
| "github.com/aws/aws-sdk-go-v2/aws" | ||
| "github.com/aws/aws-sdk-go-v2/config" | ||
| "github.com/aws/aws-sdk-go-v2/service/cloudwatchlogs" | ||
| "github.com/aws/shim-loggers-for-containerd/e2e" | ||
| "github.com/containerd/containerd/cio" | ||
| ) | ||
|
|
||
| const ( | ||
| awslogsCredentialsEndpointKey = "--awslogs-credentials-endpoint" //nolint:gosec // not credentials | ||
| awslogsRegionKey = "--awslogs-region" | ||
| awslogsStreamKey = "--awslogs-stream" | ||
| awslogsGroupKey = "--awslogs-group" | ||
| awslogsEndpointKey = "--awslogs-endpoint" | ||
| testEcsLocalEndpointPort = "51679" | ||
| testAwslogsCredentialEndpoint = ":" + testEcsLocalEndpointPort + "/creds" | ||
| testAwslogsRegion = "us-east-1" | ||
| testAwslogsStream = "test-stream" | ||
| nonExistentAwslogsStream = "non-existent-stream" | ||
| testAwslogsGroup = "test-shim-logger" | ||
| testAwslogsEndpoint = "http://localhost.localstack.cloud:4566" // Recommended endpoint: | ||
| ) | ||
|
|
||
| var ( | ||
| // Binary is the path the binary of the shim loggers for containerd. | ||
| Binary = flag.String("binary", "", "the binary of shim loggers for containerd") | ||
| ) | ||
|
|
||
| func BenchmarkAwslogs(b *testing.B) { | ||
| var cwClient *cloudwatchlogs.Client | ||
| // Reference to set up Go client for aws local stack: https://docs.localstack.cloud/user-guide/integrations/sdks/go/. | ||
| customResolver := aws.EndpointResolverWithOptionsFunc(func(service, region string, options ...interface{}) (aws.Endpoint, error) { | ||
| return aws.Endpoint{ | ||
| PartitionID: "aws", | ||
| URL: testAwslogsEndpoint, | ||
| SigningRegion: testAwslogsRegion, | ||
| }, nil | ||
| }) | ||
| cfg, err := config.LoadDefaultConfig(context.TODO(), config.WithRegion(testAwslogsRegion), | ||
| config.WithEndpointResolverWithOptions(customResolver)) | ||
| if err != nil { | ||
| b.Fatal(err) | ||
| } | ||
| cwClient = cloudwatchlogs.NewFromConfig(cfg) | ||
| _, err = cwClient.CreateLogGroup(context.TODO(), &cloudwatchlogs.CreateLogGroupInput{ | ||
| LogGroupName: aws.String(testAwslogsGroup), | ||
| }) | ||
| if err != nil { | ||
| b.Fatal(err) | ||
| } | ||
| _, err = cwClient.CreateLogStream(context.TODO(), &cloudwatchlogs.CreateLogStreamInput{ | ||
| LogGroupName: aws.String(testAwslogsGroup), | ||
| LogStreamName: aws.String(testAwslogsStream), | ||
| }) | ||
| if err != nil { | ||
| b.Fatal(err) | ||
| } | ||
| testLog := strings.Repeat("a", 1024) | ||
| args := map[string]string{ | ||
| e2e.LogDriverTypeKey: e2e.AwslogsDriverName, | ||
| e2e.ContainerIDKey: e2e.TestContainerID, | ||
| e2e.ContainerNameKey: e2e.TestContainerName, | ||
| awslogsCredentialsEndpointKey: testAwslogsCredentialEndpoint, | ||
| awslogsRegionKey: testAwslogsRegion, | ||
| awslogsGroupKey: testAwslogsGroup, | ||
| awslogsStreamKey: nonExistentAwslogsStream, | ||
| awslogsEndpointKey: testAwslogsEndpoint, | ||
| } | ||
| creator := cio.BinaryIO(*Binary, args) | ||
| err = e2e.SendTestLogByContainerd(creator, testLog) | ||
| if err != nil { | ||
| b.Fatal(err) | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,34 @@ | ||
| // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. | ||
| // SPDX-License-Identifier: Apache-2.0 | ||
|
|
||
| //go:build benchmark | ||
|
|
||
| package fluentd | ||
|
|
||
| import ( | ||
| "flag" | ||
| "strings" | ||
| "testing" | ||
|
|
||
| "github.com/aws/shim-loggers-for-containerd/e2e" | ||
| "github.com/containerd/containerd/cio" | ||
| ) | ||
|
|
||
| var ( | ||
| // Binary is the path the binary of the shim loggers for containerd. | ||
| Binary = flag.String("binary", "", "the binary of shim loggers for containerd") | ||
| ) | ||
|
|
||
| func BenchmarkFluentd(b *testing.B) { | ||
| testLog := strings.Repeat("a", 1024) | ||
| args := map[string]string{ | ||
| e2e.LogDriverTypeKey: e2e.FluentdDriverName, | ||
| e2e.ContainerIDKey: e2e.TestContainerID, | ||
| e2e.ContainerNameKey: e2e.TestContainerName, | ||
| } | ||
| creator := cio.BinaryIO(*Binary, args) | ||
| err := e2e.SendTestLogByContainerd(creator, testLog) | ||
| if err != nil { | ||
| b.Fatal(err) | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,45 @@ | ||
| // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. | ||
| // SPDX-License-Identifier: Apache-2.0 | ||
|
|
||
| //go:build benchmark | ||
|
|
||
| package splunk | ||
|
|
||
| import ( | ||
| "flag" | ||
| "strings" | ||
| "testing" | ||
|
|
||
| "github.com/aws/shim-loggers-for-containerd/e2e" | ||
| "github.com/containerd/containerd/cio" | ||
| ) | ||
|
|
||
| const ( | ||
| splunkTokenKey = "--splunk-token" //nolint:gosec // no real credential | ||
| splunkURLkey = "--splunk-url" | ||
| splunkInsecureskipverifyKey = "--splunk-insecureskipverify" | ||
| testSplunkURL = "https://localhost:8089" | ||
| ) | ||
|
|
||
| var ( | ||
| // Binary is the path the binary of the shim loggers for containerd. | ||
| Binary = flag.String("binary", "", "the binary of shim loggers for containerd") | ||
| SplunkToken = flag.String("splunk-token", "", "the token to access Splunk") | ||
| ) | ||
|
|
||
| func BenchmarkSplunk(b *testing.B) { | ||
| testLog := strings.Repeat("a", 1024) | ||
| args := map[string]string{ | ||
| e2e.LogDriverTypeKey: e2e.SplunkDriverName, | ||
| e2e.ContainerIDKey: e2e.TestContainerID, | ||
| e2e.ContainerNameKey: e2e.TestContainerName, | ||
| splunkTokenKey: *SplunkToken, | ||
| splunkURLkey: testSplunkURL, | ||
| splunkInsecureskipverifyKey: "true", | ||
| } | ||
| creator := cio.BinaryIO(*Binary, args) | ||
| err := e2e.SendTestLogByContainerd(creator, testLog) | ||
| if err != nil { | ||
| b.Fatal(err) | ||
| } | ||
| } |
Oops, something went wrong.