ci: add Splunk basic e2e tests (#73)

*Description of changes:*

The test is not perfect but can be used for basic validation because the
containerd task is ran successfully with the Splunk shim logger. It is
nice to validate the logs in Splunk after it. It is not straightforward
to do that so opened issue to track it.
#74

By submitting this pull request, I confirm that you can use, modify,
copy, and redistribute this contribution, under the terms of your
choice.

Signed-off-by: Ziwen Ning <[email protected]>

.github/workflows/ci.yaml

@@ -114,6 +114,57 @@ jobs:
         run: sudo make build
       - name: test-e2e
         run: sudo make test-e2e-for-fluentd # containerd interaction requires sudo
+  e2e-tests-for-splunk:
+    strategy:
+      fail-fast: false
+      matrix:
+        go: [ '1.20', '1.21' ]
+        os: [ ubuntu-latest ] # TODO: Add Windows e2e tests: https://github.com/aws/shim-loggers-for-containerd/issues/68
+    name: E2E tests / splunk / ${{ matrix.os }} / Go ${{ matrix.go }}
+    runs-on: ${{ matrix.os }}
+    permissions:
+      id-token: write
+      contents: write
+    steps:
+      - uses: actions/checkout@v3
+      - uses: actions/setup-go@v4
+        with:
+          go-version: ${{ matrix.go }}
+          cache: false
+      - name: install and start containerd
+        shell: bash
+        run: sudo scripts/install-containerd
+      - name: start splunk local endpoint
+        id: splunk-local-endpoint
+        shell: bash
+        run: |
+          SPLUNK_PASSWORD=1234567Aa!
+          SPLUNK_PORT=8089
+          SPLUNK_IMAGE=splunk/splunk
+          CONTAINER_ID=$(docker run -d -p $SPLUNK_PORT:8089 -p 8088:8088 \
+          -e "SPLUNK_START_ARGS=--accept-license" \
+          -e "SPLUNK_PASSWORD=$SPLUNK_PASSWORD" \
+          -e "SPLUNK_HEC_TOKEN=abcd1234" \
+          $SPLUNK_IMAGE)
+          
+          : # Splunk can only receives requests after becoming healthy
+          until [ $(docker inspect $CONTAINER_ID --format ‘{{.State.Health.Status}}’) == ‘healthy’ ]; do sleep 10s; done;
+          
+          curl -L -k -u admin:$SPLUNK_PASSWORD \
+          -X POST https://localhost:$SPLUNK_PORT/services/admin/token-auth/tokens_auth \
+          -d disabled=false
+          : # Wait for token to be enabled
+          sleep 5
+          TOKEN_OUTPUT=$(curl -L -k -u admin:$SPLUNK_PASSWORD \
+          -X POST https://localhost:$SPLUNK_PORT/services/authorization/tokens?output_mode=json \
+          -d name=admin -d audience=tests)
+          SPLUNK_TOKEN=$(echo $TOKEN_OUTPUT | jq -r '.entry[0].content.token')
+          echo "SPLUNK_TOKEN=$SPLUNK_TOKEN" >> $GITHUB_OUTPUT
+      - name: build
+        run: sudo make build
+      - name: test-e2e
+        run: |
+          sudo SPLUNK_TOKEN=${{ steps.splunk-local-endpoint.outputs.SPLUNK_TOKEN }} make test-e2e-for-splunk # containerd interaction requires sudo
   go-mod-tidy-check:
     runs-on: ubuntu-latest
     steps:

Makefile

@@ -39,6 +39,10 @@ test-e2e-for-aws-logs:
 test-e2e-for-fluentd:
 	go test -timeout 30m ./e2e -test.v -ginkgo.v --binary "$(AWS_CONTAINERD_LOGGERS_BINARY)" --log-driver "fluentd"
 
+.PHONY: test-e2e-for-splunk
+test-e2e-for-splunk:
+	go test -timeout 30m ./e2e -test.v -ginkgo.v --binary "$(AWS_CONTAINERD_LOGGERS_BINARY)" --log-driver "splunk" --splunk-token ${SPLUNK_TOKEN}
+
 .PHONY: coverage
 coverage:
 	go test -tags unit $(shell go list ./... | grep -v e2e) -coverprofile=test-coverage.out

e2e/fluentd_test.go

@@ -27,7 +27,6 @@ var testFluentd = func() {
 				logDriverTypeKey: fluentdDriverName,
 				containerIdKey:   testContainerId,
 				containerNameKey: testContainerName,
-				"--verbose":      "true",
 			}
 			creator := cio.BinaryIO(*Binary, args)
 			sendTestLogByContainerd(creator, testLog)

e2e/main_test.go

@@ -22,6 +22,7 @@ const (
 	logDriverTypeKey  = "--log-driver"
 	awslogsDriverName = "awslogs"
 	fluentdDriverName = "fluentd"
+	splunkDriverName  = "splunk"
 	containerIdKey    = "--container-id"
 	containerNameKey  = "--container-name"
 	testContainerId   = "test-container-id"
@@ -33,8 +34,9 @@ const (
 
 var (
 	// Binary is the path the binary of the shim loggers for containerd
-	Binary    = flag.String("binary", "", "the binary of shim loggers for containerd")
-	LogDriver = flag.String("log-driver", "", "the log driver to test")
+	Binary      = flag.String("binary", "", "the binary of shim loggers for containerd")
+	LogDriver   = flag.String("log-driver", "", "the log driver to test")
+	SplunkToken = flag.String("splunk-token", "", "the token to access Splunk")
 )
 
 func TestShimLoggers(t *testing.T) {
@@ -47,6 +49,9 @@ func TestShimLoggers(t *testing.T) {
 		if *LogDriver == fluentdDriverName || *LogDriver == "" {
 			testFluentd()
 		}
+		if *LogDriver == splunkDriverName || *LogDriver == "" {
+			testSplunk(*SplunkToken)
+		}
 	})
 
 	gomega.RegisterFailHandler(ginkgo.Fail)

e2e/splunk_test.go

@@ -0,0 +1,37 @@
+// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+package e2e
+
+import (
+	"github.com/containerd/containerd/cio"
+	"github.com/onsi/ginkgo/v2"
+	"github.com/onsi/gomega"
+)
+
+const (
+	splunkTokenKey              = "--splunk-token"
+	splunkUrlkey                = "--splunk-url"
+	splunkInsecureskipverifyKey = "--splunk-insecureskipverify"
+	testSplunkUrl               = "https://localhost:8089"
+)
+
+var testSplunk = func(token string) {
+	// These tests are run in serial because we only define one log driver instance.
+	ginkgo.Describe("splunk shim logger", ginkgo.Serial, func() {
+		gomega.Expect(token).ShouldNot(gomega.BeEmpty())
+		ginkgo.It("should send logs to splunk log driver", func() {
+			args := map[string]string{
+				logDriverTypeKey:            splunkDriverName,
+				containerIdKey:              testContainerId,
+				containerNameKey:            testContainerName,
+				splunkTokenKey:              token,
+				splunkUrlkey:                testSplunkUrl,
+				splunkInsecureskipverifyKey: "true",
+			}
+			creator := cio.BinaryIO(*Binary, args)
+			sendTestLogByContainerd(creator, testLog)
+			// TODO: Validate logs in Splunk local. https://github.com/aws/shim-loggers-for-containerd/issues/74
+		})
+	})
+}