ci: fix awslogs test target and add more test scenarios #190
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: CI | |
| on: | |
| push: | |
| branches: | |
| - main | |
| pull_request: | |
| branches: | |
| - main | |
| jobs: | |
| # It's recommended to run golangci-lint in a job separate from other jobs (go test, etc) because different jobs run in parallel. | |
| go-linter: | |
| strategy: | |
| fail-fast: false | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - uses: actions/setup-go@v4 | |
| with: | |
| go-version: 'stable' | |
| - name: golangci-lint | |
| uses: golangci/golangci-lint-action@v3 | |
| with: | |
| # Pin the version in case all the builds start to fail at the same time. | |
| # There may not be an automatic way (e.g., dependabot) to update a specific parameter of a GitHub Action, | |
| # so we will just update it manually whenever it makes sense (e.g., a feature that we want is added). | |
| version: v1.54.0 | |
| args: --fix=false --timeout=5m --out-format=colored-line-number | |
| unit-tests: | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| go: [ '1.20', '1.21' ] | |
| os: [ ubuntu-latest, windows-latest ] | |
| name: Unit Tests / ${{ matrix.os }} / Go ${{ matrix.go }} | |
| runs-on: ${{ matrix.os }} | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - uses: actions/setup-go@v4 | |
| with: | |
| go-version: ${{ matrix.go }} | |
| cache: false | |
| - name: build | |
| run: make build | |
| - name: test-unit | |
| run: make test-unit | |
| e2e-tests-for-awslogs: | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| go: [ '1.20', '1.21' ] | |
| os: [ ubuntu-latest ] # TODO: Add Windows e2e tests: https://github.com/aws/shim-loggers-for-containerd/issues/68 | |
| name: E2E tests / awslogs / ${{ matrix.os }} / Go ${{ matrix.go }} | |
| runs-on: ${{ matrix.os }} | |
| permissions: | |
| id-token: write | |
| contents: read | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - uses: actions/setup-go@v4 | |
| with: | |
| go-version: ${{ matrix.go }} | |
| cache: false | |
| - name: Start LocalStack # Copy from https://docs.localstack.cloud/user-guide/ci/github-actions/ | |
| run: | | |
| pip install localstack awscli-local[ver1] # install LocalStack cli and awslocal | |
| docker pull localstack/localstack # Make sure to pull the latest version of the image | |
| localstack start -d # Start LocalStack in the background | |
| echo "Waiting for LocalStack startup..." # Wait 30 seconds for the LocalStack container | |
| localstack wait -t 30 # to become ready before timing out | |
| echo "Startup complete" | |
| awslocal logs create-log-group --log-group-name test-shim-logger | |
| - name: install and start containerd | |
| shell: bash | |
| run: sudo scripts/install-containerd | |
| - name: start ecs local endpoint | |
| shell: bash | |
| run: scripts/start-ecs-local-endpoint | |
| - name: ip forwarding # awslogs driver hardcodes "169.254.170.2" as the aws credential endpoint ip so need to forward to local endpoint | |
| shell: bash | |
| run: sudo scripts/ip-forwarding | |
| - name: build | |
| run: sudo make build | |
| - name: test-e2e | |
| run: sudo -E make test-e2e-for-awslogs # containerd interaction requires sudo and aws cloudwatch interaction requires passing env vars | |
| e2e-tests-for-fluentd: | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| go: [ '1.20', '1.21' ] | |
| os: [ ubuntu-latest ] # TODO: Add Windows e2e tests: https://github.com/aws/shim-loggers-for-containerd/issues/68 | |
| name: E2E tests / fluentd / ${{ matrix.os }} / Go ${{ matrix.go }} | |
| runs-on: ${{ matrix.os }} | |
| permissions: | |
| id-token: write | |
| contents: write | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - uses: actions/setup-go@v4 | |
| with: | |
| go-version: ${{ matrix.go }} | |
| cache: false | |
| - name: install and start containerd | |
| shell: bash | |
| run: sudo scripts/install-containerd | |
| - name: start fluentd local endpoint | |
| shell: bash | |
| run: | | |
| : # not using github action env because env map cannot be defined in terms of other vars in the map. public.ecr.aws/docker/library/fluentd:v1.16-debian-1 | |
| FLUENTD_LOG_DIR=${GITHUB_WORKSPACE}/fluentd-logs | |
| FLUENTD_PORT=24224 | |
| FLUENTD_IMAGE=public.ecr.aws/docker/library/fluentd:v1.16-debian-1 | |
| : # ECR Public anonymous pull rate quota is 1 per sec. https://docs.aws.amazon.com/AmazonECR/latest/public/public-service-quotas.html. | |
| : # Running multiple jobs at the same time may exceed rate limit. Retry 3 times to prevent flaky behavior. | |
| count=0 | |
| until [ $count -ge 3 ] | |
| do | |
| docker pull $FLUENTD_IMAGE && break | |
| count=$((count+1)) | |
| [ $count -eq 3 ] && echo "Pull command failed after 3 attempts" && exit 1 | |
| sleep 3 | |
| done | |
| : # Fluentd container is not using root user so need 777 to make it writable. https://docs.github.com/en/actions/using-github-hosted-runners/about-github-hosted-runners#docker-container-filesystem | |
| sudo mkdir -m 777 $FLUENTD_LOG_DIR | |
| docker run -d -p $FLUENTD_PORT:24224 -p $FLUENTD_PORT:24224/udp -v $FLUENTD_LOG_DIR:/fluentd/log $FLUENTD_IMAGE | |
| - name: build | |
| run: sudo make build | |
| - name: test-e2e | |
| run: sudo make test-e2e-for-fluentd # containerd interaction requires sudo | |
| e2e-tests-for-splunk: | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| go: [ '1.20', '1.21' ] | |
| os: [ ubuntu-latest ] # TODO: Add Windows e2e tests: https://github.com/aws/shim-loggers-for-containerd/issues/68 | |
| name: E2E tests / splunk / ${{ matrix.os }} / Go ${{ matrix.go }} | |
| runs-on: ${{ matrix.os }} | |
| permissions: | |
| id-token: write | |
| contents: write | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - uses: actions/setup-go@v4 | |
| with: | |
| go-version: ${{ matrix.go }} | |
| cache: false | |
| - name: install and start containerd | |
| shell: bash | |
| run: sudo scripts/install-containerd | |
| - name: start splunk local endpoint | |
| id: splunk-local-endpoint | |
| shell: bash | |
| run: | | |
| SPLUNK_PASSWORD=1234567Aa! | |
| SPLUNK_PORT=8089 | |
| SPLUNK_IMAGE=splunk/splunk | |
| CONTAINER_ID=$(docker run -d -p $SPLUNK_PORT:8089 -p 8088:8088 \ | |
| -e "SPLUNK_START_ARGS=--accept-license" \ | |
| -e "SPLUNK_PASSWORD=$SPLUNK_PASSWORD" \ | |
| -e "SPLUNK_HEC_TOKEN=abcd1234" \ | |
| $SPLUNK_IMAGE) | |
| : # Splunk can only receives requests after becoming healthy | |
| until [ $(docker inspect $CONTAINER_ID --format ‘{{.State.Health.Status}}’) == ‘healthy’ ]; do sleep 10s; done; | |
| curl -L -k -u admin:$SPLUNK_PASSWORD \ | |
| -X POST https://localhost:$SPLUNK_PORT/services/admin/token-auth/tokens_auth \ | |
| -d disabled=false | |
| : # Wait for token to be enabled | |
| sleep 5 | |
| TOKEN_OUTPUT=$(curl -L -k -u admin:$SPLUNK_PASSWORD \ | |
| -X POST https://localhost:$SPLUNK_PORT/services/authorization/tokens?output_mode=json \ | |
| -d name=admin -d audience=tests) | |
| SPLUNK_TOKEN=$(echo $TOKEN_OUTPUT | jq -r '.entry[0].content.token') | |
| echo "SPLUNK_TOKEN=$SPLUNK_TOKEN" >> $GITHUB_OUTPUT | |
| - name: build | |
| run: sudo make build | |
| - name: test-e2e | |
| run: | | |
| sudo SPLUNK_TOKEN=${{ steps.splunk-local-endpoint.outputs.SPLUNK_TOKEN }} make test-e2e-for-splunk # containerd interaction requires sudo | |
| go-mod-tidy-check: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - uses: actions/setup-go@v4 | |
| with: | |
| go-version-file: go.mod | |
| # TODO: Use `go mod tidy --check` after https://github.com/golang/go/issues/27005 is fixed. | |
| - run: go mod tidy | |
| - run: git diff --exit-code | |
| mdlint: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - uses: avto-dev/markdown-lint@v1 | |
| with: | |
| args: '**/*.md' | |