validate.yml

name: Tests
on:
  pull_request:
    branches: ['**']
  merge_group:
    branches: [main]
  push:
    branches: [main]
concurrency:
  group: ${{ github.workflow }}-${{ github.event.number || ((github.event_name == 'push' && github.sha) || github.ref) }}
  cancel-in-progress: true

permissions: read-all

env:
  # Make sure we're actually testing with the intended Go release (i.e, ensure
  # no automatic toolchain download happens).
  GOTOOLCHAIN: local

jobs:
  ##############################################################################
  # Run all the code generators; and refresh the LICENSES-3rdparty.csv file
  generate:
    needs: coverage-preflight
    runs-on: ubuntu-latest
    name: Run all generators
    outputs:
      has-patch: ${{ steps.is-tree-dirty.outputs.result }}
    steps:
      - name: Checkout
        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4
      - name: Setup go
        id: setup-go
        uses: actions/setup-go@41dfa10bad2bb2ae585af6ee5bb4d7d973ad74ed # v5
        with:
          go-version: stable
          cache-dependency-path: '**/go.mod'

      - name: Run 'go generate ./...'
        run: |-
          mkdir -p ${GOCOVERDIR}
          find . -name go.mod -execdir go generate ./... \;
        env:
          GOFLAGS: -cover -covermode=atomic -coverpkg=github.com/DataDog/orchestrion/...,./...
          GOCOVERDIR: ${{ github.workspace }}/coverage
      - name: Consolidate coverage report
        if: github.event_name != 'merge_group' && !(github.event_name == 'pull_request' && github.event.pull_request.head.repo.fork)
        run: go tool covdata textfmt -i ./coverage -o ./coverage/generator.out
      - name: Determine simple go version
        if: github.event_name != 'merge_group' && !(github.event_name == 'pull_request' && github.event.pull_request.head.repo.fork)
        id: go
        run: |-
          set -euo pipefail
          echo "version=$(echo '${{ steps.setup-go.outputs.go-version }}' | cut -d'.' -f1,2)" >> "${GITHUB_OUTPUT}"
      - name: Upload coverage report
        if: github.event_name != 'merge_group' && !(github.event_name == 'pull_request' && github.event.pull_request.head.repo.fork)
        uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4
        with:
          name: coverage-generators+go${{ steps.go.outputs.version }}+${{ runner.os }}+${{ runner.arch }}
          path: ./coverage/generator.out
      - name: Run 'go mod tidy'
        # Don't run for push, it's not necessary
        if: github.event_name != 'push'
        run: find . -name go.mod -execdir go mod tidy \;

      - name: Refresh LICENSE-3rdparty.csv
        run: ./_tools/make-licenses.sh
        env:
          TMPDIR: ${{ runner.temp }}

      - name: Check if working tree is dirty
        # Don't run for push, it's not necessary
        if: github.event_name != 'push'
        id: is-tree-dirty
        run: |-
          set -euxo pipefail
          git add .
          git status
          git diff --staged --patch --exit-code > .repo.patch || echo 'result=true' >> "${GITHUB_OUTPUT}"
      - name: Upload patch
        if: github.event_name != 'push' && steps.is-tree-dirty.outputs.result == 'true'
        uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4
        with:
          if-no-files-found: error
          include-hidden-files: true
          name: repo.patch
          path: .repo.patch
      - name: Fail build if working tree is dirty
        if: github.event_name != 'push' && steps.is-tree-dirty.outputs.result == 'true'
        run: |-
          echo "::error::Files have been modified by 'go generate ./...' (see logs)."
          cat .repo.patch
          exit 1
  ##############################################################################
  # If the generators changed anything, and we can update the PR, then we'll
  # proactively do it with the mutator token.
  self-mutation:
    needs: generate
    runs-on: ubuntu-latest
    name: Update PR with generated files
    if: always() && needs.generate.outputs.has-patch == 'true' && github.event_name == 'pull_request' && (github.event.pull_request.head.repo.full_name == github.repository || github.event.pull_request.maintainer_can_modify)
    steps:
      - name: Checkout
        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4
        with:
          ref: ${{ github.event.pull_request.head.ref }}
          repository: ${{ github.event.pull_request.head.repo.full_name }}
      - name: Download patch
        uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4
        with:
          name: repo.patch
          path: ${{ runner.temp }}
      - name: Apply patch
        run: |-
          [ -s '${{ runner.temp }}/.repo.patch' ] && git apply '${{ runner.temp }}/.repo.patch' || echo 'Empty patch. Skipping.'
      # We use ghcommit to create signed commits directly using the GitHub API
      - name: Push changes
        uses: planetscale/ghcommit-action@d4176bfacef926cc2db351eab20398dfc2f593b5 # v0.2.0
        with:
          commit_message: "chore: update generated files"
          repo: ${{ github.event.pull_request.head.repo.full_name }}
          branch: ${{ github.event.pull_request.head.ref }}
        env:
          GITHUB_TOKEN: ${{ secrets.MUTATOR_GITHUB_TOKEN }}

  ##############################################################################
  # Run the various linters we have set up...
  lint:
    needs: generate
    runs-on: ubuntu-latest
    name: Go Linters
    steps:
      - name: Checkout
        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4
      - name: Setup go
        uses: actions/setup-go@41dfa10bad2bb2ae585af6ee5bb4d7d973ad74ed # v5
        with:
          go-version: stable
          cache-dependency-path: "**/go.mod"
      - name: Lint main module
        uses: golangci/golangci-lint-action@971e284b6050e8a5849b72094c50ab08da042db8 # v6
        with:
          version: v1.60.3
      - name: Lint integration tests
        uses: golangci/golangci-lint-action@971e284b6050e8a5849b72094c50ab08da042db8 # v6
        with:
          version: v1.60.3
          working-directory: _integration-tests
      - name: Verify license headers
        run: go run ./_tools/headercheck/header_check.go
      - name: vet
        run: go vet ./...

  ##############################################################################
  # Verify all GitHub workflows have hash-pinned actions
  lint-workflows:
    runs-on: ubuntu-latest
    name: GitHub Workflow Linters
    steps:
      - name: Checkout
        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4
      - name: Ensure SHA pinned actions
        uses: zgosalvez/github-actions-ensure-sha-pinned-actions@5d6ac37a4cef8b8df67f482a8e384987766f0213 # v3

  ##############################################################################
  # Run all unit tests with coverage enabled
  unit-tests:
    needs: generate
    runs-on: ubuntu-latest
    strategy:
      fail-fast: ${{ github.event_name == 'merge_group' }}
      matrix:
        go-version: [oldstable, stable]
    name: Unit tests (go ${{ matrix.go-version }})
    steps:
      - name: Checkout
        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4
      - name: Setup Go
        id: setup-go
        uses: actions/setup-go@41dfa10bad2bb2ae585af6ee5bb4d7d973ad74ed # v5
        with:
          go-version: ${{ matrix.go-version }}
          cache-dependency-path: "**/go.mod"
      - name: Run unit tests
        shell: bash
        run: |-
          mkdir -p coverage
          test_args=("-shuffle=on" "-race" "-v")
          if [ "${{ github.event_name }}" != "merge_group" ]; then
            test_args+=("-cover" "-covermode=atomic" "-coverpkg=./...,github.com/DataDog/orchestrion/..." "-coverprofile=${{ github.workspace }}/coverage/unit.out")
          fi
          go test "${test_args[@]}" ./...
      - name: Run integraton suite unit tests
        shell: bash
        run: |-
          mkdir -p coverage
          test_args=("-shuffle=on" "-race" "-v")
          if [ "${{ github.event_name }}" != "merge_group" ]; then
            test_args+=("-cover" "-covermode=atomic" "-coverpkg=./...,github.com/DataDog/orchestrion/..." "-coverprofile=${{ github.workspace }}/coverage/integration.out")
          fi
          go -C _integration-tests test "${test_args[@]}" ./...
      - name: Determine simple go version
        if: github.event_name != 'merge_group' && !(github.event_name == 'pull_request' && github.event.pull_request.head.repo.fork)
        id: go
        shell: bash
        run: |-
          set -euo pipefail
          echo "version=$(echo '${{ steps.setup-go.outputs.go-version }}' | cut -d'.' -f1,2)" >> "${GITHUB_OUTPUT}"
      - name: Upload coverage report
        if: github.event_name != 'merge_group' && !(github.event_name == 'pull_request' && github.event.pull_request.head.repo.fork)
        uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4
        with:
          name: coverage-unit+go${{ steps.go.outputs.version }}+${{ runner.os }}+${{ runner.arch }}
          path: |-
            ./coverage/unit.out
            ./coverage/integration.out

  ##############################################################################
  # Run all benchmarks and generate report
  benchmark:
    needs: generate
    runs-on: arm-8core-linux
    name: Benchmarks
    steps:
      - name: Checkout
        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4
      - name: Setup go
        id: setup-go
        uses: actions/setup-go@41dfa10bad2bb2ae585af6ee5bb4d7d973ad74ed # v5
        with:
          go-version: stable
          cache-dependency-path: "**/go.mod"
      - name: Run benchmarks
        run: |-
          set -euo pipefail
          go test -bench=. -timeout=1h -run=^$ . | tee ${{ runner.temp }}/benchmarks.txt
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
      - name: Upload benchmark report (raw)
        uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4
        with:
          if-no-files-found: error
          name: Benchmark Report
          path: ${{ runner.temp }}/benchmarks.txt
      - name: Format Report
        run: |-
          go -C _tools run golang.org/x/perf/cmd/benchstat  \
            -table=.name -row=/repo -col=/variant \
            ${{ runner.temp }}/benchmarks.txt     \
            | tee ${{ runner.temp }}/benchmarks-formatted.txt
      - name: Setting Job Summary
        run: |-
          echo "### Benchmark Report"                     >> "${GITHUB_STEP_SUMMARY}"
          echo '```'                                      >> "${GITHUB_STEP_SUMMARY}"
          cat ${{ runner.temp }}/benchmarks-formatted.txt >> "${GITHUB_STEP_SUMMARY}"
          echo '```'                                      >> "${GITHUB_STEP_SUMMARY}"


  ##############################################################################
  # Run all integration tests and gather extensive coverage
  integration-tests:
    needs: generate
    strategy:
      fail-fast: ${{ github.event_name == 'merge_group' }}
      matrix:
        runs-on:
          - macos
          - ubuntu
          # - windows # Temporarily disabled because excruciatingly slow for no obvious reason.
        go-version: [oldstable, stable]
        build-mode: [DRIVER]
        include:
          # Alternate build modes (only on ubuntu, latest go; to save CI time)
          - runs-on: ubuntu
            go-version: oldstable
            build-mode: TOOLEXEC
          - runs-on: ubuntu
            go-version: oldstable
            build-mode: GOFLAGS
    runs-on: ${{ matrix.runs-on == 'ubuntu' && fromJson('{"labels":"ubuntu-16-core-latest","group":"Large Runner Shared Public"}') || format('{0}-latest', matrix.runs-on) }}
    env:
      # Ryuk is problematic with concurrent executions, and unnecessary in ephemeral environments like GHA.
      TESTCONTAINERS_RYUK_DISABLED: true
    name: Integration tests (go ${{ matrix.go-version }}, ${{ matrix.runs-on }}, ${{ matrix.build-mode }})
    steps:
      - name: Checkout
        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4
      - name: Setup go
        id: setup-go
        uses: actions/setup-go@41dfa10bad2bb2ae585af6ee5bb4d7d973ad74ed # v5
        with:
          go-version: ${{ matrix.go-version }}
          cache-dependency-path: "**/go.mod"
      - name: Setup python
        uses: actions/setup-python@0b93645e9fea7318ecaed2b359559ac225c90a2b # v5
        with:
          python-version: 3.x
          cache: pip
          cache-dependency-path: _integration-tests/utils/agent/requirements-dev.txt
      - name: Install python dependencies
        run: pip install -r _integration-tests/utils/agent/requirements-dev.txt
      - name: Build orchestrion binary
        shell: bash
        run: |-
          build_args=()
          if [ "${{ github.event_name }}" != "merge_group" ]; then
            build_args+=("-cover" "-covermode=atomic" "-coverpkg=./...")
          fi
          go build ${build_args[@]+"${build_args[@]}"} -o="bin/orchestrion.exe" .
      - name: Run Integration Tests
        shell: bash
        run: |-
          mkdir -p "${GOCOVERDIR}"
          test_args=("-shuffle=on")
          if [ "${{ github.event_name }}" != "merge_group" ]; then
            test_args+=("-coverpkg=./...,github.com/DataDog/orchestrion/..." "-covermode=atomic" "-cover" "-coverprofile=${{ github.workspace }}/coverage/integration.run.out")
          fi

          case "${{ matrix.build-mode }}" in
          "DRIVER")
            bin/orchestrion.exe -C=_integration-tests go test "${test_args[@]}" -a ./...
            ;;
          "TOOLEXEC")
            go -C=_integration-tests test "${test_args[@]}" -toolexec="${{ github.workspace }}/bin/orchestrion.exe toolexec" ./...
            ;;
          "GOFLAGS")
            export GOFLAGS="'-toolexec=${{ github.workspace }}/bin/orchestrion.exe toolexec' ${GOFLAGS}"
            go -C=_integration-tests test "${test_args[@]}" ./...
            ;;
          *)
            echo "Unknown build mode: ${{ matrix.build-mode }}"
            exit 1
            ;;
          esac
        env:
          GOCOVERDIR: ${{ github.workspace }}/coverage/raw
          GOFLAGS: ${{ matrix.runs-on == 'ubuntu' && '-p=4' || ''}} -tags=integration,buildtag # Globally set build tags (buildtag is used by the dd-span test)
      - name: Consolidate coverage report
        if: github.event_name != 'merge_group' && !(github.event_name == 'pull_request' && github.event.pull_request.head.repo.fork)
        run: go tool covdata textfmt -i ./coverage/raw -o ./coverage/integration.out
      - name: Determine go minor version
        id: go
        if: github.event_name != 'merge_group' && !(github.event_name == 'pull_request' && github.event.pull_request.head.repo.fork)
        shell: bash
        run: |-
          set -euo pipefail
          echo "version=$(echo '${{ steps.setup-go.outputs.go-version }}' | cut -d'.' -f1,2)" >> "${GITHUB_OUTPUT}"
      - name: Upload coverage report
        if: github.event_name != 'merge_group' && !(github.event_name == 'pull_request' && github.event.pull_request.head.repo.fork)
        uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4
        with:
          name: coverage-integration+${{ matrix.build-mode }}+go${{ steps.go.outputs.version }}+${{ runner.os }}+${{ runner.arch }}
          path: |-
            ./coverage/integration.out
            ./coverage/integration.run.out

  ##############################################################################
  # Assert everything is complete. This simplifies branch protection settings
  # and allows us to have one single trigger for CodeCov reporting.
  complete:
    runs-on: ubuntu-latest
    name: Complete
    needs:
      - generate
      - lint
      - lint-workflows
      - unit-tests
      - integration-tests
      - benchmark
    if: '!cancelled()'
    steps:
      - name: Success
        if: needs.generate.result != 'failure' && needs.lint.result != 'failure' && needs.lint-workflows.result != 'failure' && needs.unit-tests.result != 'failure' && needs.integration-tests.result != 'failure'
        run: echo "OK"
      - name: Failed
        if: needs.generate.result == 'failure' || needs.lint.result == 'failure' || needs.lint-workflows.result == 'failure' || needs.unit-tests.result == 'failure' || needs.integration-tests.result == 'failure'
        run: |-
          echo "Failed!"
          exit 1

  ##############################################################################
  # Produce a CodeCov coverage report with all uploaded code coverage data.
  coverage-preflight:
    runs-on: ubuntu-latest
    name: CodeCov pre-flight
    steps:
      - name: Checkout
        if: github.event_name != 'merge_group' && !(github.event_name == 'pull_request' && github.event.pull_request.head.repo.fork)
        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4
      - name: Download codecov CLI
        id: codecov-cli
        if: github.event_name != 'merge_group' && !(github.event_name == 'pull_request' && github.event.pull_request.head.repo.fork)
        uses: ./.github/actions/codecov-cli
      - name: Register commit with CodeCov
        if: github.event_name != 'merge_group' && !(github.event_name == 'pull_request' && github.event.pull_request.head.repo.fork)
        shell: bash
        env:
          CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
        run: |-
          set -euo pipefail

          pr=()
          sha="${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}"
          parentsha="${{ github.event_name == 'pull_request' && github.event.pull_request.base.sha || github.event.before }}"
          if [ "${{ github.event_name }}" == "pull_request" ]; then
            pr+=("--pr=${{ github.event.number }}")
          fi

          echo "::group::Register commit metadata with CodeCov"
          ${{ steps.codecov-cli.outputs.codecov }}                              \
            --auto-load-params-from=GithubActions                               \
            --verbose                                                           \
            create-commit                                                       \
            --parent-sha="${parentsha}"                                         \
            ${pr[@]+"${pr[@]}"}                                                 \
            --sha="${sha}"                                                      \
            --fail-on-error                                                     \
            --git-service=github                                                \
            --token="${CODECOV_TOKEN}"                                          \
            --slug="${{ github.repository }}"
          echo "::endgroup::"

          echo "::group::Create a new blank CodeCov report"
          ${{ steps.codecov-cli.outputs.codecov }}                              \
            --auto-load-params-from=GithubActions                               \
            --verbose                                                           \
            create-report                                                       \
            ${pr[@]+"${pr[@]}"}                                                 \
            --sha="${sha}"                                                      \
            --fail-on-error                                                     \
            --git-service=github                                                \
            --token="${CODECOV_TOKEN}"                                          \
            --slug="${{ github.repository }}"
          echo "::endgroup::"

  coverage-matrix:
    runs-on: ubuntu-latest
    name: Compute Coverage Matrix
    needs:
      - coverage-preflight
      - unit-tests
      - integration-tests
    if: github.event_name != 'merge_group' && !(github.event_name == 'pull_request' && github.event.pull_request.head.repo.fork)
    outputs:
      artifacts: ${{ steps.compute.outputs.artifacts }}
      files: ${{ steps.compute.outputs.files }}
      matrix: ${{ steps.compute.outputs.matrix }}
    steps:
      - name: Setup Node
        uses: actions/setup-node@39370e3970a6d050c480ffad4ff0ed4d3fdee5af # v4
        with:
          node-version: latest
      - name: Download Artifacts
        uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4
        with:
          pattern: coverage-*
      - name: Compute Matrix
        id: compute
        run: |-
          node <<-EOF
            const fs = require('node:fs');
            const path = require('node:path');
            const process = require('node:process');

            const flags = [];
            const flagFiles = {};
            for (const dirname of fs.readdirSync(process.cwd())) {
              const prefix = 'coverage-';
              if (!dirname.startsWith(prefix)) {
                continue;
              }

              const files = fs.globSync(path.join(process.cwd(), dirname, '**', '*.out'));
              console.log('Found asset named ' + dirname + ' with ' + files.length + ' report files.');
              if (files.length == 0) {
                continue;
              }

              for (const flag of dirname.substring(prefix.length).split('+')) {
                if (!flags.includes(flag)) {
                  flags.push(flag);
                }

                flagFiles[flag] ??= [];
                flagFiles[flag].push(...files);
              }
            }

            console.log('Flags:', flags);
            console.log('Files:', flagFiles);

            // Join the lists because the workflow subsequently expects a whitespace-separted list.
            for (const [flag, list] of Object.entries(flagFiles)) {
              flagFiles[flag] = list.join(' ');
            }

            fs.writeFileSync(
              path.join(process.env.GITHUB_OUTPUT),
              [
                "matrix=" + JSON.stringify({ flag: flags }),
                "files=" + JSON.stringify(flagFiles),
              ].join('\n'),
            );
          EOF


  coverage-upload:
    runs-on: ubuntu-latest
    name: Upload report to CodeCov (${{ matrix.flag }})
    needs: [coverage-matrix]
    if: github.event_name != 'merge_group' && !(github.event_name == 'pull_request' && github.event.pull_request.head.repo.fork)
    strategy:
      fail-fast: true
      matrix: ${{ fromJson(needs.coverage-matrix.outputs.matrix) }}
    steps:
      - name: Checkout
        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4
      - name: Download Artifacts
        uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4
        with:
          pattern: coverage-*
      - name: Upload Reports
        uses: ./.github/actions/codecov-upload
        with:
          name: ${{ matrix.flag }}
          flags: ${{ matrix.flag }}
          files: ${{ fromJson(needs.coverage-matrix.outputs.files)[matrix.flag] }}
          token: ${{ secrets.CODECOV_TOKEN }}


  coverage-finalize:
    runs-on: ubuntu-latest
    name: Create CodeCov report
    needs: [coverage-upload]
    if: github.event_name != 'merge_group' && !(github.event_name == 'pull_request' && github.event.pull_request.head.repo.fork)
    steps:
      - name: Checkout
        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4
      - name: Download codecov CLI
        id: codecov-cli
        uses: ./.github/actions/codecov-cli
      - name: Create CodeCov report
        shell: bash
        env:
          CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
        run: |-
          set -euo pipefail
          sha="${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}"

          echo "::group::Create CodeCov report results"
          ${{ steps.codecov-cli.outputs.codecov }}                              \
            --auto-load-params-from=GithubActions                               \
            --verbose                                                           \
            create-report-results                                               \
            --sha="${sha}"                                                      \
            --fail-on-error                                                     \
            --git-service=github                                                \
            --token="${CODECOV_TOKEN}"                                          \
            --slug="${{ github.repository }}"
          echo "::endgroup::"

          echo "::group::Issue GitHub notifications"
          ${{ steps.codecov-cli.outputs.codecov }}                              \
            --auto-load-params-from=GithubActions                               \
            --verbose                                                           \
            send-notifications                                                  \
            --sha="${sha}"                                                      \
            --fail-on-error                                                     \
            --git-service=github                                                \
            --token="${CODECOV_TOKEN}"                                          \
            --slug=${{ github.repository }}
          echo "::endgroup::"