Merge pull request #438 from overmindtech/e2e-tests

E2E tests for the most commonly used CLI commands

.github/e2eapply.tape

@@ -0,0 +1,18 @@
+Output apply.mp4
+
+Set Width 1920
+Set Height 1080
+
+Set FontSize 12
+
+Hide
+Type "export PATH=$PWD:$PATH"
+Enter
+Type "clear"
+Enter
+Show
+
+Type@1ms "overmind terraform apply -- tfplan"
+Enter
+Sleep 60
+Screenshot apply.png

.github/e2eplan.tape

@@ -0,0 +1,20 @@
+Output plan.mp4
+
+Set Width 1920
+Set Height 1080
+
+Set FontSize 12
+
+Hide
+Type "export PATH=$PWD:$PATH"
+Enter
+Type "clear"
+Enter
+Show
+
+Type@1ms "overmind terraform plan -- -out tfplan"
+Enter
+Sleep 2
+Enter
+Sleep 60
+Screenshot plan.png

.github/workflows/e2e.yaml

@@ -0,0 +1,170 @@
+name: E2E Tests
+on:
+  push:
+    branches:
+      - main
+      - e2e-tests
+
+jobs:
+  actions:
+    runs-on: depot-ubuntu-22.04-4
+    env:
+      CGO_ENABLED: 0
+    permissions:
+      contents: read # required for checkout
+      id-token: write # mint AWS credentials through OIDC
+
+    steps:
+      - name: Cache Terraform Providers
+        id: cache-terraform
+        uses: actions/cache@v3
+        with:
+          path: .terraform
+          key: ${{ runner.os }}-${{ inputs.environment }}-${{ hashFiles('**/.terraform.lock.hcl') }}
+
+      - uses: hashicorp/setup-terraform@v2
+        with:
+          terraform_version: 1.5.7
+          terraform_wrapper: false
+
+      - name: Configure AWS Credentials
+        uses: aws-actions/configure-aws-credentials@v4
+        with:
+          audience: sts.amazonaws.com # set default value to work around https://github.com/aws-actions/configure-aws-credentials/issues/271#issuecomment-1401481855
+          aws-region: eu-west-2
+          role-to-assume: arn:aws:iam::211125377101:role/terraform-ci-test
+
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: Go Init
+        uses: ./.github/actions/go_init
+
+      - name: Build the binary
+        run: |
+          set -xe
+          go get -v -t -d ./...
+          go build -o overmind main.go
+
+      - name: Run E2E Tests
+        env:
+          OVM_API_KEY: ${{ secrets.OVM_API_KEY }}
+          HONEYCOMB_API_KEY: ${{ secrets.HONEYCOMB_API_KEY }}
+        run: |
+          set -xe
+
+          ./overmind -v
+
+          terraform init
+          terraform plan -out=tfplan
+          terraform show -json tfplan > tfplan.json
+
+          ticket_link='${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}'
+
+          ./overmind changes submit-plan \
+            --title "Test Change for e2e tests" \
+            --description "some description" \
+            --ticket-link "$ticket_link" \
+            tfplan.json \
+            > ./overmindtech-change-url
+
+          ./overmind changes get-change \
+            --change "$(< ./overmindtech-change-url)" \
+            --format markdown \
+            > ./overmindtech-message
+
+          ./overmind changes start-change \
+            --ticket-link "$ticket_link"
+
+          ./overmind changes end-change \
+            --ticket-link "$ticket_link"
+
+          echo "E2E Tests Complete"
+
+  interactive:
+    runs-on: depot-ubuntu-22.04-4
+    env:
+      CGO_ENABLED: 0
+    permissions:
+      contents: read # required for checkout
+      id-token: write # mint AWS credentials through OIDC
+
+    steps:
+      - name: Cache Terraform Providers
+        id: cache-terraform
+        uses: actions/cache@v3
+        with:
+          path: .terraform
+          key: ${{ runner.os }}-${{ inputs.environment }}-${{ hashFiles('**/.terraform.lock.hcl') }}
+
+      - uses: hashicorp/setup-terraform@v2
+        with:
+          terraform_version: 1.5.7
+          terraform_wrapper: false
+
+      - name: Configure AWS Credentials
+        uses: aws-actions/configure-aws-credentials@v4
+        with:
+          audience: sts.amazonaws.com # set default value to work around https://github.com/aws-actions/configure-aws-credentials/issues/271#issuecomment-1401481855
+          aws-region: eu-west-2
+          role-to-assume: arn:aws:iam::211125377101:role/terraform-ci-test
+
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: Go Init
+        uses: ./.github/actions/go_init
+
+      - name: Build the binary
+        run: |
+          set -xe
+          go get -v -t -d ./...
+          go build -o overmind main.go
+
+      - name: Run E2E Tests
+        env:
+          OVM_API_KEY: ${{ secrets.OVM_API_KEY }}
+          HONEYCOMB_API_KEY: ${{ secrets.HONEYCOMB_API_KEY }}
+          TEABUG: pls
+        run: |
+          set -xe
+
+          go build -o overmind main.go
+
+          ./overmind -v
+
+          terraform init
+
+          cp -a $(which terraform) . # provide a terraform binary to the containers below
+
+          docker run --rm -e AWS_ACCESS_KEY_ID -e AWS_SECRET_ACCESS_KEY -e AWS_DEFAULT_REGION -e AWS_REGION -e AWS_SESSION_TOKEN -e HONEYCOMB_API_KEY -e OVM_API_KEY -e TEABUG -v $PWD:/vhs -v ~/.aws:/root/.aws ghcr.io/charmbracelet/vhs /vhs/.github/e2eplan.tape
+          docker run --rm -e AWS_ACCESS_KEY_ID -e AWS_SECRET_ACCESS_KEY -e AWS_DEFAULT_REGION -e AWS_REGION -e AWS_SESSION_TOKEN -e HONEYCOMB_API_KEY -e OVM_API_KEY -e TEABUG -v $PWD:/vhs -v ~/.aws:/root/.aws ghcr.io/charmbracelet/vhs /vhs/.github/e2eapply.tape
+
+          sudo chown --recursive $(whoami) .
+
+          terraform apply -destroy -auto-approve
+
+          echo "E2E Tests Complete"
+
+      - name: debug output
+        if: always()
+        run: |
+          cat teabug.log
+
+      - name: Upload plan results
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: plan.mp4
+          path: plan.*
+
+      - name: Upload plan results
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: apply.mp4
+          path: apply.*

.github/workflows/tests.yml

@@ -3,7 +3,7 @@ on: push
 jobs:
   test:
     name: Run Tests
-    runs-on: ubuntu-latest
+    runs-on: depot-ubuntu-22.04-4
     env:
       CGO_ENABLED: 0
 
@@ -27,7 +27,7 @@ jobs:
 
   golangci:
     name: lint
-    runs-on: ubuntu-latest
+    runs-on: depot-ubuntu-22.04-4
 
     steps:
       - name: Checkout

cmd/tea.go

@@ -107,7 +107,13 @@ func CmdWrapper(action string, requiredScopes []string, commandModel func(args [
 				tasks:          map[string]tea.Model{},
 			}
 			m.cmd = commandModel(args, &m, m.width)
-			p := tea.NewProgram(&m)
+
+			options := []tea.ProgramOption{}
+			if os.Getenv("CI") != "" {
+				// See https://github.com/charmbracelet/bubbletea/issues/761#issuecomment-1625863769
+				options = append(options, tea.WithInput(nil))
+			}
+			p := tea.NewProgram(&m, options...)
 			result, err := p.Run()
 			if err != nil {
 				return fmt.Errorf("could not start program: %w", err)

cmd/tea_ensuretoken.go

@@ -100,10 +100,10 @@ func (m ensureTokenModel) Update(msg tea.Msg) (tea.Model, tea.Cmd) {
 	case instanceLoadedMsg:
 		m.oi = msg.instance
 		m.status = taskStatusRunning
-		cmds = append(cmds,
-			m.ensureTokenCmd(m.ctx),
-			m.spinner.Tick,
-		)
+		cmds = append(cmds, m.ensureTokenCmd(m.ctx))
+		if os.Getenv("CI") == "" {
+			cmds = append(cmds, m.spinner.Tick)
+		}
 	case displayAuthorizationInstructionsMsg:
 		m.deviceMessage = "manual"
 		m.deviceConfig = msg.config

cmd/tea_initialisesources.go

@@ -92,7 +92,9 @@ func (m initialiseSourcesModel) Update(msg tea.Msg) (tea.Model, tea.Cmd) {
 
 		m.status = taskStatusRunning
 		cmds = append(cmds, m.loadSourcesConfigCmd)
-		cmds = append(cmds, m.spinner.Tick)
+		if os.Getenv("CI") == "" {
+			cmds = append(cmds, m.spinner.Tick)
+		}
 	case askForAwsConfigMsg:
 		// load the config that was injected above. If it's not there, prompt the user.
 		aws_config := viper.GetString("aws-config")

cmd/tea_plan.go

@@ -25,7 +25,9 @@ type runPlanModel struct {
 	taskModel
 }
 type runPlanNowMsg struct{}
-type runPlanFinishedMsg struct{}
+type runPlanFinishedMsg struct {
+	err error
+}
 
 func NewRunPlanModel(args []string, planFile string, parent *cmdModel, width int) runPlanModel {
 	return runPlanModel{
@@ -60,7 +62,9 @@ func (m runPlanModel) Update(msg tea.Msg) (tea.Model, tea.Cmd) {
 		m.taskModel.status = taskStatusRunning
 		// since the taskModel will not be shown while `terraform plan` is running,
 		// there's no need to actually kick off the spinner
-		// cmds = append(cmds, m.taskModel.spinner.Tick)
+		// if os.Getenv("CI") == "" {
+		// 	cmds = append(cmds, m.taskModel.spinner.Tick)
+		// }
 
 		// defer the actual command to give the view a chance to show the header
 		cmds = append(cmds, func() tea.Msg { return runPlanNowMsg{} })
@@ -94,23 +98,23 @@ func (m runPlanModel) Update(msg tea.Msg) (tea.Model, tea.Cmd) {
 
 		_, span := tracing.Tracer().Start(m.ctx, "terraform plan") // nolint:spancheck // will be ended in the tea.Exec cleanup func
 
-		cmds = append(cmds,
-			tea.Sequence(
-				func() tea.Msg { return freezeViewMsg{} },
-				tea.Exec( // nolint:spancheck // will be ended in the tea.Exec cleanup func
-					m.parent.NewExecCommand(c),
-					func(err error) tea.Msg {
-						defer span.End()
+		cmds = append(cmds, Exec(
+			m.parent.NewExecCommand(c),
+			func(err error) tea.Msg {
+				defer span.End()
 
-						if err != nil {
-							return fatalError{err: fmt.Errorf("failed to run terraform plan: %w", err)}
-						}
-						return runPlanFinishedMsg{}
-					})))
+				if err != nil {
+					return runPlanFinishedMsg{err: fmt.Errorf("failed to run terraform plan: %w", err)}
+				}
+				return runPlanFinishedMsg{err: nil}
+			}))
 
 	case runPlanFinishedMsg:
-		m.taskModel.status = taskStatusDone
-		cmds = append(cmds, func() tea.Msg { return unfreezeViewMsg{} })
+		if msg.err != nil {
+			m.taskModel.status = taskStatusError
+		} else {
+			m.taskModel.status = taskStatusDone
+		}
 	}
 
 	var cmd tea.Cmd
@@ -136,7 +140,13 @@ func (m runPlanModel) View() string {
 	case taskStatusDone:
 		bits = append(bits, m.taskModel.View())
 		bits = append(bits, m.revlinkTask.View())
-	case taskStatusError, taskStatusSkipped:
+	case taskStatusError:
+		bits = append(bits,
+			wrap(fmt.Sprintf("%v Running 'terraform %v'",
+				RenderErr(),
+				strings.Join(m.args, " "),
+			), m.width, 2))
+	case taskStatusSkipped:
 		// handled by caller
 	}
 	return strings.Join(bits, "\n")

cmd/tea_revlink.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"errors"
 	"fmt"
+	"os"
 	"time"
 
 	"connectrpc.com/connect"
@@ -58,7 +59,9 @@ func (m revlinkWarmupModel) Update(msg tea.Msg) (revlinkWarmupModel, tea.Cmd) {
 	case sourcesInitialisedMsg:
 		m.taskModel.status = taskStatusRunning
 		// start the spinner
-		cmds = append(cmds, m.taskModel.spinner.Tick)
+		if os.Getenv("CI") == "" {
+			cmds = append(cmds, m.taskModel.spinner.Tick)
+		}
 
 		// setup the watchdog infrastructure
 		ctx, cancel := context.WithCancel(m.ctx)