Skip to content

Commit

Permalink
feat(testworkflows): TKC-1764 - support accompanying services (#5436)
Browse files Browse the repository at this point in the history
* feat(testworkflows): support `services` in the TestWorkflow objects
* feat(testworkflows): support `use` in `services`
* fix(testworkflows): orchestration fixes
* feat(testworkflows): TKC-1764 - add support for accompanying services
* chore: update integration tests to use provided URLs
* chore: update integration tests to use provided Minio URLs
* chore: fix NATS URIs in tests
* chore: hardcode again Minio buckets for tests
* fix: add missing environment variables for integration tests
* fix(testworkflows): cloning the repository
* fix(testworkflows): merging securityContext
* fix(testworkflows): sort out proper group/fsGroup/user for the services
* feat(testworkflows): add example integration tests for Testkube
* fix: adjust make integration-tests to fill the access/secret keys
* chore: update testkube-operator
* feat: expose PWD environment variable for the processing
  • Loading branch information
rangoo94 committed May 16, 2024
1 parent b4271a3 commit 2f8d106
Show file tree
Hide file tree
Showing 52 changed files with 1,635 additions and 129 deletions.
3 changes: 3 additions & 0 deletions .github/workflows/test.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -152,6 +152,9 @@ jobs:
run: sudo apt-get install -y git

- name: Integration Tests
env:
STORAGE_ACCESSKEYID: minio99
STORAGE_SECRETACCESSKEY: minio123
run: INTEGRATION=y gotestsum --format pkgname --junitfile integration-tests.xml --jsonfile integration-tests.json -- -run _Integration -coverprofile=coverage.out -covermode=atomic ./...

- name: Integration Test Summary
Expand Down
2 changes: 1 addition & 1 deletion Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -164,7 +164,7 @@ unit-tests:

.PHONY: integration-tests
integration-tests:
INTEGRATION="true" gotestsum --format pkgname -- -tags=integration -cover ./...
INTEGRATION="true" STORAGE_ACCESSKEYID="minio99" STORAGE_SECRETACCESSKEY="minio123" gotestsum --format pkgname -- -tags=integration -cover ./...

test-e2e:
go test --tags=e2e -v ./test/e2e
Expand Down
124 changes: 124 additions & 0 deletions api/v1/testkube.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -7942,6 +7942,40 @@ components:
items:
$ref: "#/components/schemas/TestWorkflowStepParallelFetch"

TestWorkflowIndependentServiceSpec:
type: object
allOf:
- $ref: "#/components/schemas/TestWorkflowStepExecuteStrategy"
- $ref: "#/components/schemas/TestWorkflowStepRun"
- properties:
timeout:
type: string
description: "maximum time until reaching readiness"
transfer:
type: array
description: list of files to send to parallel steps
items:
$ref: "#/components/schemas/TestWorkflowStepParallelTransfer"
content:
$ref: "#/components/schemas/TestWorkflowContent"
pod:
$ref: "#/components/schemas/TestWorkflowPodConfig"
restartPolicy:
type: string
readinessProbe:
$ref: "#/components/schemas/Probe"

TestWorkflowServiceSpec:
type: object
allOf:
- $ref: "#/components/schemas/TestWorkflowStepExecuteStrategy"
- $ref: "#/components/schemas/TestWorkflowIndependentServiceSpec"
- properties:
use:
type: array
items:
$ref: "#/components/schemas/TestWorkflowTemplateRef"

TestWorkflowStepExecuteStrategy:
type: object
properties:
Expand Down Expand Up @@ -8011,6 +8045,10 @@ components:
$ref: "#/components/schemas/TestWorkflowConfigSchema"
content:
$ref: "#/components/schemas/TestWorkflowContent"
services:
type: object
additionalProperties:
$ref: "#/components/schemas/TestWorkflowServiceSpec"
container:
$ref: "#/components/schemas/TestWorkflowContainerConfig"
job:
Expand Down Expand Up @@ -8041,6 +8079,10 @@ components:
$ref: "#/components/schemas/TestWorkflowConfigSchema"
content:
$ref: "#/components/schemas/TestWorkflowContent"
services:
type: object
additionalProperties:
$ref: "#/components/schemas/TestWorkflowIndependentServiceSpec"
container:
$ref: "#/components/schemas/TestWorkflowContainerConfig"
job:
Expand Down Expand Up @@ -8130,6 +8172,10 @@ components:
description: delay before the step
content:
$ref: "#/components/schemas/TestWorkflowContent"
services:
type: object
additionalProperties:
$ref: "#/components/schemas/TestWorkflowIndependentServiceSpec"
shell:
type: string
description: script to run in a default shell for the container
Expand Down Expand Up @@ -8193,6 +8239,10 @@ components:
description: delay before the step
content:
$ref: "#/components/schemas/TestWorkflowContent"
services:
type: object
additionalProperties:
$ref: "#/components/schemas/TestWorkflowServiceSpec"
shell:
type: string
description: script to run in a default shell for the container
Expand Down Expand Up @@ -9171,6 +9221,80 @@ components:
value:
$ref: "#/components/schemas/BoxedString"

Probe:
type: object
description: "Probe describes a health check to be performed against a container to determine whether it is alive or ready to receive traffic."
properties:
initialDelaySeconds:
type: integer
timeoutSeconds:
type: integer
periodSeconds:
type: integer
successThreshold:
type: integer
failureThreshold:
type: integer
terminationGracePeriodSeconds:
$ref: "#/components/schemas/BoxedInteger"
exec:
$ref: "#/components/schemas/ExecAction"
httpGet:
$ref: "#/components/schemas/HTTPGetAction"
tcpSocket:
$ref: "#/components/schemas/TCPSocketAction"
grpc:
$ref: "#/components/schemas/GRPCAction"

ExecAction:
type: object
properties:
command:
type: array
description: "Command is the command line to execute inside the container, the working directory for the command is root ('/') in the container's filesystem. Exit status of 0 is treated as live/healthy and non-zero is unhealthy."
items:
type: string

HTTPGetAction:
type: object
properties:
path:
type: string
port:
type: string
host:
type: string
scheme:
type: string
httpHeaders:
type: array
items:
$ref: "#/components/schemas/HTTPHeader"

HTTPHeader:
type: object
properties:
name:
type: string
value:
type: string

TCPSocketAction:
type: object
properties:
port:
type: string
host:
type: string

GRPCAction:
type: object
properties:
port:
type: integer
service:
$ref: "#/components/schemas/BoxedString"

VolumeMount:
description: VolumeMount describes a mounting of a Volume
within a container.
Expand Down
6 changes: 6 additions & 0 deletions cmd/tcl/testworkflow-init/data/expressions.go
Original file line number Diff line number Diff line change
Expand Up @@ -71,6 +71,12 @@ var StateMachine = expressionstcl.NewMachine().
return State.GetOutput(name[7:])
}
return nil, false, nil
}).
RegisterAccessorExt(func(name string) (interface{}, bool, error) {
if strings.HasPrefix(name, "services.") {
return State.GetOutput(name)
}
return nil, false, nil
})

var EnvMachine = expressionstcl.NewMachine().
Expand Down
8 changes: 8 additions & 0 deletions cmd/tcl/testworkflow-init/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -111,6 +111,14 @@ func main() {
}
}

// Configure PWD variable, to make it similar to shell environment variables
if os.Getenv("PWD") == "" {
cwd, err := os.Getwd()
if err == nil {
_ = os.Setenv("PWD", cwd)
}
}

// Compute environment variables
for _, name := range computed {
initial := os.Getenv(name)
Expand Down
6 changes: 5 additions & 1 deletion cmd/tcl/testworkflow-toolkit/commands/clone.go
Original file line number Diff line number Diff line change
Expand Up @@ -69,7 +69,11 @@ func NewCloneCmd() *cobra.Command {
// Clone repository
if len(paths) == 0 {
ui.Debug("full checkout")
err = Run("git", "clone", configArgs, authArgs, "--depth", 1, "--verbose", uri.String(), outputPath)
if revision == "" {
err = Run("git", "clone", configArgs, authArgs, "--depth", 1, "--verbose", uri.String(), outputPath)
} else {
err = Run("git", "clone", configArgs, authArgs, "--depth", 1, "--branch", revision, "--verbose", uri.String(), outputPath)
}
ui.ExitOnError("cloning repository", err)
} else {
ui.Debug("sparse checkout")
Expand Down
98 changes: 98 additions & 0 deletions cmd/tcl/testworkflow-toolkit/commands/kill.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,98 @@
// Copyright 2024 Testkube.
//
// Licensed as a Testkube Pro file under the Testkube Community
// License (the "License"); you may not use this file except in compliance with
// the License. You may obtain a copy of the License at
//
// https://github.com/kubeshop/testkube/blob/main/licenses/TCL.txt

package commands

import (
"context"
"fmt"
"os"
"slices"

"github.com/spf13/cobra"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"

"github.com/kubeshop/testkube/cmd/tcl/testworkflow-init/data"
"github.com/kubeshop/testkube/cmd/tcl/testworkflow-toolkit/artifacts"
"github.com/kubeshop/testkube/cmd/tcl/testworkflow-toolkit/common"
"github.com/kubeshop/testkube/cmd/tcl/testworkflow-toolkit/env"
"github.com/kubeshop/testkube/cmd/tcl/testworkflow-toolkit/spawn"
"github.com/kubeshop/testkube/pkg/tcl/testworkflowstcl/testworkflowcontroller"
"github.com/kubeshop/testkube/pkg/tcl/testworkflowstcl/testworkflowprocessor/constants"
"github.com/kubeshop/testkube/pkg/ui"
)

func NewKillCmd() *cobra.Command {
var (
logs []string
)
cmd := &cobra.Command{
Use: "kill <ref>",
Short: "Kill accompanying service(s)",
Args: cobra.ExactArgs(1),

Run: func(cmd *cobra.Command, args []string) {
groupRef := args[0]
clientSet := env.Kubernetes()

// Fast-track
if len(logs) == 0 {
os.Exit(0)
}

// Fetch the services when needed
if len(logs) > 0 {
jobs, err := clientSet.BatchV1().Jobs(env.Namespace()).List(context.Background(), metav1.ListOptions{
LabelSelector: fmt.Sprintf("%s=%s", constants.GroupIdLabelName, groupRef),
})
ui.ExitOnError("listing service resources", err)

services := make(map[string]int64)
ids := make([]string, 0)
for _, job := range jobs.Items {
service, _ := spawn.GetServiceByResourceId(job.Name)
if slices.Contains(logs, service) {
services[service]++
ids = append(ids, job.Name)
}
}

// Inform about detected services
for name, count := range services {
fmt.Printf("%s: fetching logs of %d instances\n", common.ServiceLabel(name), count)
}

// Fetch logs for them
storage := artifacts.InternalStorage()
for _, id := range ids {
service, index := spawn.GetServiceByResourceId(id)
count := index + 1
if services[service] > count {
count = services[service]
}
log := spawn.CreateLogger(service, "", index, count)

logsFilePath, err := spawn.SaveLogs(context.Background(), clientSet, storage, env.Namespace(), id, service+"/", index)
if err == nil {
data.PrintOutput(env.Ref(), "service", ServiceInfo{Group: groupRef, Name: service, Index: index, Logs: storage.FullPath(logsFilePath)})
log("saved logs")
} else {
log("warning", "problem saving the logs", err.Error())
}
}
}

err := testworkflowcontroller.CleanupGroup(context.Background(), clientSet, env.Namespace(), groupRef)
ui.ExitOnError("cleaning up resources", err)
},
}

cmd.Flags().StringArrayVarP(&logs, "logs", "l", nil, "fetch the logs for specific services")

return cmd
}
4 changes: 2 additions & 2 deletions cmd/tcl/testworkflow-toolkit/commands/parallel.go
Original file line number Diff line number Diff line change
Expand Up @@ -177,7 +177,7 @@ func NewParallelCmd() *cobra.Command {
run := func(index int64, spec *testworkflowsv1.TestWorkflowSpec) bool {
clientSet := env.Kubernetes()
log := spawn.CreateLogger("worker", descriptions[index], index, params.Count)
id, machine := spawn.CreateExecutionMachine(index)
id, machine := spawn.CreateExecutionMachine("", index)

updates <- Update{index: index}

Expand Down Expand Up @@ -209,7 +209,7 @@ func NewParallelCmd() *cobra.Command {

// Save logs
if shouldSaveLogs {
logsFilePath, err := spawn.SaveLogs(context.Background(), clientSet, storage, namespace, id, index)
logsFilePath, err := spawn.SaveLogs(context.Background(), clientSet, storage, namespace, id, "", index)
if err == nil {
data.PrintOutput(env.Ref(), "parallel", ParallelStatus{Index: int(index), Logs: storage.FullPath(logsFilePath)})
log("saved logs")
Expand Down
2 changes: 2 additions & 0 deletions cmd/tcl/testworkflow-toolkit/commands/root.go
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,8 @@ func init() {
RootCmd.AddCommand(NewExecuteCmd())
RootCmd.AddCommand(NewArtifactsCmd())
RootCmd.AddCommand(NewParallelCmd())
RootCmd.AddCommand(NewServicesCmd())
RootCmd.AddCommand(NewKillCmd())
}

var RootCmd = &cobra.Command{
Expand Down
Loading

0 comments on commit 2f8d106

Please sign in to comment.