diff --git a/go.mod b/go.mod index 456d202b..9dc44e5c 100644 --- a/go.mod +++ b/go.mod @@ -7,6 +7,7 @@ require ( github.com/docker/docker v27.4.0+incompatible github.com/dustin/go-humanize v1.0.1 github.com/dustinkirkland/golang-petname v0.0.0-20191129215211-8e5a1ed0cff0 + github.com/google/cel-go v0.22.0 github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 github.com/spf13/cobra v1.8.1 @@ -25,10 +26,12 @@ require ( ) require ( + cel.dev/expr v0.18.0 // indirect github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161 // indirect github.com/BurntSushi/toml v1.4.0 // indirect github.com/Microsoft/go-winio v0.5.1 // indirect github.com/alessio/shellescape v1.4.2 // indirect + github.com/antlr4-go/antlr/v4 v4.13.0 // indirect github.com/containerd/log v0.1.0 // indirect github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/distribution/reference v0.6.0 // indirect @@ -68,12 +71,14 @@ require ( github.com/opencontainers/image-spec v1.0.2 // indirect github.com/pelletier/go-toml v1.9.5 // indirect github.com/pkg/errors v0.9.1 // indirect + github.com/stoewer/go-strcase v1.3.0 // indirect github.com/x448/float16 v0.8.4 // indirect go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.53.0 // indirect go.opentelemetry.io/otel v1.28.0 // indirect go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.27.0 // indirect go.opentelemetry.io/otel/metric v1.28.0 // indirect go.opentelemetry.io/otel/trace v1.28.0 // indirect + golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 // indirect golang.org/x/mod v0.21.0 // indirect golang.org/x/net v0.30.0 // indirect golang.org/x/oauth2 v0.23.0 // indirect @@ -83,6 +88,8 @@ require ( golang.org/x/text v0.19.0 // indirect golang.org/x/time v0.7.0 // indirect golang.org/x/tools v0.26.0 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20240826202546-f6391c0de4c7 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20240826202546-f6391c0de4c7 // indirect google.golang.org/protobuf v1.35.1 // indirect gopkg.in/evanphx/json-patch.v4 v4.12.0 // indirect gopkg.in/inf.v0 v0.9.1 // indirect diff --git a/go.sum b/go.sum index b3221752..a1ac4e96 100644 --- a/go.sum +++ b/go.sum @@ -1,3 +1,5 @@ +cel.dev/expr v0.18.0 h1:CJ6drgk+Hf96lkLikr4rFf19WrU0BOWEihyZnI2TAzo= +cel.dev/expr v0.18.0/go.mod h1:MrpN08Q+lEBs+bGYdLxxHkZoUSsCp0nSKTs0nTymJgw= github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161 h1:L/gRVlceqvL25UVaW/CKtUDjefjrs0SPonmDGUVOYP0= github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E= github.com/BurntSushi/toml v1.4.0 h1:kuoIxZQy2WRRk1pttg9asf+WVv6tWQuBNVmK8+nqPr0= @@ -8,6 +10,8 @@ github.com/Microsoft/go-winio v0.5.1 h1:aPJp2QD7OOrhO5tQXqQoGSJc+DjDtWTGLOmNyAm6 github.com/Microsoft/go-winio v0.5.1/go.mod h1:JPGBdM1cNvN/6ISo+n8V5iA4v8pBzdOpzfwIujj1a84= github.com/alessio/shellescape v1.4.2 h1:MHPfaU+ddJ0/bYWpgIeUnQUqKrlJ1S7BfEYPM4uEoM0= github.com/alessio/shellescape v1.4.2/go.mod h1:PZAiSCk0LJaZkiCSkPv8qIobYglO3FPpyFjDCtHLS30= +github.com/antlr4-go/antlr/v4 v4.13.0 h1:lxCg3LAv+EUK6t1i0y1V6/SLeUi0eKEKdhQAlS8TVTI= +github.com/antlr4-go/antlr/v4 v4.13.0/go.mod h1:pfChB/xh/Unjila75QW7+VU4TSnWnnk9UTnmpPaOR2g= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK343L8= @@ -69,6 +73,8 @@ github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= +github.com/google/cel-go v0.22.0 h1:b3FJZxpiv1vTMo2/5RDUqAHPxkT8mmMfJIrq1llbf7g= +github.com/google/cel-go v0.22.0/go.mod h1:BuznPXXfQDpXKWQ9sPW3TzlAJN5zzFe+i9tIs0yC4s8= github.com/google/gnostic-models v0.6.8 h1:yo/ABAfM5IMRsS1VnXjTBvUb61tFIHozhlYvRgGre9I= github.com/google/gnostic-models v0.6.8/go.mod h1:5n7qKqH0f5wFt+aWF8CW6pZLLNOfYuF5OpfBSENuI8U= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= @@ -162,6 +168,8 @@ github.com/spf13/cobra v1.8.1/go.mod h1:wHxEcudfqmLYa8iTfL+OuZPbBZkmvliBWKIezN3k github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/stoewer/go-strcase v1.3.0 h1:g0eASXYtp+yvN9fK8sH94oCIk0fau9uV1/ZdJ0AVEzs= +github.com/stoewer/go-strcase v1.3.0/go.mod h1:fAH5hQ5pehh+j3nZfvwdk2RgEgQjAoM8wodgtPmh1xo= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= @@ -253,7 +261,6 @@ golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8T golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= gomodules.xyz/jsonpatch/v2 v2.4.0 h1:Ci3iUJyx9UeRx7CeFN8ARgGbkESwJK+KB9lLcWxY/Zw= gomodules.xyz/jsonpatch/v2 v2.4.0/go.mod h1:AH3dM2RI6uoBZxn3LVrfvJ3E0/9dG4cSrbuBJT4moAY= -google.golang.org/genproto v0.0.0-20240123012728-ef4313101c80 h1:KAeGQVN3M9nD0/bQXnr/ClcEMJ968gUXJQ9pwfSynuQ= google.golang.org/genproto/googleapis/api v0.0.0-20240826202546-f6391c0de4c7 h1:YcyjlL1PRr2Q17/I0dPk2JmYS5CDXfcdb2Z3YRioEbw= google.golang.org/genproto/googleapis/api v0.0.0-20240826202546-f6391c0de4c7/go.mod h1:OCdP9MfskevB/rbYvHTsXTtKC+3bHWajPdoKgjcYkfo= google.golang.org/genproto/googleapis/rpc v0.0.0-20240826202546-f6391c0de4c7 h1:2035KHhUv+EpyB+hWgJnaWKJOdX1E95w2S8Rr4uWKTs= diff --git a/pkg/apis/testharness/v1beta1/expression.go b/pkg/apis/testharness/v1beta1/expression.go new file mode 100644 index 00000000..2ff215cb --- /dev/null +++ b/pkg/apis/testharness/v1beta1/expression.go @@ -0,0 +1,65 @@ +package v1beta1 + +import ( + "errors" + "fmt" + "strings" + + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/types" +) + +var ( + errAPIVersionInvalid = errors.New("apiVersion not of the format (/)") + errKindNotSpecified = errors.New("kind not specified") + errNameNotSpecified = errors.New("name not specified") + errRefNotSpecified = errors.New("ref not specified") +) + +func (t *TestResourceRef) BuildResourceReference() (namespacedName types.NamespacedName, referencedResource *unstructured.Unstructured) { + referencedResource = &unstructured.Unstructured{} + apiVersionSplit := strings.Split(t.APIVersion, "/") + gvk := schema.GroupVersionKind{ + Version: apiVersionSplit[len(apiVersionSplit)-1], + Kind: t.Kind, + } + if len(apiVersionSplit) > 1 { + gvk.Group = apiVersionSplit[0] + } + referencedResource.SetGroupVersionKind(gvk) + + namespacedName = types.NamespacedName{ + Namespace: t.Namespace, + Name: t.Name, + } + + return +} + +func (t *TestResourceRef) Validate() error { + apiVersionSplit := strings.Split(t.APIVersion, "/") + switch { + case t.APIVersion == "" || len(apiVersionSplit) > 2: + return errAPIVersionInvalid + case t.Kind == "": + return errKindNotSpecified + case t.Name == "": + return errNameNotSpecified + case t.Ref == "": + return errRefNotSpecified + } + + return nil +} + +func (t *TestResourceRef) String() string { + return fmt.Sprintf( + "apiVersion=%v, kind=%v, namespace=%v, name=%v, ref=%v", + t.APIVersion, + t.Kind, + t.Namespace, + t.Name, + t.Ref, + ) +} diff --git a/pkg/apis/testharness/v1beta1/expression_test.go b/pkg/apis/testharness/v1beta1/expression_test.go new file mode 100644 index 00000000..cac25e4d --- /dev/null +++ b/pkg/apis/testharness/v1beta1/expression_test.go @@ -0,0 +1,181 @@ +package v1beta1 + +import ( + "reflect" + "testing" + + "github.com/stretchr/testify/assert" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/types" +) + +func TestValidate(t *testing.T) { + testCases := []struct { + name string + testResourceRef TestResourceRef + errored bool + expectedError error + }{ + { + name: "apiVersion is not specified", + testResourceRef: TestResourceRef{ + Kind: "Pod", + Namespace: "test", + Name: "test-pod", + Ref: "testPod", + }, + errored: true, + expectedError: errAPIVersionInvalid, + }, + { + name: "apiVersion is invalid", + testResourceRef: TestResourceRef{ + APIVersion: "x/y/z", + Kind: "Pod", + Namespace: "test", + Name: "test-pod", + Ref: "testPod", + }, + errored: true, + expectedError: errAPIVersionInvalid, + }, + { + name: "apiVersion is valid and group is vacuous", + testResourceRef: TestResourceRef{ + APIVersion: "v1", + Kind: "Pod", + Namespace: "test", + Name: "test-pod", + Ref: "testPod", + }, + errored: false, + }, + { + name: "apiVersion has both group name and version", + testResourceRef: TestResourceRef{ + APIVersion: "apps/v1", + Kind: "Deployment", + Namespace: "test", + Name: "test-deployment", + Ref: "testDeployment", + }, + errored: false, + }, + { + name: "kind is not specified", + testResourceRef: TestResourceRef{ + APIVersion: "apps/v1", + Namespace: "test", + Name: "test-deployment", + Ref: "testDeployment", + }, + errored: true, + expectedError: errKindNotSpecified, + }, + { + name: "name is not specified", + testResourceRef: TestResourceRef{ + APIVersion: "apps/v1", + Kind: "Deployment", + Namespace: "test", + Ref: "testDeployment", + }, + errored: true, + expectedError: errNameNotSpecified, + }, + { + name: "ref is not specified", + testResourceRef: TestResourceRef{ + APIVersion: "apps/v1", + Kind: "Deployment", + Namespace: "test", + Name: "test-deployment", + }, + errored: true, + expectedError: errRefNotSpecified, + }, + { + name: "all attributes are present and valid", + testResourceRef: TestResourceRef{ + APIVersion: "apps/v1", + Kind: "Deployment", + Namespace: "test", + Name: "test-deployment", + Ref: "testDeployment", + }, + errored: false, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + err := tc.testResourceRef.Validate() + if !tc.errored { + assert.NoError(t, err) + } else { + assert.ErrorIs(t, err, tc.expectedError) + } + }) + } +} + +func TestBuildResourceReference(t *testing.T) { + buildObject := func(gvk schema.GroupVersionKind) *unstructured.Unstructured { + obj := &unstructured.Unstructured{} + obj.SetGroupVersionKind(gvk) + return obj + } + + testCases := []struct { + name string + testResourceRef TestResourceRef + namespacedName types.NamespacedName + resourceReference *unstructured.Unstructured + }{ + { + name: "group name is vacuous", + testResourceRef: TestResourceRef{ + APIVersion: "v1", + Kind: "Pod", + Namespace: "test", + Name: "test-pod", + Ref: "testPod", + }, + namespacedName: types.NamespacedName{ + Namespace: "test", + Name: "test-pod", + }, + resourceReference: buildObject(schema.GroupVersionKind{Group: "", Version: "v1", Kind: "Pod"}), + }, + { + name: "group name is present", + testResourceRef: TestResourceRef{ + APIVersion: "apps/v1", + Kind: "Deployment", + Namespace: "test", + Name: "test-deployment", + Ref: "testDeployment", + }, + namespacedName: types.NamespacedName{ + Namespace: "test", + Name: "test-deployment", + }, + resourceReference: buildObject(schema.GroupVersionKind{Group: "apps", Version: "v1", Kind: "Deployment"}), + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + namspacedName, referencedResource := tc.testResourceRef.BuildResourceReference() + assert.Equal(t, tc.namespacedName, namspacedName) + assert.True( + t, + reflect.DeepEqual(tc.resourceReference, referencedResource), + "constructed unstructured reference does not match, expected '%s', got '%s'", + tc.resourceReference, + referencedResource, + ) + }) + } +} diff --git a/pkg/apis/testharness/v1beta1/test_types.go b/pkg/apis/testharness/v1beta1/test_types.go index 850ae4de..ad726053 100644 --- a/pkg/apis/testharness/v1beta1/test_types.go +++ b/pkg/apis/testharness/v1beta1/test_types.go @@ -157,6 +157,11 @@ type TestAssert struct { Collectors []*TestCollector `json:"collectors,omitempty"` // Commands is a set of commands to be run as assertions for the current step Commands []TestAssertCommand `json:"commands,omitempty"` + + ResourceRefs []TestResourceRef `json:"resourceRefs,omitempty"` + + AssertAny []*Assertion `json:"assertAny,omitempty"` + AssertAll []*Assertion `json:"assertAll,omitempty"` } // TestAssertCommand an assertion based on the result of the execution of a command @@ -227,6 +232,18 @@ type TestCollector struct { Cmd string `json:"command,omitempty"` } +type TestResourceRef struct { + APIVersion string `json:"apiVersion,omitempty"` + Kind string `json:"kind,omitempty"` + Namespace string `json:"namespace,omitempty"` + Name string `json:"name,omitempty"` + Ref string `json:"ref,omitempty"` +} + +type Assertion struct { + CELExpression string `json:"celExpr,omitempty"` +} + // DefaultKINDContext defines the default kind context to use. const DefaultKINDContext = "kind" diff --git a/pkg/apis/testharness/v1beta1/zz_generated.deepcopy.go b/pkg/apis/testharness/v1beta1/zz_generated.deepcopy.go index 4698a0c6..303263f4 100644 --- a/pkg/apis/testharness/v1beta1/zz_generated.deepcopy.go +++ b/pkg/apis/testharness/v1beta1/zz_generated.deepcopy.go @@ -24,6 +24,22 @@ import ( runtime "k8s.io/apimachinery/pkg/runtime" ) +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *Assertion) DeepCopyInto(out *Assertion) { + *out = *in + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Assertion. +func (in *Assertion) DeepCopy() *Assertion { + if in == nil { + return nil + } + out := new(Assertion) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *Command) DeepCopyInto(out *Command) { *out = *in @@ -95,6 +111,33 @@ func (in *TestAssert) DeepCopyInto(out *TestAssert) { *out = make([]TestAssertCommand, len(*in)) copy(*out, *in) } + if in.ResourceRefs != nil { + in, out := &in.ResourceRefs, &out.ResourceRefs + *out = make([]TestResourceRef, len(*in)) + copy(*out, *in) + } + if in.AssertAny != nil { + in, out := &in.AssertAny, &out.AssertAny + *out = make([]*Assertion, len(*in)) + for i := range *in { + if (*in)[i] != nil { + in, out := &(*in)[i], &(*out)[i] + *out = new(Assertion) + **out = **in + } + } + } + if in.AssertAll != nil { + in, out := &in.AssertAll, &out.AssertAll + *out = make([]*Assertion, len(*in)) + for i := range *in { + if (*in)[i] != nil { + in, out := &(*in)[i], &(*out)[i] + *out = new(Assertion) + **out = **in + } + } + } return } @@ -179,6 +222,22 @@ func (in *TestFile) DeepCopyObject() runtime.Object { return nil } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *TestResourceRef) DeepCopyInto(out *TestResourceRef) { + *out = *in + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new TestResourceRef. +func (in *TestResourceRef) DeepCopy() *TestResourceRef { + if in == nil { + return nil + } + out := new(TestResourceRef) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *TestStep) DeepCopyInto(out *TestStep) { *out = *in diff --git a/pkg/expressions/cel.go b/pkg/expressions/cel.go new file mode 100644 index 00000000..0f675ad2 --- /dev/null +++ b/pkg/expressions/cel.go @@ -0,0 +1,141 @@ +package expressions + +import ( + "errors" + "fmt" + + "github.com/google/cel-go/cel" + + harness "github.com/kudobuilder/kuttl/pkg/apis/testharness/v1beta1" +) + +func buildProgram(expr string, env *cel.Env) (cel.Program, error) { + ast, issues := env.Compile(expr) + if issues != nil && issues.Err() != nil { + return nil, fmt.Errorf("type-check error: %s", issues.Err()) + } + + prg, err := env.Program(ast) + if err != nil { + return nil, fmt.Errorf("program construction error: %w", err) + } + + return prg, nil +} + +func buildEnv(resourceRefs []harness.TestResourceRef) (*cel.Env, error) { + var errs []error + for _, resourceRef := range resourceRefs { + if err := resourceRef.Validate(); err != nil { + errs = append(errs, fmt.Errorf("validation failed for reference '%v': %w", resourceRef.String(), err)) + } + } + + if len(errs) > 0 { + return nil, fmt.Errorf("failed to load resource reference(s): %w", errors.Join(errs...)) + } + + env, err := cel.NewEnv() + if err != nil { + return nil, fmt.Errorf("failed to create environment: %w", err) + } + + for _, resourceRef := range resourceRefs { + env, err = env.Extend(cel.Variable(resourceRef.Ref, cel.DynType)) + if err != nil { + return nil, fmt.Errorf("failed to add resource parameter '%v' to environment: %w", resourceRef.Ref, err) + } + } + + return env, nil +} + +// RunAssertExpressions evaluates a set of CEL expressions specified as AnyAllExpressions +func RunAssertExpressions( + programs map[string]cel.Program, + variables map[string]interface{}, + assertAny, + assertAll []*harness.Assertion, +) []error { + var errs []error + if len(assertAny) == 0 && len(assertAll) == 0 { + return errs + } + + var anyExprErrors, allExprErrors []error + for _, expr := range assertAny { + if err := evaluateExpression(expr.CELExpression, programs, variables); err != nil { + anyExprErrors = append(anyExprErrors, err) + } + } + + for _, expr := range assertAll { + if err := evaluateExpression(expr.CELExpression, programs, variables); err != nil { + allExprErrors = append(allExprErrors, err) + } + } + + if len(assertAny) != 0 && len(anyExprErrors) == len(assertAny) { + errs = append(errs, fmt.Errorf("no expression evaluated to true: %w", errors.Join(anyExprErrors...))) + } + + if len(allExprErrors) > 0 { + errs = append(errs, fmt.Errorf("not all assertAll expressions evaluated to true: %w", errors.Join(allExprErrors...))) + } + + return errs +} + +func LoadPrograms(testAssert *harness.TestAssert) (map[string]cel.Program, error) { + var errs []error + var assertions []*harness.Assertion + assertions = append(assertions, testAssert.AssertAny...) + assertions = append(assertions, testAssert.AssertAll...) + + env, err := buildEnv(testAssert.ResourceRefs) + if err != nil { + return nil, fmt.Errorf("failed to build CEL environment: %w", err) + } + + if len(assertions) == 0 { + return nil, nil + } + programs := make(map[string]cel.Program) + + for _, assertion := range assertions { + if prg, err := buildProgram(assertion.CELExpression, env); err != nil { + errs = append( + errs, + fmt.Errorf("failed to build CEL program from expression %q: %w", assertion.CELExpression, err), + ) + } else { + programs[assertion.CELExpression] = prg + } + } + + if len(errs) > 0 { + return nil, fmt.Errorf("failed to load expression(s): %w", errors.Join(errs...)) + } + + return programs, nil +} + +func evaluateExpression(expr string, + programs map[string]cel.Program, + variables map[string]interface{}, +) error { + prg, ok := programs[expr] + if !ok { + return fmt.Errorf("couldn't find pre-built parsed CEL expression '%v'", expr) + } + out, _, err := prg.Eval(variables) + if err != nil { + return fmt.Errorf("failed to evaluate CEL expression: %w", err) + } + + if out.Value() != true { + return fmt.Errorf("expression '%v' evaluated to '%v'", expr, out.Value()) + } + + return nil +} diff --git a/pkg/test/expression_integration_test.go b/pkg/test/expression_integration_test.go new file mode 100644 index 00000000..51322946 --- /dev/null +++ b/pkg/test/expression_integration_test.go @@ -0,0 +1,171 @@ +//go:build integration + +package test + +import ( + "context" + "errors" + "fmt" + "os" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/discovery" + "sigs.k8s.io/controller-runtime/pkg/client" + + testutils "github.com/kudobuilder/kuttl/pkg/test/utils" +) + +func buildTestStep(t *testing.T) *Step { + return &Step{ + Name: t.Name(), + Index: 0, + Logger: testutils.NewTestLogger(t, t.Name()), + Client: func(bool) (client.Client, error) { + return testenv.Client, nil + }, + DiscoveryClient: func() (discovery.DiscoveryInterface, error) { + return testenv.DiscoveryClient, nil + }, + } +} + +func TestAssertExpressions(t *testing.T) { + codednsDeployment := &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{ + Name: "coredns", + Namespace: "kube-system", + Labels: map[string]string{"k8s-app": "kube-dns"}, + }, + Spec: appsv1.DeploymentSpec{ + Selector: &metav1.LabelSelector{ + MatchLabels: map[string]string{"k8s-app": "kube-dns"}, + }, + Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{"k8s-app": "kube-dns"}, + }, + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "coredns", + Image: "registry.k8s.io/coredns/coredns:v1.11.1", + }, + }, + }, + }, + }, + } + metricServerPod := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "metrics-server-xyz-pqr", + Namespace: "kube-system", + Labels: map[string]string{ + "k8s-app": "metrics-server", + }, + }, + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "metrics-server", + Image: "registry.k8s.io/metrics-server/metrics-server:v0.7.2", + }, + }, + }, + } + + assert.NoError(t, testenv.Client.Create(context.TODO(), codednsDeployment)) + assert.NoError(t, testenv.Client.Create(context.TODO(), metricServerPod)) + + testCases := []struct { + name string + expectLoadFailure bool + expectRunFailure bool + expectedErrorMessage string + }{ + { + name: "invalid expression", + expectLoadFailure: true, + expectedErrorMessage: "undeclared reference", + }, + { + name: "check deployment name", + }, + { + name: "check incorrect deployment name", + expectRunFailure: true, + expectedErrorMessage: "not all assertAll expressions evaluated to true", + }, + { + name: "check multiple assert all", + }, + { + name: "check multiple assert all with one failing", + expectRunFailure: true, + expectedErrorMessage: "not all assertAll expressions evaluated to true", + }, + { + name: "check multiple assert any", + }, + { + name: "check multiple assert any with all failing", + expectRunFailure: true, + expectedErrorMessage: "no expression evaluated to true", + }, + { + name: "check expression for ephemeral namespace", + }, + } + + const testNamespace = "kuttl-ephemeral-xyz" + assert.NoError(t, testenv.Client.Create(context.TODO(), &corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: testNamespace, + }, + TypeMeta: metav1.TypeMeta{ + Kind: "Namespace", + }, + })) + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + dirName := fmt.Sprintf( + "step_integration_test_data/assert_expressions/%s", + strings.ReplaceAll(tc.name, " ", "_"), + ) + + files, err := os.ReadDir(dirName) + assert.NoError(t, err) + + for i := 0; i < len(files)-1; i++ { + fName := fmt.Sprintf("%s/%s", dirName, files[i].Name()) + step := buildTestStep(t) + assert.NoError(t, step.LoadYAML(fName)) + assert.NoError(t, errors.Join(errors.Join(step.Run(t, testNamespace)...))) + } + + step := buildTestStep(t) + + fName := fmt.Sprintf("%s/%s", dirName, files[len(files)-1].Name()) + + err = step.LoadYAML(fName) + if !tc.expectLoadFailure { + assert.NoError(t, err) + } else { + assert.ErrorContains(t, err, tc.expectedErrorMessage) + return + } + + err = errors.Join(step.Run(t, testNamespace)...) + if !tc.expectRunFailure { + assert.NoError(t, err) + } else { + assert.ErrorContains(t, err, tc.expectedErrorMessage) + } + }) + } +} diff --git a/pkg/test/step.go b/pkg/test/step.go index a076d963..52b072e5 100644 --- a/pkg/test/step.go +++ b/pkg/test/step.go @@ -10,6 +10,7 @@ import ( "testing" "time" + "github.com/google/cel-go/cel" k8serrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -23,6 +24,7 @@ import ( harness "github.com/kudobuilder/kuttl/pkg/apis/testharness/v1beta1" "github.com/kudobuilder/kuttl/pkg/env" + "github.com/kudobuilder/kuttl/pkg/expressions" kfile "github.com/kudobuilder/kuttl/pkg/file" "github.com/kudobuilder/kuttl/pkg/http" "github.com/kudobuilder/kuttl/pkg/kubernetes" @@ -46,6 +48,8 @@ type Step struct { Step *harness.TestStep Assert *harness.TestAssert + Programs map[string]cel.Program + Asserts []client.Object Apply []client.Object Errors []client.Object @@ -412,6 +416,28 @@ func (s *Step) CheckAssertCommands(ctx context.Context, namespace string, comman return testErrors } +func (s *Step) CheckAssertExpressions(namespace string) []error { + client, err := s.Client(false) + if err != nil { + return []error{err} + } + + variables := make(map[string]interface{}) + for _, resourceRef := range s.Assert.ResourceRefs { + if resourceRef.Namespace == "" { + resourceRef.Namespace = namespace + } + namespacedName, referencedResource := resourceRef.BuildResourceReference() + if err := client.Get(context.TODO(), namespacedName, referencedResource); err != nil { + return []error{fmt.Errorf("failed to get referenced resource '%v': %w", namespacedName, err)} + } + + variables[resourceRef.Ref] = referencedResource.Object + } + + return expressions.RunAssertExpressions(s.Programs, variables, s.Assert.AssertAny, s.Assert.AssertAll) +} + // Check checks if the resources defined in Asserts and Errors are in the correct state. func (s *Step) Check(namespace string, timeout int) []error { testErrors := []error{} @@ -422,6 +448,7 @@ func (s *Step) Check(namespace string, timeout int) []error { if s.Assert != nil { testErrors = append(testErrors, s.CheckAssertCommands(context.TODO(), namespace, s.Assert.Commands, timeout)...) + testErrors = append(testErrors, s.CheckAssertExpressions(namespace)...) } for _, expected := range s.Errors { @@ -533,6 +560,11 @@ func (s *Step) LoadYAML(file string) error { } else { return fmt.Errorf("failed to load TestAssert object from %s: it contains an object of type %T", file, obj) } + + s.Programs, err = expressions.LoadPrograms(s.Assert) + if err != nil { + return fmt.Errorf("failed to prepare expression evaluation: %w", err) + } } else { asserts = append(asserts, obj) } diff --git a/pkg/test/step_integration_test_data/assert_expressions/check_deployment_name/00-assert.yaml b/pkg/test/step_integration_test_data/assert_expressions/check_deployment_name/00-assert.yaml new file mode 100644 index 00000000..9b0cda37 --- /dev/null +++ b/pkg/test/step_integration_test_data/assert_expressions/check_deployment_name/00-assert.yaml @@ -0,0 +1,11 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +resourceRefs: + - apiVersion: apps/v1 + kind: Deployment + namespace: kube-system + name: coredns + ref: coredns +assertAll: + - celExpr: "coredns.metadata.name == 'coredns'" +timeout: 1 diff --git a/pkg/test/step_integration_test_data/assert_expressions/check_expression_for_ephemeral_namespace/00-create.yaml b/pkg/test/step_integration_test_data/assert_expressions/check_expression_for_ephemeral_namespace/00-create.yaml new file mode 100644 index 00000000..156afc8c --- /dev/null +++ b/pkg/test/step_integration_test_data/assert_expressions/check_expression_for_ephemeral_namespace/00-create.yaml @@ -0,0 +1,12 @@ +apiVersion: v1 +kind: Pod +metadata: + name: nginx-pod + labels: + app: nginx +spec: + containers: + - name: nginx-container + image: nginx:latest + ports: + - containerPort: 80 diff --git a/pkg/test/step_integration_test_data/assert_expressions/check_expression_for_ephemeral_namespace/01-assert.yaml b/pkg/test/step_integration_test_data/assert_expressions/check_expression_for_ephemeral_namespace/01-assert.yaml new file mode 100644 index 00000000..fd6fb16a --- /dev/null +++ b/pkg/test/step_integration_test_data/assert_expressions/check_expression_for_ephemeral_namespace/01-assert.yaml @@ -0,0 +1,10 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +resourceRefs: + - apiVersion: v1 + kind: Pod + name: nginx-pod + ref: nginxPod +assertAll: + - celExpr: "nginxPod.metadata.name == 'nginx-pod'" +timeout: 1 diff --git a/pkg/test/step_integration_test_data/assert_expressions/check_incorrect_deployment_name/00-assert.yaml b/pkg/test/step_integration_test_data/assert_expressions/check_incorrect_deployment_name/00-assert.yaml new file mode 100644 index 00000000..3ec2e308 --- /dev/null +++ b/pkg/test/step_integration_test_data/assert_expressions/check_incorrect_deployment_name/00-assert.yaml @@ -0,0 +1,11 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +resourceRefs: + - apiVersion: apps/v1 + kind: Deployment + namespace: kube-system + name: coredns + ref: coredns +assertAll: + - celExpr: "coredns.metadata.name == 'metrics-server'" +timeout: 1 diff --git a/pkg/test/step_integration_test_data/assert_expressions/check_multiple_assert_all/00-assert.yaml b/pkg/test/step_integration_test_data/assert_expressions/check_multiple_assert_all/00-assert.yaml new file mode 100644 index 00000000..c05990d4 --- /dev/null +++ b/pkg/test/step_integration_test_data/assert_expressions/check_multiple_assert_all/00-assert.yaml @@ -0,0 +1,17 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +resourceRefs: + - apiVersion: apps/v1 + kind: Deployment + namespace: kube-system + name: coredns + ref: coredns + - apiVersion: v1 + kind: Pod + namespace: kube-system + name: metrics-server-xyz-pqr + ref: metricsServer +assertAll: + - celExpr: "coredns.metadata.name == 'coredns'" + - celExpr: "metricsServer.metadata.labels['k8s-app'] == 'metrics-server'" +timeout: 1 diff --git a/pkg/test/step_integration_test_data/assert_expressions/check_multiple_assert_all_with_one_failing/00-assert.yaml b/pkg/test/step_integration_test_data/assert_expressions/check_multiple_assert_all_with_one_failing/00-assert.yaml new file mode 100644 index 00000000..0cd17ac4 --- /dev/null +++ b/pkg/test/step_integration_test_data/assert_expressions/check_multiple_assert_all_with_one_failing/00-assert.yaml @@ -0,0 +1,17 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +resourceRefs: + - apiVersion: apps/v1 + kind: Deployment + namespace: kube-system + name: coredns + ref: coredns + - apiVersion: v1 + kind: Pod + namespace: kube-system + name: metrics-server-xyz-pqr + ref: metricsServer +assertAll: + - celExpr: "coredns.metadata.name == 'metrics-server'" + - celExpr: "metricsServer.metadata.labels['k8s-app'] == 'metrics-server'" +timeout: 1 diff --git a/pkg/test/step_integration_test_data/assert_expressions/check_multiple_assert_any/00-assert.yaml b/pkg/test/step_integration_test_data/assert_expressions/check_multiple_assert_any/00-assert.yaml new file mode 100644 index 00000000..00e3f30e --- /dev/null +++ b/pkg/test/step_integration_test_data/assert_expressions/check_multiple_assert_any/00-assert.yaml @@ -0,0 +1,17 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +resourceRefs: + - apiVersion: apps/v1 + kind: Deployment + namespace: kube-system + name: coredns + ref: coredns + - apiVersion: v1 + kind: Pod + namespace: kube-system + name: metrics-server-xyz-pqr + ref: metricsServer +assertAny: + - celExpr: "coredns.metadata.name == 'coredns'" + - celExpr: "metricsServer.metadata.labels['k8s-app'] == 'metrics-server-1.6'" +timeout: 1 diff --git a/pkg/test/step_integration_test_data/assert_expressions/check_multiple_assert_any_with_all_failing/00-assert.yaml b/pkg/test/step_integration_test_data/assert_expressions/check_multiple_assert_any_with_all_failing/00-assert.yaml new file mode 100644 index 00000000..8d559331 --- /dev/null +++ b/pkg/test/step_integration_test_data/assert_expressions/check_multiple_assert_any_with_all_failing/00-assert.yaml @@ -0,0 +1,17 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +resourceRefs: + - apiVersion: apps/v1 + kind: Deployment + namespace: kube-system + name: coredns + ref: coredns + - apiVersion: v1 + kind: Pod + namespace: kube-system + name: metrics-server-xyz-pqr + ref: metricsServer +assertAny: + - celExpr: "coredns.metadata.name == 'metrics-server'" + - celExpr: "metricsServer.metadata.labels['k8s-app'] == 'metrics-server-1.6'" +timeout: 1 diff --git a/pkg/test/step_integration_test_data/assert_expressions/invalid_expression/00-assert.yaml b/pkg/test/step_integration_test_data/assert_expressions/invalid_expression/00-assert.yaml new file mode 100644 index 00000000..86a8b392 --- /dev/null +++ b/pkg/test/step_integration_test_data/assert_expressions/invalid_expression/00-assert.yaml @@ -0,0 +1,11 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +resourceRefs: + - apiVersion: apps/v1 + kind: Deployment + namespace: kube-system + name: coredns + ref: coredns +assertAll: + - celExpr: "badVariable.metadata.name == 'coredns'" +timeout: 1