[KEP-0009] feat: add expression based assertions

go.mod

@@ -7,6 +7,7 @@ require (
 	github.com/docker/docker v27.4.0+incompatible
 	github.com/dustin/go-humanize v1.0.1
 	github.com/dustinkirkland/golang-petname v0.0.0-20191129215211-8e5a1ed0cff0
+	github.com/google/cel-go v0.22.0
 	github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510
 	github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2
 	github.com/spf13/cobra v1.8.1
@@ -25,10 +26,12 @@ require (
 )
 
 require (
+	cel.dev/expr v0.18.0 // indirect
 	github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161 // indirect
 	github.com/BurntSushi/toml v1.4.0 // indirect
 	github.com/Microsoft/go-winio v0.5.1 // indirect
 	github.com/alessio/shellescape v1.4.2 // indirect
+	github.com/antlr4-go/antlr/v4 v4.13.0 // indirect
 	github.com/containerd/log v0.1.0 // indirect
 	github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
 	github.com/distribution/reference v0.6.0 // indirect
@@ -68,12 +71,14 @@ require (
 	github.com/opencontainers/image-spec v1.0.2 // indirect
 	github.com/pelletier/go-toml v1.9.5 // indirect
 	github.com/pkg/errors v0.9.1 // indirect
+	github.com/stoewer/go-strcase v1.3.0 // indirect
 	github.com/x448/float16 v0.8.4 // indirect
 	go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.53.0 // indirect
 	go.opentelemetry.io/otel v1.28.0 // indirect
 	go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.27.0 // indirect
 	go.opentelemetry.io/otel/metric v1.28.0 // indirect
 	go.opentelemetry.io/otel/trace v1.28.0 // indirect
+	golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 // indirect
 	golang.org/x/mod v0.21.0 // indirect
 	golang.org/x/net v0.30.0 // indirect
 	golang.org/x/oauth2 v0.23.0 // indirect
@@ -83,6 +88,8 @@ require (
 	golang.org/x/text v0.19.0 // indirect
 	golang.org/x/time v0.7.0 // indirect
 	golang.org/x/tools v0.26.0 // indirect
+	google.golang.org/genproto/googleapis/api v0.0.0-20240826202546-f6391c0de4c7 // indirect
+	google.golang.org/genproto/googleapis/rpc v0.0.0-20240826202546-f6391c0de4c7 // indirect
 	google.golang.org/protobuf v1.35.1 // indirect
 	gopkg.in/evanphx/json-patch.v4 v4.12.0 // indirect
 	gopkg.in/inf.v0 v0.9.1 // indirect

go.sum

@@ -1,3 +1,5 @@
+cel.dev/expr v0.18.0 h1:CJ6drgk+Hf96lkLikr4rFf19WrU0BOWEihyZnI2TAzo=
+cel.dev/expr v0.18.0/go.mod h1:MrpN08Q+lEBs+bGYdLxxHkZoUSsCp0nSKTs0nTymJgw=
 github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161 h1:L/gRVlceqvL25UVaW/CKtUDjefjrs0SPonmDGUVOYP0=
 github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E=
 github.com/BurntSushi/toml v1.4.0 h1:kuoIxZQy2WRRk1pttg9asf+WVv6tWQuBNVmK8+nqPr0=
@@ -8,6 +10,8 @@ github.com/Microsoft/go-winio v0.5.1 h1:aPJp2QD7OOrhO5tQXqQoGSJc+DjDtWTGLOmNyAm6
 github.com/Microsoft/go-winio v0.5.1/go.mod h1:JPGBdM1cNvN/6ISo+n8V5iA4v8pBzdOpzfwIujj1a84=
 github.com/alessio/shellescape v1.4.2 h1:MHPfaU+ddJ0/bYWpgIeUnQUqKrlJ1S7BfEYPM4uEoM0=
 github.com/alessio/shellescape v1.4.2/go.mod h1:PZAiSCk0LJaZkiCSkPv8qIobYglO3FPpyFjDCtHLS30=
+github.com/antlr4-go/antlr/v4 v4.13.0 h1:lxCg3LAv+EUK6t1i0y1V6/SLeUi0eKEKdhQAlS8TVTI=
+github.com/antlr4-go/antlr/v4 v4.13.0/go.mod h1:pfChB/xh/Unjila75QW7+VU4TSnWnnk9UTnmpPaOR2g=
 github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
 github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
 github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK343L8=
@@ -69,6 +73,8 @@ github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q=
 github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q=
 github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek=
 github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps=
+github.com/google/cel-go v0.22.0 h1:b3FJZxpiv1vTMo2/5RDUqAHPxkT8mmMfJIrq1llbf7g=
+github.com/google/cel-go v0.22.0/go.mod h1:BuznPXXfQDpXKWQ9sPW3TzlAJN5zzFe+i9tIs0yC4s8=
 github.com/google/gnostic-models v0.6.8 h1:yo/ABAfM5IMRsS1VnXjTBvUb61tFIHozhlYvRgGre9I=
 github.com/google/gnostic-models v0.6.8/go.mod h1:5n7qKqH0f5wFt+aWF8CW6pZLLNOfYuF5OpfBSENuI8U=
 github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
@@ -162,6 +168,8 @@ github.com/spf13/cobra v1.8.1/go.mod h1:wHxEcudfqmLYa8iTfL+OuZPbBZkmvliBWKIezN3k
 github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
 github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
 github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
+github.com/stoewer/go-strcase v1.3.0 h1:g0eASXYtp+yvN9fK8sH94oCIk0fau9uV1/ZdJ0AVEzs=
+github.com/stoewer/go-strcase v1.3.0/go.mod h1:fAH5hQ5pehh+j3nZfvwdk2RgEgQjAoM8wodgtPmh1xo=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
 github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
@@ -253,7 +261,6 @@ golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8T
 golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 gomodules.xyz/jsonpatch/v2 v2.4.0 h1:Ci3iUJyx9UeRx7CeFN8ARgGbkESwJK+KB9lLcWxY/Zw=
 gomodules.xyz/jsonpatch/v2 v2.4.0/go.mod h1:AH3dM2RI6uoBZxn3LVrfvJ3E0/9dG4cSrbuBJT4moAY=
-google.golang.org/genproto v0.0.0-20240123012728-ef4313101c80 h1:KAeGQVN3M9nD0/bQXnr/ClcEMJ968gUXJQ9pwfSynuQ=
 google.golang.org/genproto/googleapis/api v0.0.0-20240826202546-f6391c0de4c7 h1:YcyjlL1PRr2Q17/I0dPk2JmYS5CDXfcdb2Z3YRioEbw=
 google.golang.org/genproto/googleapis/api v0.0.0-20240826202546-f6391c0de4c7/go.mod h1:OCdP9MfskevB/rbYvHTsXTtKC+3bHWajPdoKgjcYkfo=
 google.golang.org/genproto/googleapis/rpc v0.0.0-20240826202546-f6391c0de4c7 h1:2035KHhUv+EpyB+hWgJnaWKJOdX1E95w2S8Rr4uWKTs=

pkg/apis/testharness/v1beta1/expression.go

@@ -0,0 +1,65 @@
+package v1beta1
+
+import (
+	"errors"
+	"fmt"
+	"strings"
+
+	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
+	"k8s.io/apimachinery/pkg/runtime/schema"
+	"k8s.io/apimachinery/pkg/types"
+)
+
+var (
+	errAPIVersionInvalid = errors.New("apiVersion not of the format (<group>/)<version>")
+	errKindNotSpecified  = errors.New("kind not specified")
+	errNameNotSpecified  = errors.New("name not specified")
+	errRefNotSpecified   = errors.New("ref not specified")
+)
+
+func (t *TestResourceRef) BuildResourceReference() (namespacedName types.NamespacedName, referencedResource *unstructured.Unstructured) {
+	referencedResource = &unstructured.Unstructured{}
+	apiVersionSplit := strings.Split(t.APIVersion, "/")
+	gvk := schema.GroupVersionKind{
+		Version: apiVersionSplit[len(apiVersionSplit)-1],
+		Kind:    t.Kind,
+	}
+	if len(apiVersionSplit) > 1 {
+		gvk.Group = apiVersionSplit[0]
+	}
+	referencedResource.SetGroupVersionKind(gvk)
+
+	namespacedName = types.NamespacedName{
+		Namespace: t.Namespace,
+		Name:      t.Name,
+	}
+
+	return
+}
+
+func (t *TestResourceRef) Validate() error {
+	apiVersionSplit := strings.Split(t.APIVersion, "/")
+	switch {
+	case t.APIVersion == "" || len(apiVersionSplit) > 2:
+		return errAPIVersionInvalid
+	case t.Kind == "":
+		return errKindNotSpecified
+	case t.Name == "":
+		return errNameNotSpecified
+	case t.Ref == "":
+		return errRefNotSpecified
+	}
+
+	return nil
+}
+
+func (t *TestResourceRef) String() string {
+	return fmt.Sprintf(
+		"apiVersion=%v, kind=%v, namespace=%v, name=%v, ref=%v",
+		t.APIVersion,
+		t.Kind,
+		t.Namespace,
+		t.Name,
+		t.Ref,
+	)
+}

pkg/apis/testharness/v1beta1/expression_test.go

@@ -0,0 +1,181 @@
+package v1beta1
+
+import (
+	"reflect"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
+	"k8s.io/apimachinery/pkg/runtime/schema"
+	"k8s.io/apimachinery/pkg/types"
+)
+
+func TestValidate(t *testing.T) {
+	testCases := []struct {
+		name            string
+		testResourceRef TestResourceRef
+		errored         bool
+		expectedError   error
+	}{
+		{
+			name: "apiVersion is not specified",
+			testResourceRef: TestResourceRef{
+				Kind:      "Pod",
+				Namespace: "test",
+				Name:      "test-pod",
+				Ref:       "testPod",
+			},
+			errored:       true,
+			expectedError: errAPIVersionInvalid,
+		},
+		{
+			name: "apiVersion is invalid",
+			testResourceRef: TestResourceRef{
+				APIVersion: "x/y/z",
+				Kind:       "Pod",
+				Namespace:  "test",
+				Name:       "test-pod",
+				Ref:        "testPod",
+			},
+			errored:       true,
+			expectedError: errAPIVersionInvalid,
+		},
+		{
+			name: "apiVersion is valid and group is vacuous",
+			testResourceRef: TestResourceRef{
+				APIVersion: "v1",
+				Kind:       "Pod",
+				Namespace:  "test",
+				Name:       "test-pod",
+				Ref:        "testPod",
+			},
+			errored: false,
+		},
+		{
+			name: "apiVersion has both group name and version",
+			testResourceRef: TestResourceRef{
+				APIVersion: "apps/v1",
+				Kind:       "Deployment",
+				Namespace:  "test",
+				Name:       "test-deployment",
+				Ref:        "testDeployment",
+			},
+			errored: false,
+		},
+		{
+			name: "kind is not specified",
+			testResourceRef: TestResourceRef{
+				APIVersion: "apps/v1",
+				Namespace:  "test",
+				Name:       "test-deployment",
+				Ref:        "testDeployment",
+			},
+			errored:       true,
+			expectedError: errKindNotSpecified,
+		},
+		{
+			name: "name is not specified",
+			testResourceRef: TestResourceRef{
+				APIVersion: "apps/v1",
+				Kind:       "Deployment",
+				Namespace:  "test",
+				Ref:        "testDeployment",
+			},
+			errored:       true,
+			expectedError: errNameNotSpecified,
+		},
+		{
+			name: "ref is not specified",
+			testResourceRef: TestResourceRef{
+				APIVersion: "apps/v1",
+				Kind:       "Deployment",
+				Namespace:  "test",
+				Name:       "test-deployment",
+			},
+			errored:       true,
+			expectedError: errRefNotSpecified,
+		},
+		{
+			name: "all attributes are present and valid",
+			testResourceRef: TestResourceRef{
+				APIVersion: "apps/v1",
+				Kind:       "Deployment",
+				Namespace:  "test",
+				Name:       "test-deployment",
+				Ref:        "testDeployment",
+			},
+			errored: false,
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			err := tc.testResourceRef.Validate()
+			if !tc.errored {
+				assert.NoError(t, err)
+			} else {
+				assert.ErrorIs(t, err, tc.expectedError)
+			}
+		})
+	}
+}
+
+func TestBuildResourceReference(t *testing.T) {
+	buildObject := func(gvk schema.GroupVersionKind) *unstructured.Unstructured {
+		obj := &unstructured.Unstructured{}
+		obj.SetGroupVersionKind(gvk)
+		return obj
+	}
+
+	testCases := []struct {
+		name              string
+		testResourceRef   TestResourceRef
+		namespacedName    types.NamespacedName
+		resourceReference *unstructured.Unstructured
+	}{
+		{
+			name: "group name is vacuous",
+			testResourceRef: TestResourceRef{
+				APIVersion: "v1",
+				Kind:       "Pod",
+				Namespace:  "test",
+				Name:       "test-pod",
+				Ref:        "testPod",
+			},
+			namespacedName: types.NamespacedName{
+				Namespace: "test",
+				Name:      "test-pod",
+			},
+			resourceReference: buildObject(schema.GroupVersionKind{Group: "", Version: "v1", Kind: "Pod"}),
+		},
+		{
+			name: "group name is present",
+			testResourceRef: TestResourceRef{
+				APIVersion: "apps/v1",
+				Kind:       "Deployment",
+				Namespace:  "test",
+				Name:       "test-deployment",
+				Ref:        "testDeployment",
+			},
+			namespacedName: types.NamespacedName{
+				Namespace: "test",
+				Name:      "test-deployment",
+			},
+			resourceReference: buildObject(schema.GroupVersionKind{Group: "apps", Version: "v1", Kind: "Deployment"}),
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			namspacedName, referencedResource := tc.testResourceRef.BuildResourceReference()
+			assert.Equal(t, tc.namespacedName, namspacedName)
+			assert.True(
+				t,
+				reflect.DeepEqual(tc.resourceReference, referencedResource),
+				"constructed unstructured reference does not match, expected '%s', got '%s'",
+				tc.resourceReference,
+				referencedResource,
+			)
+		})
+	}
+}

pkg/apis/testharness/v1beta1/test_types.go

@@ -157,6 +157,11 @@ type TestAssert struct {
 	Collectors []*TestCollector `json:"collectors,omitempty"`
 	// Commands is a set of commands to be run as assertions for the current step
 	Commands []TestAssertCommand `json:"commands,omitempty"`
+
+	ResourceRefs []TestResourceRef `json:"resourceRefs,omitempty"`
+
+	AssertAny []*Assertion `json:"assertAny,omitempty"`
+	AssertAll []*Assertion `json:"assertAll,omitempty"`
 }
 
 // TestAssertCommand an assertion based on the result of the execution of a command
@@ -227,6 +232,18 @@ type TestCollector struct {
 	Cmd string `json:"command,omitempty"`
 }
 
+type TestResourceRef struct {
+	APIVersion string `json:"apiVersion,omitempty"`
+	Kind       string `json:"kind,omitempty"`
+	Namespace  string `json:"namespace,omitempty"`
+	Name       string `json:"name,omitempty"`
+	Ref        string `json:"ref,omitempty"`
+}
+
+type Assertion struct {
+	CELExpression string `json:"celExpr,omitempty"`
+}
+
 // DefaultKINDContext defines the default kind context to use.
 const DefaultKINDContext = "kind"