feat(expressions): add more functionalities (#5234)

* feat(expressions): add `len()` standard function for expressions
* feat(expressions): add floor/ceil/round functions to stdlib
* feat(expressions): add chunk function to stdlib
* feat(expressions): add map() function to stdlib
* feat(expressions): add filter() function to stdlib
* feat(expressions): add jq() function to stdlib
* feat(expressions): add at() function to stdlib
* feat(expressions): add eval() function to stdlib
* feat(expressions): add support for wildcards and nested property accessors
* feat(expressions): prepare common FS library
* feat(expressions): support spread operator for Call arguments
* feat(expressions): add simple option to simplify expression
* fix(expressions): ternary operator
* feat(expressions): add shellargs method to parse shell arguments

cmd/tcl/testworkflow-init/data/expressions.go

@@ -9,13 +9,11 @@
 package data
 
 import (
-	"fmt"
 	"os"
 	"strings"
 
-	"github.com/pkg/errors"
-
 	"github.com/kubeshop/testkube/pkg/tcl/expressionstcl"
+	"github.com/kubeshop/testkube/pkg/tcl/expressionstcl/libs"
 )
 
 var aliases = map[string]string{
@@ -95,21 +93,8 @@ var RefStatusMachine = expressionstcl.NewMachine().
 		return string(State.GetStep(ref).Status), true
 	})
 
-var FileMachine = expressionstcl.NewMachine().
-	RegisterFunction("file", func(values ...expressionstcl.StaticValue) (interface{}, bool, error) {
-		if len(values) != 1 {
-			return nil, true, errors.New("file() function takes a single argument")
-		}
-		if !values[0].IsString() {
-			return nil, true, fmt.Errorf("file() function expects a string argument, provided: %v", values[0].String())
-		}
-		filePath, _ := values[0].StringValue()
-		file, err := os.ReadFile(filePath)
-		if err != nil {
-			return nil, true, fmt.Errorf("reading file(%s): %s", filePath, err.Error())
-		}
-		return string(file), true, nil
-	})
+var wd, _ = os.Getwd()
+var FileMachine = libs.NewFsMachine(os.DirFS("/"), wd)
 
 func Template(tpl string, m ...expressionstcl.Machine) (string, error) {
 	m = append(m, AliasMachine, baseTestWorkflowMachine)

cmd/tcl/testworkflow-toolkit/artifacts/walker.go

@@ -27,9 +27,13 @@ func mapSlice[T any, U any](s []T, fn func(T) U) []U {
 
 func deduplicateRoots(paths []string) []string {
 	result := make([]string, 0)
+	unique := make(map[string]struct{})
+	for _, p := range paths {
+		unique[p] = struct{}{}
+	}
 loop:
-	for _, path := range paths {
-		for _, path2 := range paths {
+	for path := range unique {
+		for path2 := range unique {
 			if strings.HasPrefix(path, path2+"/") {
 				continue loop
 			}
@@ -176,6 +180,7 @@ func (w *walker) Patterns() []string {
 	return w.patterns
 }
 
+// TODO: Support negative patterns
 func (w *walker) matches(filePath string) bool {
 	for _, p := range w.patterns {
 		v, _ := doublestar.PathMatch(p, filePath)

go.mod

@@ -25,6 +25,7 @@ require (
 	github.com/gookit/color v1.5.3
 	github.com/gorilla/websocket v1.5.0
 	github.com/h2non/filetype v1.1.3
+	github.com/itchyny/gojq v0.12.14
 	github.com/joshdk/go-junit v1.0.0
 	github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51
 	github.com/kelseyhightower/envconfig v1.4.0
@@ -95,7 +96,6 @@ require (
 	github.com/gorilla/css v1.0.1 // indirect
 	github.com/hashicorp/golang-lru/v2 v2.0.1 // indirect
 	github.com/henvic/httpretty v0.1.0 // indirect
-	github.com/itchyny/gojq v0.12.14 // indirect
 	github.com/itchyny/timefmt-go v0.1.5 // indirect
 	github.com/josharian/intern v1.0.0 // indirect
 	github.com/klauspost/cpuid/v2 v2.2.3 // indirect

pkg/tcl/expressionstcl/accessor.go

@@ -10,14 +10,36 @@ package expressionstcl
 
 import (
 	"fmt"
+	"strings"
 )
 
 type accessor struct {
-	name string
+	name     string
+	fallback *Expression
 }
 
 func newAccessor(name string) Expression {
-	return &accessor{name: name}
+	// Map values based on wildcard
+	segments := strings.Split(name, ".*")
+	if len(segments) > 1 {
+		return newCall("map", []callArgument{
+			{expr: newAccessor(strings.Join(segments[0:len(segments)-1], ".*"))},
+			{expr: NewStringValue("_.value" + segments[len(segments)-1])},
+		})
+	}
+
+	// Prepare fallback based on the segments
+	segments = strings.Split(name, ".")
+	var fallback *Expression
+	if len(segments) > 1 {
+		f := newPropertyAccessor(
+			newAccessor(strings.Join(segments[0:len(segments)-1], ".")),
+			segments[len(segments)-1],
+		)
+		fallback = &f
+	}
+
+	return &accessor{name: name, fallback: fallback}
 }
 
 func (s *accessor) Type() Type {
@@ -43,12 +65,19 @@ func (s *accessor) SafeResolve(m ...Machine) (v Expression, changed bool, err er
 
 	for i := range m {
 		result, ok, err := m[i].Get(s.name)
+		if ok && err == nil {
+			return result, true, nil
+		}
+		if s.fallback != nil {
+			var err2 error
+			result, ok, err2 = (*s.fallback).SafeResolve(m...)
+			if ok && err2 == nil {
+				return result, true, nil
+			}
+		}
 		if err != nil {
 			return nil, false, fmt.Errorf("error while accessing %s: %s", s.String(), err.Error())
 		}
-		if ok {
-			return result, true, nil
-		}
 	}
 	return s, false, nil
 }

pkg/tcl/expressionstcl/call.go

@@ -16,13 +16,18 @@ import (
 
 type call struct {
 	name string
-	args []Expression
+	args []callArgument
 }
 
-func newCall(name string, args []Expression) Expression {
+type callArgument struct {
+	expr   Expression
+	spread bool
+}
+
+func newCall(name string, args []callArgument) Expression {
 	for i := range args {
-		if args[i] == nil {
-			args[i] = None
+		if args[i].expr == nil {
+			args[i].expr = None
 		}
 	}
 	return &call{name: name, args: args}
@@ -38,7 +43,10 @@ func (s *call) Type() Type {
 func (s *call) String() string {
 	args := make([]string, len(s.args))
 	for i, arg := range s.args {
-		args[i] = arg.String()
+		args[i] = arg.expr.String()
+		if arg.spread {
+			args[i] += "..."
+		}
 	}
 	return fmt.Sprintf("%s(%s)", s.name, strings.Join(args, ","))
 }
@@ -51,7 +59,7 @@ func (s *call) Template() string {
 	if s.name == stringCastStdFn {
 		args := make([]string, len(s.args))
 		for i, a := range s.args {
-			args[i] = a.Template()
+			args[i] = a.expr.Template()
 		}
 		return strings.Join(args, "")
 	}
@@ -60,32 +68,51 @@ func (s *call) Template() string {
 
 func (s *call) isResolved() bool {
 	for i := range s.args {
-		if s.args[i].Static() == nil {
+		if s.args[i].expr.Static() == nil {
 			return false
 		}
 	}
 	return true
 }
 
-func (s *call) resolvedArgs() []StaticValue {
-	v := make([]StaticValue, len(s.args))
-	for i, vv := range s.args {
-		v[i] = vv.Static()
+func (s *call) resolvedArgs() ([]StaticValue, error) {
+	v := make([]StaticValue, 0)
+	for _, vv := range s.args {
+		value := vv.expr.Static()
+		if vv.spread {
+			if value.IsNone() {
+				continue
+			}
+			items, err := value.SliceValue()
+			if err != nil {
+				return nil, fmt.Errorf("spread operator (...) used against non-list parameter: %s", value)
+			}
+			staticItems := make([]StaticValue, len(items))
+			for i := range items {
+				staticItems[i] = NewValue(items[i])
+			}
+			v = append(v, staticItems...)
+		} else {
+			v = append(v, value)
+		}
 	}
-	return v
+	return v, nil
 }
 
 func (s *call) SafeResolve(m ...Machine) (v Expression, changed bool, err error) {
 	var ch bool
 	for i := range s.args {
-		s.args[i], ch, err = s.args[i].SafeResolve(m...)
+		s.args[i].expr, ch, err = s.args[i].expr.SafeResolve(m...)
 		changed = changed || ch
 		if err != nil {
 			return nil, changed, err
 		}
 	}
 	if s.isResolved() {
-		args := s.resolvedArgs()
+		args, err := s.resolvedArgs()
+		if err != nil {
+			return nil, true, err
+		}
 		result, ok, err := StdLibMachine.Call(s.name, args...)
 		if ok {
 			if err != nil {
@@ -117,15 +144,15 @@ func (s *call) Static() StaticValue {
 func (s *call) Accessors() map[string]struct{} {
 	result := make(map[string]struct{})
 	for i := range s.args {
-		maps.Copy(result, s.args[i].Accessors())
+		maps.Copy(result, s.args[i].expr.Accessors())
 	}
 	return result
 }
 
 func (s *call) Functions() map[string]struct{} {
 	result := make(map[string]struct{})
 	for i := range s.args {
-		maps.Copy(result, s.args[i].Functions())
+		maps.Copy(result, s.args[i].expr.Functions())
 	}
 	result[s.name] = struct{}{}
 	return result

pkg/tcl/expressionstcl/convert.go

@@ -117,6 +117,16 @@ func toMap(s interface{}) (map[string]interface{}, error) {
 		return nil, nil
 	}
 	// Convert
+	if isStruct(s) {
+		v, err := json.Marshal(s)
+		if err != nil {
+			return nil, fmt.Errorf("error while marshaling value: %v: %v", s, err)
+		}
+		err = json.Unmarshal(v, &s)
+		if err != nil {
+			return nil, fmt.Errorf("error while unmarshaling value: %v: %v", s, err)
+		}
+	}
 	if isMap(s) {
 		value := reflect.ValueOf(s)
 		res := make(map[string]interface{}, value.Len())