release v1.7.0-beta-4

.travis.yml

@@ -6,7 +6,7 @@ go:
 matrix:
   include:
     - arch: amd64
-      go: 1.15.x
+      go: 1.18.x
 
 install:
   - curl https://raw.githubusercontent.com/golang/dep/master/install.sh | sh

Makefile

@@ -11,7 +11,7 @@ linux:
 	GOOS=linux GOARCH=amd64 go build -o spiff++/spiff++ .
 
 test:
-	go test $(VERBOSE) ./...
+	go test $(VERBOSE) --count=1 ./...
 
 spiff_linux_amd64.zip:
 	GOOS=linux GOARCH=amd64 go build -o spiff++/spiff++ .

README.md

@@ -99,7 +99,7 @@ Contents:
 		- [(( parseurl("http://github.com") ))](#-parseurlhttpgithubcom-)
 		- [(( sort(list) ))](#-sortlist-)
 		- [(( replace(string, "foo", "bar") ))](#-replacestring-foo-bar-)
-		- [(( substr(string, 1, 3) ))](#-substrstring-1-3-)
+		- [(( substr(string, 1, 2) ))](#-substrstring-1-2-)
 		- [(( match("(f.*)(b.*)", "xxxfoobar") ))](#-matchfb-xxxfoobar-)
 		- [(( keys(map) ))](#-keysmap-)
 		- [(( length(list) ))](#-lengthlist-)
@@ -236,7 +236,7 @@ Contents:
 
 Official release executable binaries can be downloaded via [Github releases](https://github.com/mandelsoft/spiff/releases) for Darwin, Linux and PowerPC machines (and virtual machines).
 
-Some of spiff's dependencies have changed since the last official release, and spiff will not be updated to keep up with these dependencies.  Working dependencies are vendored in the `Godeps` directory (more information on the `godep` tool is available [here](https://github.com/tools/godep)).  As such, trying to `go get` spiff will likely fail; the only supported way to use spiff is to use an official binary release.
+Some of spiff's dependencies have changed since the last official release, and spiff will not be updated to keep up with these dependencies. THose dependencies are either fixed or copied into the local code base.
 
 # Usage
 
@@ -312,6 +312,10 @@ The ` merge` command offers several options:
 - With option `--define <key>=<value>` (shorthand`-D`) additional binding values
   can be specified on the command line overriding binding values from the
   binding file. The option may occur multiple times.
+
+  If the *key* contains dots (`.`), it will be interpreted as path expression to 
+  describe fields in deep map values. A dot (and a `\` before a dot) can be escaped
+  by `\` to keep it in the field name.
 
 - The option `--preserve-escapes` will preserve the escaping for dynaml
   expressions and list/map merge directives. This option can be used
@@ -515,10 +519,14 @@ from:
 ```
 
 If the path starts with a dot (`.`) the path is always evaluated from the root
-of the document.
+of the document. If the document root is a list, the first map level is used to resolve the path expression if it starts with `.__map`. This can be used to avoid the need 
+for using the own list index (like `.[1].path`), which might change if
+list entries are added.
 
 List entries consisting of a map with `name` field can directly be addressed
-by their name value as path component.
+by their name value as path component. 
+
+**Note**: This also works for the absolute paths for list documents.
 
 e.g.:
 
@@ -561,6 +569,11 @@ alice:
 
 This new key field will also be observed during the merging of lists.
 
+If the selected key field starts with a `!`, the key feature is disabled.
+The exclamation mark is removed from the effective field name, also.
+
+If the values for the key field are not unqiue, it is disables, also.
+
 ## `(( foo.[bar].baz ))`
 
 Look for the nearest 'foo' key, and from there follow through to the
@@ -570,7 +583,7 @@ The index may be an integer constant (without spaces) as described in the
 last section. But it might also be an arbitrary dynaml expression (even
 an integer, but with spaces). If the expression evaluates to a string,
 it lookups the dedicated field. If the expression evaluates to an integer,
-the array element with this index is addressed.
+the array element with this index is addressed. The dot (`.`) in front of the index operator is optional.
 
 e.g.:
 
@@ -607,6 +620,11 @@ properties:
 
 resolves `foo` again to the value `42`.
 
+**Note**: The index operator is usable on the root element (`.[index]`), also.
+
+It is possible, to specify multiple comma separated indicies to successive lists
+(`foo[0][1]` is equivalent to `foo[0,1]). In such case the indices may not be again lists.
+
 ## `(( list.[1..3] ))`
 
 The slice expression can be used to extract a dedicated sub list from a list
@@ -2274,8 +2292,8 @@ Alternatively the `merge` operation could be used, for example `merge foo.bar`.
 ### `(( tagdef("tag", value) ))`
 
 The function `tagdef` can be used to define dynamic tags (see [Tags](#tags)).
-In contrast to the tag marker this function aloows to specify the tag name 
-and its intended value by an expression. Therefit can be used in composing
+In contrast to the tag marker this function allows to specify the tag name 
+and its intended value by an expression. Therefore, it can be used in composing
 elements like `map` or `sum` to create dynamic tag with calculated values.
 
 An optional third argument can be used to specify the intended scope
@@ -5251,9 +5269,9 @@ the tagged reference `X::data.alice` describes the value `25`.
 For tagged references with a path other than `.` (the whole tag value),
 structured tags feature a more sophisticated resolution mechanism. A structured
 tag consist of multiple tag components separated by a colon (`:`), for
-example `lib:mylib`. Therefore tags span a tree of namespaces or scopes
+example `lib:mylib`. Therefore, tags span a tree of namespaces or scopes
 used to resolve path references. A tag-less reference just uses 
-the actual document or binding to resolve a path extression.
+the actual document or binding to resolve a path expression.
 
 Evaluation of a path reference for a tag tries to resolve the path in the
 first tag tree level where the path is available (breadth-first search).

cmd/merge.go

@@ -161,11 +161,15 @@ func merge(stdin bool, templateFilePath string, opts flow.Options, json, split b
 		for k, v := range values {
 			i, err := strconv.ParseInt(v, 10, 64)
 			if err == nil {
-				m[k] = yaml.NewNode(i, "<values>")
+				err = addValue(m, k, yaml.NewNode(i, "<values>"))
 			} else {
-				m[k] = yaml.NewNode(v, "<values>")
+				err = addValue(m, k, yaml.NewNode(v, "<values>"))
+			}
+			if err != nil {
+				log.Fatalln(fmt.Sprintf("error in value definitions (-D): %s", err))
 			}
 		}
+
 	}
 
 	tags := []*dynaml.Tag{}
@@ -241,9 +245,11 @@ func merge(stdin bool, templateFilePath string, opts flow.Options, json, split b
 			}
 		}
 	}
-	if bindingYAML != nil || interpolation || len(tags) > 0 || len(templateYAMLs) > 1 {
-		defstate := flow.NewDefaultState().SetInterpolation(interpolation)
-		defstate.SetTags(tags...)
+	if interpolation {
+		features.SetInterpolation(true)
+	}
+	if bindingYAML != nil || features.Size() > 0 || len(tags) > 0 || len(templateYAMLs) > 1 {
+		defstate := flow.NewDefaultState().SetTags(tags...).SetFeatures(features)
 		binding = flow.NewEnvironment(
 			nil, "context", defstate)
 		if bindingYAML != nil {
@@ -391,3 +397,43 @@ func merge(stdin bool, templateFilePath string, opts flow.Options, json, split b
 		}
 	}
 }
+
+func addValue(m map[string]yaml.Node, name string, value yaml.Node) error {
+	comps := strings.Split(name, ".")
+	for i := 0; i < len(comps)-1; i++ {
+		if comps[i] == "" {
+			return fmt.Errorf("empty path component in %q", name)
+		}
+		mask := 0
+		for strings.HasSuffix(comps[i], "\\") {
+			mask++
+			comps[i] = comps[i][:len(comps[i])-1]
+		}
+		for mask > 1 {
+			comps[i] += "\\"
+			mask -= 2
+		}
+		if mask > 0 {
+			comps[i] += "." + comps[i+1]
+			copy(comps[i+1:], comps[i+2:])
+			comps = comps[:len(comps)-1]
+			i--
+		}
+	}
+	for i := 0; i < len(comps)-1; i++ {
+		c := comps[i]
+		if m[c] == nil {
+			n := map[string]yaml.Node{}
+			m[c] = yaml.NewNode(n, "<values>")
+			m = n
+		} else {
+			if n, ok := m[c].Value().(map[string]yaml.Node); !ok {
+				return fmt.Errorf("field %q in %s is no map", c, name)
+			} else {
+				m = n
+			}
+		}
+	}
+	m[comps[len(comps)-1]] = value
+	return nil
+}

dynaml/convert.go

@@ -108,8 +108,14 @@ func func_bool(arguments []interface{}, binding Binding) (interface{}, Evaluatio
 	if len(arguments) != 1 {
 		return info.Error("bool requires one argument")
 	}
+	if arguments[0] == nil {
+		return false, info, true
+	}
 	switch v := arguments[0].(type) {
 	case string:
+		if v == "" {
+			return false, info, true
+		}
 		i, err := strconv.ParseBool(v)
 		if err != nil {
 			return info.Error("%q is no bool value: %s", err)

dynaml/dynamic_expression.go

@@ -8,28 +8,36 @@ import (
 )
 
 type DynamicExpr struct {
-	Expression Expression
-	Reference  Expression
+	Root  Expression
+	Index Expression
 }
 
 func (e DynamicExpr) Evaluate(binding Binding, locally bool) (interface{}, EvaluationInfo, bool) {
 
-	root, info, ok := e.Expression.Evaluate(binding, locally)
+	// if root is a reference expression and the type is known allow for element selection if element is resolved
+	// regardless of the resolution state of the root
+	// enables .["dyn"]
+	_, isRef := e.Root.(ReferenceExpr)
+
+	root, info, ok := e.Root.Evaluate(binding, locally || isRef)
 	if !ok {
 		return nil, info, false
 	}
 
-	locally = locally || info.Raw
-
 	if !isLocallyResolvedValue(root, binding) {
 		return e, info, true
 	}
 
-	if !locally && !isResolvedValue(root, binding) {
-		return e, info, true
-	}
+	locally = locally || info.Raw
+	/*
+		if !locally && !isResolvedValue(root, binding) {
+			info.Issue = yaml.NewIssue("'%s' unresolved", e.Root)
+			return e, info, true
+		}
+	*/
+
+	dyn, infoe, ok := e.Index.Evaluate(binding, locally)
 
-	dyn, infoe, ok := e.Reference.Evaluate(binding, locally)
 	info.Join(infoe)
 	if !ok {
 		return nil, info, false
@@ -40,6 +48,11 @@ func (e DynamicExpr) Evaluate(binding Binding, locally bool) (interface{}, Evalu
 
 	debug.Debug("dynamic reference: %+v\n", dyn)
 
+	if a, ok := dyn.([]yaml.Node); ok {
+		if len(a) == 1 {
+			dyn = a[0].Value()
+		}
+	}
 	var qual []string
 	switch v := dyn.(type) {
 	case int64:
@@ -51,6 +64,9 @@ func (e DynamicExpr) Evaluate(binding Binding, locally bool) (interface{}, Evalu
 	case string:
 		qual = []string{v}
 	case []yaml.Node:
+		if len(v) == 0 {
+			return info.Error("at least one index or field name required for reference qualifier")
+		}
 		qual = make([]string, len(v))
 		for i, e := range v {
 			switch v := e.Value().(type) {
@@ -65,11 +81,20 @@ func (e DynamicExpr) Evaluate(binding Binding, locally bool) (interface{}, Evalu
 	default:
 		return info.Error("index or field name required for reference qualifier")
 	}
-	return NewReferenceExpr(qual...).find(func(end int, path []string) (yaml.Node, bool) {
+
+	t, info, ok := NewReferenceExpr(qual...).find(func(end int, path []string) (yaml.Node, bool) {
 		return yaml.Find(NewNode(root, nil), binding.GetFeatures(), path[:end+1]...)
-	}, binding, locally)
+	}, binding, true)
+
+	if !ok {
+		return nil, info, false
+	}
+	if isResolvedValue(t, binding) {
+		return t, info, true
+	}
+	return e, info, true
 }
 
 func (e DynamicExpr) String() string {
-	return fmt.Sprintf("%s.[%s]", e.Expression, e.Reference)
+	return fmt.Sprintf("%s.%s", e.Root, e.Index)
 }

dynaml/dynamic_expression_test.go

@@ -26,7 +26,7 @@ var _ = Describe("dynamic references", func() {
 		})
 	})
 
-	Context("when a dynamic array refernce is found", func() {
+	Context("when a dynamic array reference is found", func() {
 		It("evaluates to the indexed array entry", func() {
 			ref := ReferenceExpr{Path: []string{"foo"}}
 			idx := IntegerExpr{1}
@@ -39,5 +39,31 @@ var _ = Describe("dynamic references", func() {
 
 			Expect(expr).To(EvaluateAs(42, binding))
 		})
+
+		It("evaluates to the indexed array entry", func() {
+			ref := ReferenceExpr{Path: []string{"foo"}}
+			idx := ListExpr{[]Expression{IntegerExpr{1}}}
+			expr := DynamicExpr{ref, idx}
+			binding := FakeBinding{
+				FoundReferences: map[string]yaml.Node{
+					"foo": NewNode([]yaml.Node{NewNode(1, nil), NewNode(42, nil)}, nil),
+				},
+			}
+
+			Expect(expr).To(EvaluateAs(42, binding))
+		})
+
+		It("evaluates to the multi-indexed array entry", func() {
+			ref := ReferenceExpr{Path: []string{"foo"}}
+			idx := ListExpr{[]Expression{IntegerExpr{0}, IntegerExpr{1}}}
+			expr := DynamicExpr{ref, idx}
+			binding := FakeBinding{
+				FoundReferences: map[string]yaml.Node{
+					"foo": NewNode([]yaml.Node{NewNode([]yaml.Node{NewNode(1, nil), NewNode(42, nil)}, nil)}, nil),
+				},
+			}
+
+			Expect(expr).To(EvaluateAs(42, binding))
+		})
 	})
 })

dynaml/dynaml.peg

@@ -47,10 +47,12 @@ Modulo <-  '%' req_ws Level0
 Level0 <- IP / String / Number / Boolean / Undefined / Nil / Symbol / Not /
           Substitution / Merge / Auto / Lambda / Chained
 
-Chained <- ( MapMapping / Sync / Catch / Mapping / MapSelection / Selection / Sum / List / Map / Range / Grouped / Reference ) ChainedQualifiedExpression*
+Chained <- ( MapMapping / Sync / Catch / Mapping / MapSelection / Selection / Sum / List / Map / Range / Grouped / Reference / TopIndex ) ChainedQualifiedExpression*
 ChainedQualifiedExpression <- ChainedCall / Currying / ChainedRef / ChainedDynRef / Projection
 ChainedRef <-  PathComponent FollowUpRef
-ChainedDynRef <- '.'? '[' Expression ']'
+ChainedDynRef <- '.'? Indices
+TopIndex <- '.' Indices
+Indices <- StartList ExpressionList ']'
 Slice <- Range
 Currying <- '*' ChainedCall
 ChainedCall <- StartArguments  NameArgumentList? ')'