From 6efd671bc68776d3c9dfd88619e30d3d832183bc Mon Sep 17 00:00:00 2001 From: Uwe Krueger Date: Fri, 2 Oct 2020 10:30:15 +0200 Subject: [PATCH 01/10] prepare next release --- cmd/root.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/root.go b/cmd/root.go index ed9975e..00f657b 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -17,7 +17,7 @@ var cfgFile string var rootCmd = &cobra.Command{ Use: "spiff", Short: "YAML in-domain templating processor", - Version: "v1.4.1-dev", + Version: "v1.6.0-dev", } // Execute adds all child commands to the root command and sets flags appropriately. From 79e5883d8c44f1ddebbf0de0eb4440fa05d01c82 Mon Sep 17 00:00:00 2001 From: Uwe Krueger Date: Wed, 14 Oct 2020 16:56:39 +0200 Subject: [PATCH 02/10] bindings for the command line --- README.md | 5 +++++ cmd/merge.go | 50 ++++++++++++++++++++++++++++++++++++-------------- cmd/run.go | 6 +++--- 3 files changed, 44 insertions(+), 17 deletions(-) diff --git a/README.md b/README.md index 14b370c..cdf06e8 100644 --- a/README.md +++ b/README.md @@ -236,6 +236,11 @@ The ` merge` command offers several options: state file with the `.bak` suffix. This can be used together with a manual merging as offered by the [state](libraries/state/README.md) utility library. +- With option `--bindings ` a yaml file can be specified, whose content + is used to build additional bindings for the processing. The yaml document must + consist of a map. Each key is used as additional binding. The bindings document + is not processed, the values are used as defined. + - The option `--preserve-escapes` will preserve the escaping for dynaml expressions and list/map merge directives. This option can be used if further processing steps of a processing result with *spiff* is intended. diff --git a/cmd/merge.go b/cmd/merge.go index aac09bf..9dac151 100644 --- a/cmd/merge.go +++ b/cmd/merge.go @@ -24,6 +24,7 @@ var selection []string var split bool var processingOptions flow.Options var state string +var bindings string // mergeCmd represents the merge command var mergeCmd = &cobra.Command{ @@ -38,7 +39,7 @@ var mergeCmd = &cobra.Command{ return nil }, Run: func(cmd *cobra.Command, args []string) { - merge(false, args[0], processingOptions, asJSON, split, outputPath, selection, state, nil, args[1:]) + merge(false, args[0], processingOptions, asJSON, split, outputPath, selection, state, bindings, nil, args[1:]) }, } @@ -53,6 +54,7 @@ func init() { mergeCmd.Flags().BoolVar(&processingOptions.PreserveEscapes, "preserve-escapes", false, "preserve escaping for escaped expressions and merges") mergeCmd.Flags().BoolVar(&processingOptions.PreserveTemporary, "preserve-temporary", false, "preserve temporary fields") mergeCmd.Flags().StringVar(&state, "state", "", "select state file to maintain") + mergeCmd.Flags().StringVar(&bindings, "bindings", "", "yaml file with additional bindings to use") mergeCmd.Flags().StringArrayVar(&selection, "select", []string{}, "filter dedicated output fields") } @@ -64,8 +66,25 @@ func fileExists(filename string) bool { return !info.IsDir() } +func readYAML(filename string, desc string, required bool) yaml.Node { + if filename != "" { + if fileExists(filename) { + data, err := ioutil.ReadFile(filename) + if required && err != nil { + log.Fatalln(fmt.Sprintf("error reading %s [%s]:", desc, path.Clean(filename)), err) + } + doc, err := yaml.Parse(filename, data) + if err != nil { + log.Fatalln(fmt.Sprintf("error parsing %s [%s]:", desc, path.Clean(filename)), err) + } + return doc + } + } + return nil +} + func merge(stdin bool, templateFilePath string, opts flow.Options, json, split bool, - subpath string, selection []string, stateFilePath string, stubs []yaml.Node, stubFilePaths []string) { + subpath string, selection []string, stateFilePath, bindingFilePath string, stubs []yaml.Node, stubFilePaths []string) { var templateFile []byte var err error @@ -85,16 +104,14 @@ func merge(stdin bool, templateFilePath string, opts flow.Options, json, split b log.Fatalln(fmt.Sprintf("error parsing template [%s]:", path.Clean(templateFilePath)), err) } - var stateData []byte - + var stateYAML yaml.Node if stateFilePath != "" { if len(templateYAMLs) > 1 { log.Fatalln(fmt.Sprintf("state handling not supported gor multi documents [%s]:", path.Clean(templateFilePath)), err) } - if fileExists(stateFilePath) { - stateData, err = ioutil.ReadFile(stateFilePath) - } + stateYAML = readYAML(stateFilePath, "state file", false) } + bindingYAML := readYAML(bindingFilePath, "bindings file", true) if stubs == nil { stubs = []yaml.Node{} @@ -125,11 +142,7 @@ func merge(stdin bool, templateFilePath string, opts flow.Options, json, split b stubs = append(stubs, stubYAML) } - if stateData != nil { - stateYAML, err := yaml.Parse(stateFilePath, stateData) - if err != nil { - log.Fatalln(fmt.Sprintf("error parsing state [%s]:", path.Clean(stateFilePath)), err) - } + if stateYAML != nil { stubs = append(stubs, stateYAML) } @@ -138,7 +151,16 @@ func merge(stdin bool, templateFilePath string, opts flow.Options, json, split b " @: dependent of or involved in a cycle\n" + " -: depending on a node with an error" - prepared, err := flow.PrepareStubs(nil, processingOptions.Partial, stubs...) + var binding dynaml.Binding + if bindingYAML != nil { + values, ok := bindingYAML.Value().(map[string]yaml.Node) + if !ok { + log.Fatalln("bindings must be given as map") + } + binding = flow.NewEnvironment( + nil, "context").WithLocalScope(values) + } + prepared, err := flow.PrepareStubs(binding, processingOptions.Partial, stubs...) if !processingOptions.Partial && err != nil { log.Fatalln("error generating manifest:", err, legend) } @@ -153,7 +175,7 @@ func merge(stdin bool, templateFilePath string, opts flow.Options, json, split b var bytes []byte if templateYAML.Value() != nil { count++ - flowed, err := flow.Apply(nil, templateYAML, prepared, opts) + flowed, err := flow.Apply(binding, templateYAML, prepared, opts) if !opts.Partial && err != nil { log.Fatalln(fmt.Sprintf("error generating manifest%s:", doc), err, legend) } diff --git a/cmd/run.go b/cmd/run.go index a2fdce1..e15dd05 100644 --- a/cmd/run.go +++ b/cmd/run.go @@ -26,7 +26,7 @@ var processCmd = &cobra.Command{ return nil }, Run: func(cmd *cobra.Command, args []string) { - run(args[0], args[1], processingOptions, asJSON, split, outputPath, selection, state, args[2:]) + run(args[0], args[1], processingOptions, asJSON, split, outputPath, selection, state, bindings, args[2:]) }, } @@ -45,7 +45,7 @@ func init() { } func run(documentFilePath, templateFilePath string, opts flow.Options, json, split bool, - subpath string, selection []string, stateFilePath string, stubFilePaths []string) { + subpath string, selection []string, stateFilePath, bindingFilePath string, stubFilePaths []string) { var err error var stdin = false var documentFile []byte @@ -64,5 +64,5 @@ func run(documentFilePath, templateFilePath string, opts flow.Options, json, spl documentYAML = yaml.NewNode(map[string]yaml.Node{"document": documentYAML}, "<"+documentFilePath+">") stub := yaml.NewNode(map[string]yaml.Node{"document": yaml.NewNode("(( &temporary &inject (merge) ))", ")")}, "") - merge(stdin, templateFilePath, opts, json, split, subpath, selection, stateFilePath, []yaml.Node{stub, documentYAML}, stubFilePaths) + merge(stdin, templateFilePath, opts, json, split, subpath, selection, stateFilePath, bindingFilePath, []yaml.Node{stub, documentYAML}, stubFilePaths) } From 21a9a76e6c39af6aef12fecea7052509ed4514f1 Mon Sep 17 00:00:00 2001 From: Uwe Krueger Date: Fri, 6 Nov 2020 13:57:27 +0100 Subject: [PATCH 03/10] function check --- README.md | 12 +- dynaml/call.go | 2 + dynaml/validate.go | 71 ++++- flow/val_test.go | 660 ++++++++++++++++++++++++--------------------- 4 files changed, 431 insertions(+), 314 deletions(-) diff --git a/README.md b/README.md index cdf06e8..c8a38eb 100644 --- a/README.md +++ b/README.md @@ -117,6 +117,7 @@ Contents: - [(( asyaml(expr) ))](#-asjsonexpr-) - [(( catch(expr) ))](#-catchexpr-) - [(( validate(value,"dnsdomain") ))](#-validatevaluednsdomain-) + - [(( check(value,"dnsdomain") ))](#-checkvaluednsdomain-) - [(( error("message") ))](#-errormessage-) - [Accessing External Content](#accessing-external-content) - [(( read("file.yml") ))](#-readfileyml-) @@ -2756,7 +2757,8 @@ The following validators are available: | `ca`| none | certificate for CA | | `type`| list of accepted type keys | at least one [type key](#-typefoobar-) must match | | `valueset` | list argument with values | possible values | -| `match` | regular expression | string value matching regular expression | +| `value` | `=` | value | check dedicated value | +| `match` | `~=` | regular expression | string value matching regular expression | | `list` | optional list of entry validators | is list and entries match given validators | | `map` | [[ <key validator>, ] <entry validator> ] | is map and keys and entries match given validators | | `mapfield` | <field name> [ , <validator>] | required entry in map | @@ -2847,6 +2849,14 @@ validator: val: (( validate( map, validator) )) ``` +### `(( check(value,"dnsdomain") ))` + +The function `check` can be used to match a yaml structure against a yaml +based value checker. Hereby the same check description already described for +[validate](#-validatevaluednsdomain-) can be used. The result of the call is +a boolean value indicating the match result. It does not fail if the check +fails. + ### `(( error("message") ))` The function `error` can be used to cause explicit evaluation failures with diff --git a/dynaml/call.go b/dynaml/call.go index 3cd0fc1..5d80359 100644 --- a/dynaml/call.go +++ b/dynaml/call.go @@ -308,6 +308,8 @@ func (e CallExpr) Evaluate(binding Binding, locally bool) (interface{}, Evaluati case "validate": resolved, result, sub, ok = func_validate(values, binding) + case "check": + resolved, result, sub, ok = func_check(values, binding) case "type": if info.Undefined { diff --git a/dynaml/validate.go b/dynaml/validate.go index 3c854a3..20051da 100644 --- a/dynaml/validate.go +++ b/dynaml/validate.go @@ -26,12 +26,12 @@ func func_validate(arguments []interface{}, binding Binding) (bool, interface{}, value := arguments[0] for i, c := range arguments[1:] { - r, m, err, valid := validate(value, NewNode(c, binding), binding) + r, m, err, ok := EvalValidationExpression(value, NewNode(c, binding), binding) if err != nil { info.SetError("condition %d has problem: %s", i+1, err) return true, nil, info, false } - if !valid { + if !ok { return false, nil, info, true } if !r { @@ -42,7 +42,37 @@ func func_validate(arguments []interface{}, binding Binding) (bool, interface{}, return true, value, info, true } -func validate(value interface{}, cond yaml.Node, binding Binding) (bool, string, error, bool) { +func func_check(arguments []interface{}, binding Binding) (bool, interface{}, EvaluationInfo, bool) { + info := DefaultInfo() + if len(arguments) < 2 { + info.Error("at least two arguments required for check") + return true, nil, info, false + } + + value := arguments[0] + + for i, c := range arguments[1:] { + r, _, err, ok := EvalValidationExpression(value, NewNode(c, binding), binding) + if err != nil { + info.SetError("condition %d has problem: %s", i+1, err) + return true, nil, info, false + } + if !ok { + return false, nil, info, true + } + if !r { + return true, false, info, true + } + } + return true, true, info, true +} + +// first result: validation successful +// second: message +// third: condition error +// fourth: expression already resolved + +func EvalValidationExpression(value interface{}, cond yaml.Node, binding Binding) (bool, string, error, bool) { if cond == nil || cond.Value() == nil { return ValidatorResult(true, "no condition") } @@ -147,7 +177,7 @@ func handleStringType(value interface{}, op string, binding Binding, args ...yam } for i, e := range l { for j, c := range args { - r, m, err, valid := validate(e.Value(), c, binding) + r, m, err, valid := EvalValidationExpression(e.Value(), c, binding) if err != nil { return ValidatorErrorf("list entry %d condition %d: %s", i, j, err) } @@ -180,7 +210,7 @@ func handleStringType(value interface{}, op string, binding Binding, args ...yam for k, e := range l { if ck != nil { - r, m, err, valid := validate(k, ck, binding) + r, m, err, valid := EvalValidationExpression(k, ck, binding) if err != nil { return ValidatorErrorf("map key %q %s", k, err) } @@ -192,7 +222,7 @@ func handleStringType(value interface{}, op string, binding Binding, args ...yam } } - r, m, err, valid := validate(e.Value(), ce, binding) + r, m, err, valid := EvalValidationExpression(e.Value(), ce, binding) if err != nil { return ValidatorErrorf("map entry %q: %s", k, err) } @@ -228,7 +258,7 @@ func handleStringType(value interface{}, op string, binding Binding, args ...yam return ValidatorResult(false, "has no field %q", field) } if len(args) == 2 { - r, m, err, valid := validate(val.Value(), args[1], binding) + r, m, err, valid := EvalValidationExpression(val.Value(), args[1], binding) if err != nil { return ValidatorErrorf("map entry %q %s", field, err) } @@ -244,7 +274,7 @@ func handleStringType(value interface{}, op string, binding Binding, args ...yam return ValidatorErrorf("validator argument required") } for _, c := range args { - r, m, err, resolved := validate(value, c, binding) + r, m, err, resolved := EvalValidationExpression(value, c, binding) if err != nil || !resolved { return false, "", err, resolved } @@ -265,7 +295,7 @@ func handleStringType(value interface{}, op string, binding Binding, args ...yam return ValidatorErrorf("validator argument required") } for _, c := range args { - r, m, err, resolved := validate(value, c, binding) + r, m, err, resolved := EvalValidationExpression(value, c, binding) if err != nil || !resolved { return false, "", err, resolved } @@ -303,7 +333,7 @@ func handleStringType(value interface{}, op string, binding Binding, args ...yam } for _, v := range l { if ok, _, _ := compareEquals(value, v.Value()); ok { - return ValidatorResult(true, "matches valuset") + return ValidatorResult(true, "matches valueset") } } s, ok := value.(string) @@ -316,7 +346,25 @@ func handleStringType(value interface{}, op string, binding Binding, args ...yam } return ValidatorResult(false, "invalid value") - case "match": + case "value", "=": + if len(args) != 1 { + return ValidatorErrorf("value requires a value argument") + } + s := args[0].Value() + + sv, isStr := value.(string) + if ok, _, _ := compareEquals(s, value); !ok { + if isStr { + return ValidatorResult(false, "invalid value %q", sv) + } + return ValidatorResult(false, "invalid value") + } + if isStr { + return ValidatorResult(true, "valid value %q", sv) + } + return ValidatorResult(true, "valid value") + + case "match", "~=": if len(args) != 1 { return ValidatorErrorf("match requires a regexp argument") } @@ -357,6 +405,7 @@ func handleStringType(value interface{}, op string, binding Binding, args ...yam return ValidatorResult(false, reason[1:]) } return ValidatorResult(false, reason+")") + case "dnsname": s, err := StringValue(op, value) if err != nil { diff --git a/flow/val_test.go b/flow/val_test.go index 5cf453d..6e8dfb2 100644 --- a/flow/val_test.go +++ b/flow/val_test.go @@ -6,78 +6,79 @@ import ( ) var _ = Describe("Flowing YAML for validation", func() { - Context("cidr", func() { - It("accepts", func() { - source := parseYAML(` + Context("validate", func() { + Context("cidr", func() { + It("accepts", func() { + source := parseYAML(` --- val: (( validate("1.2.3.4/20", "cidr") )) `) - resolved := parseYAML(` + resolved := parseYAML(` --- val: 1.2.3.4/20 `) - Expect(source).To(FlowAs(resolved)) - }) - It("rejects", func() { - source := parseYAML(` + Expect(source).To(FlowAs(resolved)) + }) + It("rejects", func() { + source := parseYAML(` --- val: (( catch(validate("1.2.3.4/200", "cidr")) )) `) - resolved := parseYAML(` + resolved := parseYAML(` --- val: valid: false error: "condition 1 failed: is no CIDR: invalid CIDR address: 1.2.3.4/200" `) - Expect(source).To(FlowAs(resolved)) + Expect(source).To(FlowAs(resolved)) + }) }) - }) - Context("ip", func() { - It("accepts", func() { - source := parseYAML(` + Context("ip", func() { + It("accepts", func() { + source := parseYAML(` --- val: (( validate("1.2.3.4", "ip") )) `) - resolved := parseYAML(` + resolved := parseYAML(` --- val: 1.2.3.4 `) - Expect(source).To(FlowAs(resolved)) - }) - It("rejects", func() { - source := parseYAML(` + Expect(source).To(FlowAs(resolved)) + }) + It("rejects", func() { + source := parseYAML(` --- val: (( catch(validate("1.2.3.4.5", "ip")) )) `) - resolved := parseYAML(` + resolved := parseYAML(` --- val: valid: false error: "condition 1 failed: is no ip address: 1.2.3.4.5" `) - Expect(source).To(FlowAs(resolved)) + Expect(source).To(FlowAs(resolved)) + }) }) - }) - Context("wildcarddnsdomain", func() { - It("accepts", func() { - source := parseYAML(` + Context("wildcarddnsdomain", func() { + It("accepts", func() { + source := parseYAML(` --- val: (( validate("*.mandelsoft.org", "wildcarddnsdomain") )) `) - resolved := parseYAML(` + resolved := parseYAML(` --- val: "*.mandelsoft.org" `) - Expect(source).To(FlowAs(resolved)) - }) - It("rejects", func() { - source := parseYAML(` + Expect(source).To(FlowAs(resolved)) + }) + It("rejects", func() { + source := parseYAML(` --- val: (( catch(validate("mandelsoft.org", "wildcarddnsdomain")) )) `) - resolved := parseYAML(` + resolved := parseYAML(` --- val: valid: false @@ -86,28 +87,28 @@ val: of lower case alphanumeric characters, ''-'' or ''.'' and end with an alphanumeric character (e.g. ''*.example.com'', regex used for validation is ''\*\.[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*'')]' `) - Expect(source).To(FlowAs(resolved)) + Expect(source).To(FlowAs(resolved)) + }) }) - }) - Context("dnsdomain", func() { - It("accepts", func() { - source := parseYAML(` + Context("dnsdomain", func() { + It("accepts", func() { + source := parseYAML(` --- val: (( validate("spiff.mandelsoft.org", "dnsdomain") )) `) - resolved := parseYAML(` + resolved := parseYAML(` --- val: "spiff.mandelsoft.org" `) - Expect(source).To(FlowAs(resolved)) - }) - It("rejects", func() { - source := parseYAML(` + Expect(source).To(FlowAs(resolved)) + }) + It("rejects", func() { + source := parseYAML(` --- val: (( catch(validate("*.mandelsoft.org", "dnsdomain")) )) `) - resolved := parseYAML(` + resolved := parseYAML(` --- val: valid: false @@ -116,28 +117,28 @@ val: with an alphanumeric character (e.g. ''example.com'', regex used for validation is ''[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*'')]' `) - Expect(source).To(FlowAs(resolved)) + Expect(source).To(FlowAs(resolved)) + }) }) - }) - Context("dnslabel", func() { - It("accepts", func() { - source := parseYAML(` + Context("dnslabel", func() { + It("accepts", func() { + source := parseYAML(` --- val: (( validate("alice-bob", "dnslabel") )) `) - resolved := parseYAML(` + resolved := parseYAML(` --- val: "alice-bob" `) - Expect(source).To(FlowAs(resolved)) - }) - It("rejects", func() { - source := parseYAML(` + Expect(source).To(FlowAs(resolved)) + }) + It("rejects", func() { + source := parseYAML(` --- val: (( catch(validate("alice+bob", "dnslabel")) )) `) - resolved := parseYAML(` + resolved := parseYAML(` --- val: valid: false @@ -145,30 +146,30 @@ val: case alphanumeric characters or ''-'', and must start and end with an alphanumeric character (e.g. ''my-name'', or ''123-abc'', regex used for validation is ''[a-z0-9]([-a-z0-9]*[a-z0-9])?'')]' `) - Expect(source).To(FlowAs(resolved)) + Expect(source).To(FlowAs(resolved)) + }) }) - }) - Context("dnsname", func() { - It("accepts", func() { - source := parseYAML(` + Context("dnsname", func() { + It("accepts", func() { + source := parseYAML(` --- val1: (( validate("spiff.mandelsoft.org", "dnsname") )) val2: (( validate("*.mandelsoft.org", "dnsname") )) `) - resolved := parseYAML(` + resolved := parseYAML(` --- val1: "spiff.mandelsoft.org" val2: "*.mandelsoft.org" `) - Expect(source).To(FlowAs(resolved)) - }) - It("rejects", func() { - source := parseYAML(` + Expect(source).To(FlowAs(resolved)) + }) + It("rejects", func() { + source := parseYAML(` --- val: (( catch(validate("alice+bob", "dnsname")) )) `) - resolved := parseYAML(` + resolved := parseYAML(` --- val: valid: false @@ -177,171 +178,198 @@ val: an alphanumeric character (e.g. ''example.com'', regex used for validation is ''[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*'')]' `) - Expect(source).To(FlowAs(resolved)) + Expect(source).To(FlowAs(resolved)) + }) }) - }) - Context("type", func() { - It("accepts", func() { - source := parseYAML(` + Context("type", func() { + It("accepts", func() { + source := parseYAML(` --- val: (( validate("spiff.mandelsoft.org", [ "type", "string" ]) )) `) - resolved := parseYAML(` + resolved := parseYAML(` --- val: "spiff.mandelsoft.org" `) - Expect(source).To(FlowAs(resolved)) - }) - It("rejects", func() { - source := parseYAML(` + Expect(source).To(FlowAs(resolved)) + }) + It("rejects", func() { + source := parseYAML(` --- val: (( catch(validate([], [ "type", "string" ])) )) `) - resolved := parseYAML(` + resolved := parseYAML(` --- val: valid: false error: 'condition 1 failed: is not of type string' `) - Expect(source).To(FlowAs(resolved)) + Expect(source).To(FlowAs(resolved)) + }) }) - }) - Context("match", func() { - It("accepts", func() { - source := parseYAML(` + Context("match", func() { + It("accepts", func() { + source := parseYAML(` --- val: (( validate("alice", [ "match", "a.*e" ]) )) `) - resolved := parseYAML(` + resolved := parseYAML(` --- val: "alice" `) - Expect(source).To(FlowAs(resolved)) - }) - It("rejects", func() { - source := parseYAML(` + Expect(source).To(FlowAs(resolved)) + }) + It("rejects", func() { + source := parseYAML(` --- val: (( catch(validate("bob", [ "match", "a.*e" ])) )) `) - resolved := parseYAML(` + resolved := parseYAML(` --- val: valid: false error: 'condition 1 failed: invalid value "bob"' `) - Expect(source).To(FlowAs(resolved)) + Expect(source).To(FlowAs(resolved)) + }) }) - }) - Context("valueset", func() { - It("accepts", func() { - source := parseYAML(` + Context("valueset", func() { + It("accepts", func() { + source := parseYAML(` --- val: (( validate("alice", [ "valueset", [ "alice", "bob" ] ]) )) `) - resolved := parseYAML(` + resolved := parseYAML(` --- val: "alice" `) - Expect(source).To(FlowAs(resolved)) - }) - It("rejects", func() { - source := parseYAML(` + Expect(source).To(FlowAs(resolved)) + }) + It("rejects", func() { + source := parseYAML(` --- val: (( catch(validate("peter", [ "valueset", [ "alice", "bob" ] ])) )) `) - resolved := parseYAML(` + resolved := parseYAML(` --- val: valid: false error: 'condition 1 failed: invalid value "peter"' `) - Expect(source).To(FlowAs(resolved)) + Expect(source).To(FlowAs(resolved)) + }) }) - }) - Context("empty", func() { - It("accepts string", func() { - source := parseYAML(` + Context("value", func() { + It("accepts", func() { + source := parseYAML(` +--- +val: (( validate("alice", [ "value", "alice" ]) )) +`) + resolved := parseYAML(` +--- +val: "alice" +`) + Expect(source).To(FlowAs(resolved)) + }) + It("rejects", func() { + source := parseYAML(` +--- +val: (( catch(validate("peter", [ "value", "alice" ])) )) +`) + resolved := parseYAML(` +--- +val: + valid: false + error: 'condition 1 failed: invalid value "peter"' +`) + Expect(source).To(FlowAs(resolved)) + }) + }) + + Context("empty", func() { + It("accepts string", func() { + source := parseYAML(` --- val: (( validate("", "empty") )) `) - resolved := parseYAML(` + resolved := parseYAML(` --- val: "" `) - Expect(source).To(FlowAs(resolved)) - }) - It("rejects string", func() { - source := parseYAML(` + Expect(source).To(FlowAs(resolved)) + }) + It("rejects string", func() { + source := parseYAML(` --- val: (( catch(validate("foobar", "empty")) )) `) - resolved := parseYAML(` + resolved := parseYAML(` --- val: valid: false error: 'condition 1 failed: is not empty' `) - Expect(source).To(FlowAs(resolved)) - }) + Expect(source).To(FlowAs(resolved)) + }) - It("accepts map", func() { - source := parseYAML(` + It("accepts map", func() { + source := parseYAML(` --- val: (( validate({}, "empty") )) `) - resolved := parseYAML(` + resolved := parseYAML(` --- val: {} `) - Expect(source).To(FlowAs(resolved)) - }) - It("rejects map", func() { - source := parseYAML(` + Expect(source).To(FlowAs(resolved)) + }) + It("rejects map", func() { + source := parseYAML(` --- val: (( catch(validate({ $foo="bar"}, "empty")) )) `) - resolved := parseYAML(` + resolved := parseYAML(` --- val: valid: false error: 'condition 1 failed: is not empty' `) - Expect(source).To(FlowAs(resolved)) - }) + Expect(source).To(FlowAs(resolved)) + }) - It("accepts list", func() { - source := parseYAML(` + It("accepts list", func() { + source := parseYAML(` --- val: (( validate([], "empty") )) `) - resolved := parseYAML(` + resolved := parseYAML(` --- val: [] `) - Expect(source).To(FlowAs(resolved)) - }) - It("rejects list", func() { - source := parseYAML(` + Expect(source).To(FlowAs(resolved)) + }) + It("rejects list", func() { + source := parseYAML(` --- val: (( catch(validate(["foobar"], "empty")) )) `) - resolved := parseYAML(` + resolved := parseYAML(` --- val: valid: false error: 'condition 1 failed: is not empty' `) - Expect(source).To(FlowAs(resolved)) + Expect(source).To(FlowAs(resolved)) + }) }) - }) - Context("privatekey", func() { - It("accepts", func() { - source := parseYAML(` + Context("privatekey", func() { + It("accepts", func() { + source := parseYAML(` --- key: | -----BEGIN RSA PRIVATE KEY----- @@ -350,7 +378,7 @@ key: | -----END RSA PRIVATE KEY----- val: (( validate(key, "privatekey") )) `) - resolved := parseYAML(` + resolved := parseYAML(` --- key: | -----BEGIN RSA PRIVATE KEY----- @@ -363,27 +391,27 @@ val: | w4g1AgQEpjbBAgUA0oHQfw== -----END RSA PRIVATE KEY----- `) - Expect(source).To(FlowAs(resolved)) - }) - It("rejects", func() { - source := parseYAML(` + Expect(source).To(FlowAs(resolved)) + }) + It("rejects", func() { + source := parseYAML(` --- val: (( catch(validate("peter", "privatekey")) )) `) - resolved := parseYAML(` + resolved := parseYAML(` --- val: valid: false error: 'condition 1 failed: is no private key: invalid private key format (expected pem block)' `) - Expect(source).To(FlowAs(resolved)) + Expect(source).To(FlowAs(resolved)) + }) }) - }) - Context("publickey", func() { - It("accepts", func() { - source := parseYAML(` + Context("publickey", func() { + It("accepts", func() { + source := parseYAML(` --- key: | -----BEGIN RSA PUBLIC KEY----- @@ -391,7 +419,7 @@ key: | -----END RSA PUBLIC KEY----- val: (( validate(key, "publickey") )) `) - resolved := parseYAML(` + resolved := parseYAML(` --- key: | -----BEGIN RSA PUBLIC KEY----- @@ -402,119 +430,119 @@ val: | MBACCQC60m+vYsCt7wIDAQAB -----END RSA PUBLIC KEY----- `) - Expect(source).To(FlowAs(resolved)) - }) - It("rejects", func() { - source := parseYAML(` + Expect(source).To(FlowAs(resolved)) + }) + It("rejects", func() { + source := parseYAML(` --- val: (( catch(validate("peter", "publickey")) )) `) - resolved := parseYAML(` + resolved := parseYAML(` --- val: valid: false error: 'condition 1 failed: is no public key: invalid public key format (expected pem block)' `) - Expect(source).To(FlowAs(resolved)) + Expect(source).To(FlowAs(resolved)) + }) }) - }) - Context("or", func() { - It("accepts", func() { - source := parseYAML(` + Context("or", func() { + It("accepts", func() { + source := parseYAML(` --- val: (( validate("alice", [ "or", "empty", "dnslabel" ]) )) `) - resolved := parseYAML(` + resolved := parseYAML(` --- val: "alice" `) - Expect(source).To(FlowAs(resolved)) - }) - It("rejects", func() { - source := parseYAML(` + Expect(source).To(FlowAs(resolved)) + }) + It("rejects", func() { + source := parseYAML(` --- val: (( catch(validate("alice", [ "or", "empty", "!dnslabel" ])) )) `) - resolved := parseYAML(` + resolved := parseYAML(` --- val: valid: false error: 'condition 1 failed: (is not empty and is dns label)' `) - Expect(source).To(FlowAs(resolved)) + Expect(source).To(FlowAs(resolved)) + }) }) - }) - Context("and", func() { - It("accepts", func() { - source := parseYAML(` + Context("and", func() { + It("accepts", func() { + source := parseYAML(` --- val: (( validate("alice", [ "and", "!empty", "dnslabel" ]) )) `) - resolved := parseYAML(` + resolved := parseYAML(` --- val: "alice" `) - Expect(source).To(FlowAs(resolved)) - }) - It("rejects", func() { - source := parseYAML(` + Expect(source).To(FlowAs(resolved)) + }) + It("rejects", func() { + source := parseYAML(` --- val: (( catch(validate("alice", [ "and", "empty", "dnslabel" ])) )) `) - resolved := parseYAML(` + resolved := parseYAML(` --- val: valid: false error: 'condition 1 failed: is not empty' `) - Expect(source).To(FlowAs(resolved)) + Expect(source).To(FlowAs(resolved)) + }) }) - }) - Context("mapfield", func() { - It("accept exists", func() { - source := parseYAML(` + Context("mapfield", func() { + It("accept exists", func() { + source := parseYAML(` --- map: alice: 25 val: (( validate(map, [ "mapfield", "alice" ]) )) `) - resolved := parseYAML(` + resolved := parseYAML(` --- map: alice: 25 val: alice: 25 `) - Expect(source).To(FlowAs(resolved)) - }) - It("accept validated", func() { - source := parseYAML(` + Expect(source).To(FlowAs(resolved)) + }) + It("accept validated", func() { + source := parseYAML(` --- map: alice: 25 val: (( validate(map, [ "mapfield", "alice", [ "type", "int" ]]) )) `) - resolved := parseYAML(` + resolved := parseYAML(` --- map: alice: 25 val: alice: 25 `) - Expect(source).To(FlowAs(resolved)) - }) - It("fail exists", func() { - source := parseYAML(` + Expect(source).To(FlowAs(resolved)) + }) + It("fail exists", func() { + source := parseYAML(` --- map: alice: 25 val: (( catch(validate(map, [ "mapfield", "bob" ])) )) `) - resolved := parseYAML(` + resolved := parseYAML(` --- map: alice: 25 @@ -522,16 +550,16 @@ val: valid: false error: 'condition 1 failed: has no field "bob"' `) - Expect(source).To(FlowAs(resolved)) - }) - It("fail validator", func() { - source := parseYAML(` + Expect(source).To(FlowAs(resolved)) + }) + It("fail validator", func() { + source := parseYAML(` --- map: alice: 25 val: (( catch(validate(map, [ "mapfield", "alice", [ "type", "string"]])) )) `) - resolved := parseYAML(` + resolved := parseYAML(` --- map: alice: 25 @@ -539,67 +567,67 @@ val: valid: false error: 'condition 1 failed: map entry "alice" is not of type string' `) - Expect(source).To(FlowAs(resolved)) + Expect(source).To(FlowAs(resolved)) + }) }) - }) - Context("optionalfield", func() { - It("accept exists", func() { - source := parseYAML(` + Context("optionalfield", func() { + It("accept exists", func() { + source := parseYAML(` --- map: alice: 25 val: (( validate(map, [ "optionalfield", "alice" ]) )) `) - resolved := parseYAML(` + resolved := parseYAML(` --- map: alice: 25 val: alice: 25 `) - Expect(source).To(FlowAs(resolved)) - }) - It("accept validated", func() { - source := parseYAML(` + Expect(source).To(FlowAs(resolved)) + }) + It("accept validated", func() { + source := parseYAML(` --- map: alice: 25 val: (( validate(map, [ "optionalfield", "alice", [ "type", "int" ]]) )) `) - resolved := parseYAML(` + resolved := parseYAML(` --- map: alice: 25 val: alice: 25 `) - Expect(source).To(FlowAs(resolved)) - }) - It("accept not exists", func() { - source := parseYAML(` + Expect(source).To(FlowAs(resolved)) + }) + It("accept not exists", func() { + source := parseYAML(` --- map: alice: 25 val: (( validate(map, [ "optionalfield", "bob" ]) )) `) - resolved := parseYAML(` + resolved := parseYAML(` --- map: alice: 25 val: alice: 25 `) - Expect(source).To(FlowAs(resolved)) - }) - It("fail validator", func() { - source := parseYAML(` + Expect(source).To(FlowAs(resolved)) + }) + It("fail validator", func() { + source := parseYAML(` --- map: alice: 25 val: (( catch(validate(map, [ "optionalfield", "alice", [ "type", "string"]])) )) `) - resolved := parseYAML(` + resolved := parseYAML(` --- map: alice: 25 @@ -607,99 +635,99 @@ val: valid: false error: 'condition 1 failed: map entry "alice" is not of type string' `) - Expect(source).To(FlowAs(resolved)) + Expect(source).To(FlowAs(resolved)) + }) }) - }) - Context("map", func() { - It("accepts map", func() { - source := parseYAML(` + Context("map", func() { + It("accepts map", func() { + source := parseYAML(` --- map: alice: 25 val: (( validate(map, "map") )) `) - resolved := parseYAML(` + resolved := parseYAML(` --- map: alice: 25 val: alice: 25 `) - Expect(source).To(FlowAs(resolved)) - }) - It("accept keys", func() { - source := parseYAML(` + Expect(source).To(FlowAs(resolved)) + }) + It("accept keys", func() { + source := parseYAML(` --- map: alice: 25 val: (( validate(map, [ "map", "dnslabel", ~ ]) )) `) - resolved := parseYAML(` + resolved := parseYAML(` --- map: alice: 25 val: alice: 25 `) - Expect(source).To(FlowAs(resolved)) - }) - It("accept values", func() { - source := parseYAML(` + Expect(source).To(FlowAs(resolved)) + }) + It("accept values", func() { + source := parseYAML(` --- map: alice: 25 val: (( validate(map, [ "map", ["type", "int"] ]) )) `) - resolved := parseYAML(` + resolved := parseYAML(` --- map: alice: 25 val: alice: 25 `) - Expect(source).To(FlowAs(resolved)) - }) - It("accept key and value", func() { - source := parseYAML(` + Expect(source).To(FlowAs(resolved)) + }) + It("accept key and value", func() { + source := parseYAML(` --- map: alice: 25 val: (( validate(map, [ "map", "dnslabel", ["type", "int"] ]) )) `) - resolved := parseYAML(` + resolved := parseYAML(` --- map: alice: 25 val: alice: 25 `) - Expect(source).To(FlowAs(resolved)) - }) + Expect(source).To(FlowAs(resolved)) + }) - It("rejects non-map", func() { - source := parseYAML(` + It("rejects non-map", func() { + source := parseYAML(` --- map: [] val: (( catch(validate(map, "map")) )) `) - resolved := parseYAML(` + resolved := parseYAML(` --- map: [] val: valid: false error: 'condition 1 failed: is no map' `) - Expect(source).To(FlowAs(resolved)) - }) - It("rejects keys", func() { - source := parseYAML(` + Expect(source).To(FlowAs(resolved)) + }) + It("rejects keys", func() { + source := parseYAML(` --- map: alice: 25 val: (( catch(validate(map, [ "map", "ip", ~ ])) )) `) - resolved := parseYAML(` + resolved := parseYAML(` --- map: alice: 25 @@ -707,16 +735,16 @@ val: valid: false error: 'condition 1 failed: map key "alice" is no ip address: alice' `) - Expect(source).To(FlowAs(resolved)) - }) - It("reject values", func() { - source := parseYAML(` + Expect(source).To(FlowAs(resolved)) + }) + It("reject values", func() { + source := parseYAML(` --- map: alice: 25 val: (( catch(validate(map, [ "map", ["type", "string"] ])) )) `) - resolved := parseYAML(` + resolved := parseYAML(` --- map: alice: 25 @@ -724,17 +752,17 @@ val: valid: false error: 'condition 1 failed: map entry "alice" is not of type string' `) - Expect(source).To(FlowAs(resolved)) - }) + Expect(source).To(FlowAs(resolved)) + }) - It("reject key and value", func() { - source := parseYAML(` + It("reject key and value", func() { + source := parseYAML(` --- map: alice: 25 val: (( catch(validate(map, [ "map", "dnslabel", ["type", "string"] ])) )) `) - resolved := parseYAML(` + resolved := parseYAML(` --- map: alice: 25 @@ -742,127 +770,155 @@ val: valid: false error: 'condition 1 failed: map entry "alice" is not of type string' `) - Expect(source).To(FlowAs(resolved)) + Expect(source).To(FlowAs(resolved)) + }) }) - }) - Context("lambda", func() { - It("accepts", func() { - source := parseYAML(` + Context("lambda", func() { + It("accepts", func() { + source := parseYAML(` --- val: (( validate(5, |x|-> x >= 5) )) `) - resolved := parseYAML(` + resolved := parseYAML(` --- val: 5 `) - Expect(source).To(FlowAs(resolved)) - }) + Expect(source).To(FlowAs(resolved)) + }) - It("rejects", func() { - source := parseYAML(` + It("rejects", func() { + source := parseYAML(` --- val: (( catch(validate(4, |x|-> x >= 5)) )) `) - resolved := parseYAML(` + resolved := parseYAML(` --- val: valid: false error: 'condition 1 failed: lambda|x|->x >= 5 failed' `) - Expect(source).To(FlowAs(resolved)) - }) + Expect(source).To(FlowAs(resolved)) + }) - It("accepts with arg", func() { - source := parseYAML(` + It("accepts with arg", func() { + source := parseYAML(` --- val: (( validate(5, [ |x,m|-> x >= m, 5 ]) )) `) - resolved := parseYAML(` + resolved := parseYAML(` --- val: 5 `) - Expect(source).To(FlowAs(resolved)) - }) + Expect(source).To(FlowAs(resolved)) + }) - It("rejects with arg", func() { - source := parseYAML(` + It("rejects with arg", func() { + source := parseYAML(` --- val: (( catch(validate(4, [ |x,m|-> x >= m, 5 ])) )) `) - resolved := parseYAML(` + resolved := parseYAML(` --- val: valid: false error: 'condition 1 failed: lambda|x,m|->x >= m failed' `) - Expect(source).To(FlowAs(resolved)) - }) + Expect(source).To(FlowAs(resolved)) + }) - It("accepts with arg and message", func() { - source := parseYAML(` + It("accepts with arg and message", func() { + source := parseYAML(` --- val: (( validate(5, [ |x,m|-> [x >= m, "is larger than or equal to " m], 5 ]) )) `) - resolved := parseYAML(` + resolved := parseYAML(` --- val: 5 `) - Expect(source).To(FlowAs(resolved)) - }) + Expect(source).To(FlowAs(resolved)) + }) - It("rejects with arg and message", func() { - source := parseYAML(` + It("rejects with arg and message", func() { + source := parseYAML(` --- val: (( catch(validate(5, [ "not" ,[|x,m|-> [ x >= m, "is larger than or equal to " m] , 5 ]])) )) `) - resolved := parseYAML(` + resolved := parseYAML(` --- val: valid: false error: 'condition 1 failed: is larger than or equal to 5' `) - Expect(source).To(FlowAs(resolved)) - }) + Expect(source).To(FlowAs(resolved)) + }) - It("accepts with arg and messages", func() { - source := parseYAML(` + It("accepts with arg and messages", func() { + source := parseYAML(` --- val: (( validate(5, [ |x,m|-> [x >= m, "is larger than or equal to " m, "is less than " m], 5 ]) )) `) - resolved := parseYAML(` + resolved := parseYAML(` --- val: 5 `) - Expect(source).To(FlowAs(resolved)) - }) + Expect(source).To(FlowAs(resolved)) + }) - It("rejects with arg and messages 1", func() { - source := parseYAML(` + It("rejects with arg and messages 1", func() { + source := parseYAML(` --- val: (( catch(validate(5, [ "!" , [ |x,m|-> [x >= m, "is larger than or equal to " m, "is less than " m], 5 ]])) )) `) - resolved := parseYAML(` + resolved := parseYAML(` --- val: valid: false error: 'condition 1 failed: is larger than or equal to 5' `) - Expect(source).To(FlowAs(resolved)) - }) + Expect(source).To(FlowAs(resolved)) + }) - It("rejects with arg and message 2", func() { - source := parseYAML(` + It("rejects with arg and message 2", func() { + source := parseYAML(` --- val: (( catch(validate(4, [|x,m|-> [ x >= m, "is larger than or equal to " m, "is less than " m] , 5 ])) )) `) - resolved := parseYAML(` + resolved := parseYAML(` --- val: valid: false error: 'condition 1 failed: is less than 5' `) - Expect(source).To(FlowAs(resolved)) + Expect(source).To(FlowAs(resolved)) + }) + }) + }) + + Context("check", func() { + Context("value", func() { + It("accepts", func() { + source := parseYAML(` +--- +val: (( check("alice", [ "value", "alice" ]) )) +`) + resolved := parseYAML(` +--- +val: true +`) + Expect(source).To(FlowAs(resolved)) + }) + It("rejects", func() { + source := parseYAML(` +--- +val: (( check("peter", [ "value", "alice" ]) )) +`) + resolved := parseYAML(` +--- +val: false +`) + Expect(source).To(FlowAs(resolved)) + }) }) }) }) From 3687de5100f57166b5b9065538e73d995101764a Mon Sep 17 00:00:00 2001 From: Uwe Krueger Date: Thu, 3 Dec 2020 10:00:03 +0100 Subject: [PATCH 04/10] function contains_ip --- .gitignore | 1 + README.md | 4 +++- dynaml/call.go | 3 +++ dynaml/call_test.go | 49 +++++++++++++++++++++++++++++++++++++++++++++ dynaml/ip.go | 35 ++++++++++++++++++++++++++++++++ 5 files changed, 91 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 27d9474..061a3d6 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,7 @@ *.sw[pq] tags .idea +.godev /local /spiff /spiff++ diff --git a/README.md b/README.md index c8a38eb..ebaabe9 100644 --- a/README.md +++ b/README.md @@ -1240,6 +1240,7 @@ cidr: 192.168.0.1/24 range: (( min_ip(cidr) "-" max_ip(cidr) )) next: (( max_ip(cidr) + 1 )) num: (( min_ip(cidr) "+" num_ip(cidr) "=" min_ip(cidr) + num_ip(cidr) )) +contains: (( contains_ip(cidr, "192.168.0.2") )) ``` yields @@ -1249,6 +1250,7 @@ cidr: 192.168.0.1/24 range: 192.168.0.0-192.168.0.255 next: 192.168.1.0 num: 192.168.0.0+256=192.168.1.0 +contains: true ``` ## `(( a > 1 ? foo :bar ))` @@ -3565,7 +3567,7 @@ static scope of the lambda dedinition followed by the static yaml scope of the caller. Absolute references are always evalated in the document scope of the caller. -The name `_` can also be used as an anchor to refer to the static dfinition +The name `_` can also be used as an anchor to refer to the static definition scope of the lambda expression in the yaml document that was used to define the lambda function. Those references are always interpreted as relative references related to the this static yaml document scope. There is no diff --git a/dynaml/call.go b/dynaml/call.go index 5d80359..f798145 100644 --- a/dynaml/call.go +++ b/dynaml/call.go @@ -254,6 +254,9 @@ func (e CallExpr) Evaluate(binding Binding, locally bool) (interface{}, Evaluati case "num_ip": result, sub, ok = func_numIP(values, binding) + case "contains_ip": + result, sub, ok = func_containsIP(values, binding) + case "makemap": result, sub, ok = func_makemap(values, binding) diff --git a/dynaml/call_test.go b/dynaml/call_test.go index 7902f5a..a7d6e6b 100644 --- a/dynaml/call_test.go +++ b/dynaml/call_test.go @@ -24,6 +24,55 @@ func newNetworkFakeBinding(subnets yaml.Node, instances interface{}) Binding { var _ = Describe("calls", func() { Describe("CIDR functions", func() { + It("contains IP", func() { + expr := CallExpr{ + Function: ReferenceExpr{[]string{"contains_ip"}}, + Arguments: []Expression{ + StringExpr{"192.168.1.1/24"}, + StringExpr{"192.168.1.2"}, + }, + } + + Expect(expr).To( + EvaluateAs( + true, + FakeBinding{}, + ), + ) + }) + It("not contains IP", func() { + expr := CallExpr{ + Function: ReferenceExpr{[]string{"contains_ip"}}, + Arguments: []Expression{ + StringExpr{"192.168.1.1/24"}, + StringExpr{"192.168.2.0"}, + }, + } + + Expect(expr).To( + EvaluateAs( + false, + FakeBinding{}, + ), + ) + }) + It("not contains IP", func() { + expr := CallExpr{ + Function: ReferenceExpr{[]string{"contains_ip"}}, + Arguments: []Expression{ + StringExpr{"192.168.1.1/24"}, + StringExpr{"192.168.0.255"}, + }, + } + + Expect(expr).To( + EvaluateAs( + false, + FakeBinding{}, + ), + ) + }) + It("determines minimal IP", func() { expr := CallExpr{ Function: ReferenceExpr{[]string{"min_ip"}}, diff --git a/dynaml/ip.go b/dynaml/ip.go index 61cfc1d..c1e4c33 100644 --- a/dynaml/ip.go +++ b/dynaml/ip.go @@ -30,6 +30,41 @@ func func_ip(op func(ip net.IP, cidr *net.IPNet) interface{}, arguments []interf return op(ip, cidr), info, true } +func func_containsIP(arguments []interface{}, binding Binding) (interface{}, EvaluationInfo, bool) { + info := DefaultInfo() + + if len(arguments) != 2 { + info.Issue = yaml.NewIssue("contains_ip requires CIDR and IP argument") + return nil, info, false + } + + str, ok := arguments[0].(string) + if !ok { + info.Issue = yaml.NewIssue("CIDR required as first argument") + return nil, info, false + } + + _, cidr, err := net.ParseCIDR(str) + + if err != nil { + info.Issue = yaml.NewIssue("CIDR argument required: %s", err) + return nil, info, false + } + + str, ok = arguments[1].(string) + if !ok { + info.Issue = yaml.NewIssue("IP required as second argument") + return nil, info, false + } + + ip := net.ParseIP(str) + if ip == nil { + info.Issue = yaml.NewIssue("IP argument required: %s", str) + return nil, info, false + } + return cidr.Contains(ip), info, true +} + func func_minIP(arguments []interface{}, binding Binding) (interface{}, EvaluationInfo, bool) { return func_ip(func(ip net.IP, cidr *net.IPNet) interface{} { return ip.Mask(cidr.Mask).String() From 6b300cd5b726105dcd3f6133585d8279452b5fb6 Mon Sep 17 00:00:00 2001 From: Uwe Krueger Date: Fri, 4 Dec 2020 10:42:21 +0100 Subject: [PATCH 05/10] allow merging root nodes --- dynaml/merge.go | 7 +++++-- dynaml/parser.go | 6 ++++-- dynaml/parser_test.go | 14 +++++++------- flow/cascade_test.go | 21 +++++++++++++++++++++ flow/environment.go | 4 ++++ flow/flow.go | 3 +++ spiffing/interface.go | 3 +++ 7 files changed, 47 insertions(+), 11 deletions(-) diff --git a/dynaml/merge.go b/dynaml/merge.go index 4602de4..bc06503 100644 --- a/dynaml/merge.go +++ b/dynaml/merge.go @@ -1,8 +1,9 @@ package dynaml import ( - "github.com/mandelsoft/spiff/debug" "strings" + + "github.com/mandelsoft/spiff/debug" ) type MergeExpr struct { @@ -10,6 +11,7 @@ type MergeExpr struct { Redirect bool Replace bool Required bool + None bool KeyName string } @@ -20,7 +22,8 @@ func (e MergeExpr) Evaluate(binding Binding, locally bool) (interface{}, Evaluat if e.Redirect { info.RedirectPath = e.Path } - if len(e.Path) == 0 { + // if len(e.Path) == 0 { + if e.None { info.Merged = true return nil, info, true } diff --git a/dynaml/parser.go b/dynaml/parser.go index f41d8f6..2c7aaf7 100644 --- a/dynaml/parser.go +++ b/dynaml/parser.go @@ -179,15 +179,17 @@ func buildExpression(grammar *DynamlGrammar, path []string, stubPath []string) ( case ruleSimpleMerge: debug.Debug("*** rule simple merge\n") redirect := !equals(path, stubPath) - tokens.Push(MergeExpr{stubPath, redirect, replace, replace || required || redirect, keyName}) + tokens.Push(MergeExpr{stubPath, redirect, replace, replace || required || redirect, false, keyName}) case ruleRefMerge: debug.Debug("*** rule ref merge\n") rhs := tokens.Pop() merge := rhs.(ReferenceExpr).Path + none := false if len(merge) == 1 && merge[0] == "none" { merge = []string{} + none = true } - tokens.Push(MergeExpr{merge, true, replace, len(merge) > 0, keyName}) + tokens.Push(MergeExpr{merge, true, replace, len(merge) > 0, none, keyName}) case ruleReplace: replace = true case ruleRequired: diff --git a/dynaml/parser_test.go b/dynaml/parser_test.go index 7252631..0dfc7d5 100644 --- a/dynaml/parser_test.go +++ b/dynaml/parser_test.go @@ -37,31 +37,31 @@ var _ = Describe("parsing", func() { Describe("merge", func() { It("parses as a merge node with the given path", func() { - parsesAs("merge alice.bob", MergeExpr{[]string{"alice", "bob"}, true, false, true, ""}, "foo", "bar") + parsesAs("merge alice.bob", MergeExpr{[]string{"alice", "bob"}, true, false, true, false, ""}, "foo", "bar") }) It("parses as a merge node with the environment path", func() { - parsesAs("merge", MergeExpr{[]string{"foo", "bar"}, false, false, false, ""}, "foo", "bar") + parsesAs("merge", MergeExpr{[]string{"foo", "bar"}, false, false, false, false, ""}, "foo", "bar") }) It("parses as a merge replace node with the given path", func() { - parsesAs("merge replace alice.bob", MergeExpr{[]string{"alice", "bob"}, true, true, true, ""}, "foo", "bar") + parsesAs("merge replace alice.bob", MergeExpr{[]string{"alice", "bob"}, true, true, true, false, ""}, "foo", "bar") }) It("parses as a merge replace node with the environment path", func() { - parsesAs("merge replace", MergeExpr{[]string{"foo", "bar"}, false, true, true, ""}, "foo", "bar") + parsesAs("merge replace", MergeExpr{[]string{"foo", "bar"}, false, true, true, false, ""}, "foo", "bar") }) It("parses as a merge require node", func() { - parsesAs("merge required", MergeExpr{[]string{"foo", "bar"}, false, false, true, ""}, "foo", "bar") + parsesAs("merge required", MergeExpr{[]string{"foo", "bar"}, false, false, true, false, ""}, "foo", "bar") }) It("parses as a merge require node", func() { - parsesAs("merge on key", MergeExpr{[]string{"foo", "bar"}, false, false, false, "key"}, "foo", "bar") + parsesAs("merge on key", MergeExpr{[]string{"foo", "bar"}, false, false, false, false, "key"}, "foo", "bar") }) It("parses as a merge require node", func() { - parsesAs("merge on key alice.bob", MergeExpr{[]string{"alice", "bob"}, true, false, true, "key"}, "foo", "bar") + parsesAs("merge on key alice.bob", MergeExpr{[]string{"alice", "bob"}, true, false, true, false, "key"}, "foo", "bar") }) }) diff --git a/flow/cascade_test.go b/flow/cascade_test.go index b7cd18d..43dc2be 100644 --- a/flow/cascade_test.go +++ b/flow/cascade_test.go @@ -37,6 +37,27 @@ baz: 42 Expect(source).To(CascadeAs(resolved, secondary, stub)) }) + It("merges root node", func() { + source := parseYAML(` +--- +<<: (( merge )) +bar: alice +`) + + stub := parseYAML(` +--- +foo: bob +`) + + resolved := parseYAML(` +--- +foo: bob +bar: alice +`) + + Expect(source).To(CascadeAs(resolved, stub)) + }) + Context("with multiple mutually-exclusive templates", func() { It("flows through both", func() { source := parseYAML(` diff --git a/flow/environment.go b/flow/environment.go index 3607ac0..56b355e 100644 --- a/flow/environment.go +++ b/flow/environment.go @@ -180,12 +180,16 @@ func (e DefaultEnvironment) FindReference(path []string) (yaml.Node, bool) { } func (e DefaultEnvironment) FindInStubs(path []string) (yaml.Node, bool) { + debug.Debug("lookup %v in stubs\n", path) for _, stub := range e.stubs { + debug.Debug("checking stub %s\n", stub.SourceName()) val, found := yaml.Find(stub, path...) if found { if !val.Flags().Implied() { + debug.Debug("found %v\n", path) return val, true } + debug.Debug("skipping found stub %v\n", path) } } diff --git a/flow/flow.go b/flow/flow.go index 9d89fa5..087ad99 100644 --- a/flow/flow.go +++ b/flow/flow.go @@ -248,6 +248,7 @@ func flowMap(root yaml.Node, env dynaml.Binding) yaml.Node { return yaml.IssueNode(root, true, true, yaml.NewIssue("multiple merge keys not allowed")) } mergefound = true + debug.Debug("handle map merge %#v\n", val) _, initial := val.Value().(string) base := flow(val, env, false) if base.Undefined() { @@ -281,8 +282,10 @@ func flowMap(root yaml.Node, env dynaml.Binding) yaml.Node { debug.Debug(" insert expression: %v\n", val) } else { if simpleMergeCompatibilityCheck(initial, base) { + debug.Debug(" skip merge\n") continue } + debug.Debug(" continue merge\n") val = base } processed = false diff --git a/spiffing/interface.go b/spiffing/interface.go index 9924202..c5464bc 100644 --- a/spiffing/interface.go +++ b/spiffing/interface.go @@ -8,6 +8,9 @@ import ( "github.com/mandelsoft/spiff/yaml" ) +// MODE_PRIVATE does not allow access to any external resources +const MODE_PRIVATE = 0 + // MODE_OS_ACCESS allows os command execution (pipe, exec) const MODE_OS_ACCESS = flow.MODE_OS_ACCESS From 2c5ea71bbbbf09782d812df38979374f792c6923 Mon Sep 17 00:00:00 2001 From: Uwe Krueger Date: Fri, 4 Dec 2020 11:12:10 +0100 Subject: [PATCH 06/10] spiffing util functions --- spiffing/utils.go | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/spiffing/utils.go b/spiffing/utils.go index 744b995..c6710b9 100644 --- a/spiffing/utils.go +++ b/spiffing/utils.go @@ -3,6 +3,8 @@ package spiffing import ( "fmt" "strings" + + "github.com/mandelsoft/spiff/yaml" ) // Process just processes a template with the values set in the execution @@ -82,3 +84,11 @@ func Cascade(s Spiff, template Source, stubs []Source, optstate ...Source) ([]by } return rdata, nil, err } + +func ToNode(name string, data interface{}) (Node, error) { + return yaml.Sanitize(name, data) +} + +func Normalize(n Node) (interface{}, error) { + return yaml.Normalize(n) +} From d51e9387d706dc618fe525dc72bf85f71e5db82a Mon Sep 17 00:00:00 2001 From: Uwe Krueger Date: Fri, 12 Feb 2021 12:03:17 +0100 Subject: [PATCH 07/10] convert sub command and option --define --- README.md | 11 ++++ cmd/convert.go | 136 +++++++++++++++++++++++++++++++++++++++++++++++++ cmd/merge.go | 47 ++++++++++++++++- cmd/run.go | 16 ++++-- 4 files changed, 203 insertions(+), 7 deletions(-) create mode 100644 cmd/convert.go diff --git a/README.md b/README.md index ebaabe9..42f0216 100644 --- a/README.md +++ b/README.md @@ -241,6 +241,10 @@ The ` merge` command offers several options: is used to build additional bindings for the processing. The yaml document must consist of a map. Each key is used as additional binding. The bindings document is not processed, the values are used as defined. + +- With option `--define =` (shorthand`-D`) additional binding values + can be specified on the command line overriding binding values from the + binding file. The option may occur multiple times. - The option `--preserve-escapes` will preserve the escaping for dynaml expressions and list/map merge directives. This option can be used @@ -285,6 +289,13 @@ $ bosh deployment deployment.yml $ bosh deploy ``` +### `spiff convert --json manifest.yml ` + +The `convert` sub command can be used to convert input files to json or +just to normalize the order of the fields. +Available options are `--json`, `--path`, `--split` or `--select` according +to their meanings for the `merge` sub command. + ### `spiff encrypt secret.yaml` The `encrypt` sub command can be used to encrypt or decrypt data diff --git a/cmd/convert.go b/cmd/convert.go new file mode 100644 index 0000000..8cd50b5 --- /dev/null +++ b/cmd/convert.go @@ -0,0 +1,136 @@ +package cmd + +import ( + "errors" + "fmt" + "io/ioutil" + "log" + "os" + "path" + + "github.com/spf13/cobra" + + "github.com/mandelsoft/spiff/dynaml" + "github.com/mandelsoft/spiff/legacy/candiedyaml" + "github.com/mandelsoft/spiff/yaml" +) + +// convertCmd represents the merge command +var convertCmd = &cobra.Command{ + Use: "convert", + Aliases: []string{"c"}, + Short: "Convert template", + Long: `A given template file is normalized and converted to json or yaml.`, + Args: func(cmd *cobra.Command, args []string) error { + if len(args) != 1 { + return errors.New("requires at one arg") + } + return nil + }, + Run: func(cmd *cobra.Command, args []string) { + convert(false, args[0], asJSON, split, outputPath, selection) + }, +} + +func init() { + rootCmd.AddCommand(convertCmd) + + convertCmd.Flags().BoolVar(&asJSON, "json", false, "print output in json format") + convertCmd.Flags().StringVar(&outputPath, "path", "", "output is taken from given path") + convertCmd.Flags().BoolVar(&split, "split", false, "if the output is alist it will be split into separate documents") + convertCmd.Flags().StringArrayVar(&selection, "select", []string{}, "filter dedicated output fields") +} + +func convert(stdin bool, templateFilePath string, json, split bool, subpath string, selection []string) { + var templateFile []byte + var err error + + if templateFilePath == "-" { + templateFile, err = ioutil.ReadAll(os.Stdin) + stdin = true + } else { + templateFile, err = ReadFile(templateFilePath) + } + + if err != nil { + log.Fatalln(fmt.Sprintf("error reading template [%s]:", path.Clean(templateFilePath)), err) + } + + templateYAMLs, err := yaml.ParseMulti(templateFilePath, templateFile) + if err != nil { + log.Fatalln(fmt.Sprintf("error parsing template [%s]:", path.Clean(templateFilePath)), err) + } + + result := [][]byte{} + count := 0 + for no, templateYAML := range templateYAMLs { + doc := "" + if len(templateYAMLs) > 1 { + doc = fmt.Sprintf(" (document %d)", no+1) + } + var bytes []byte + if templateYAML.Value() != nil { + count++ + flowed := templateYAML + if subpath != "" { + comps := dynaml.PathComponents(subpath, false) + node, ok := yaml.FindR(true, flowed, comps...) + if !ok { + log.Fatalln(fmt.Sprintf("path %q not found%s", subpath, doc)) + } + flowed = node + } + + if len(selection) > 0 { + new := map[string]yaml.Node{} + for _, p := range selection { + comps := dynaml.PathComponents(p, false) + node, ok := yaml.FindR(true, flowed, comps...) + if !ok { + log.Fatalln(fmt.Sprintf("path %q not found%s", subpath, doc)) + } + new[comps[len(comps)-1]] = node + + } + flowed = yaml.NewNode(new, "") + } + if split { + if list, ok := flowed.Value().([]yaml.Node); ok { + for _, d := range list { + if json { + bytes, err = yaml.ToJSON(d) + } else { + bytes, err = candiedyaml.Marshal(d) + } + if err != nil { + log.Fatalln(fmt.Sprintf("error marshalling manifest%s:", doc), err) + } + result = append(result, bytes) + } + continue + } + } + if json { + bytes, err = yaml.ToJSON(flowed) + } else { + bytes, err = candiedyaml.Marshal(flowed) + } + if err != nil { + log.Fatalln(fmt.Sprintf("error marshalling manifest%s:", doc), err) + } + } + result = append(result, bytes) + } + + for _, bytes := range result { + if !json && (len(result) > 1 || len(bytes) == 0) { + fmt.Println("---") + } + if bytes != nil { + fmt.Print(string(bytes)) + if json { + fmt.Println() + } + } + } +} diff --git a/cmd/merge.go b/cmd/merge.go index 9dac151..248787a 100644 --- a/cmd/merge.go +++ b/cmd/merge.go @@ -7,6 +7,7 @@ import ( "log" "os" "path" + "strconv" "strings" "github.com/spf13/cobra" @@ -25,6 +26,7 @@ var split bool var processingOptions flow.Options var state string var bindings string +var values []string // mergeCmd represents the merge command var mergeCmd = &cobra.Command{ @@ -39,7 +41,11 @@ var mergeCmd = &cobra.Command{ return nil }, Run: func(cmd *cobra.Command, args []string) { - merge(false, args[0], processingOptions, asJSON, split, outputPath, selection, state, bindings, nil, args[1:]) + vals, err := createValuesFromArgs(values) + if err != nil { + log.Fatalf("%s\n", err) + } + merge(false, args[0], processingOptions, asJSON, split, outputPath, selection, state, bindings, vals, nil, args[1:]) }, } @@ -55,9 +61,28 @@ func init() { mergeCmd.Flags().BoolVar(&processingOptions.PreserveTemporary, "preserve-temporary", false, "preserve temporary fields") mergeCmd.Flags().StringVar(&state, "state", "", "select state file to maintain") mergeCmd.Flags().StringVar(&bindings, "bindings", "", "yaml file with additional bindings to use") + mergeCmd.Flags().StringArrayVarP(&values, "define", "D", nil, "key/value bindings") mergeCmd.Flags().StringArrayVar(&selection, "select", []string{}, "filter dedicated output fields") } +func createValuesFromArgs(values []string) (map[string]string, error) { + if len(values) == 0 { + return nil, nil + } + result := map[string]string{} + for _, s := range values { + parts := strings.Split(s, "=") + if len(parts) != 2 { + return nil, fmt.Errorf("invalid value definition %q\n", s) + } + if parts[0] == "" { + return nil, fmt.Errorf("empty key in value definition %q\n", s) + } + result[parts[0]] = parts[1] + } + return result, nil +} + func fileExists(filename string) bool { info, err := os.Stat(filename) if os.IsNotExist(err) { @@ -84,7 +109,7 @@ func readYAML(filename string, desc string, required bool) yaml.Node { } func merge(stdin bool, templateFilePath string, opts flow.Options, json, split bool, - subpath string, selection []string, stateFilePath, bindingFilePath string, stubs []yaml.Node, stubFilePaths []string) { + subpath string, selection []string, stateFilePath, bindingFilePath string, values map[string]string, stubs []yaml.Node, stubFilePaths []string) { var templateFile []byte var err error @@ -113,6 +138,24 @@ func merge(stdin bool, templateFilePath string, opts flow.Options, json, split b } bindingYAML := readYAML(bindingFilePath, "bindings file", true) + if len(values) > 0 { + if bindingYAML == nil { + bindingYAML = yaml.NewNode(map[string]yaml.Node{}, "") + } + m, ok := bindingYAML.Value().(map[string]yaml.Node) + if !ok { + log.Fatalf(fmt.Sprintf("binding %q must be a map\n", bindingFilePath)) + } + for k, v := range values { + i, err := strconv.ParseInt(v, 10, 64) + if err == nil { + m[k] = yaml.NewNode(i, "") + } else { + m[k] = yaml.NewNode(v, "") + } + } + } + if stubs == nil { stubs = []yaml.Node{} } diff --git a/cmd/run.go b/cmd/run.go index e15dd05..f7906cc 100644 --- a/cmd/run.go +++ b/cmd/run.go @@ -3,14 +3,16 @@ package cmd import ( "errors" "fmt" - "github.com/mandelsoft/spiff/debug" - "github.com/mandelsoft/spiff/flow" - "github.com/mandelsoft/spiff/yaml" - "github.com/spf13/cobra" "io/ioutil" "log" "os" "path" + + "github.com/spf13/cobra" + + "github.com/mandelsoft/spiff/debug" + "github.com/mandelsoft/spiff/flow" + "github.com/mandelsoft/spiff/yaml" ) // runCmd represents the merge command @@ -64,5 +66,9 @@ func run(documentFilePath, templateFilePath string, opts flow.Options, json, spl documentYAML = yaml.NewNode(map[string]yaml.Node{"document": documentYAML}, "<"+documentFilePath+">") stub := yaml.NewNode(map[string]yaml.Node{"document": yaml.NewNode("(( &temporary &inject (merge) ))", ")")}, "") - merge(stdin, templateFilePath, opts, json, split, subpath, selection, stateFilePath, bindingFilePath, []yaml.Node{stub, documentYAML}, stubFilePaths) + vals, err := createValuesFromArgs(values) + if err != nil { + log.Fatalf("%s\n", err) + } + merge(stdin, templateFilePath, opts, json, split, subpath, selection, stateFilePath, bindingFilePath, vals, []yaml.Node{stub, documentYAML}, stubFilePaths) } From eaa4f4d79b89b9e6b834b8cc7c47ff98688b382a Mon Sep 17 00:00:00 2001 From: Uwe Krueger Date: Fri, 12 Feb 2021 12:03:38 +0100 Subject: [PATCH 08/10] wireguard key support --- dynaml/wireguard/genkey.go | 41 +++++++++++++++++ dynaml/wireguard/key.go | 91 ++++++++++++++++++++++++++++++++++++++ dynaml/wireguard/pubkey.go | 29 ++++++++++++ flow/flow.go | 1 + 4 files changed, 162 insertions(+) create mode 100644 dynaml/wireguard/genkey.go create mode 100644 dynaml/wireguard/key.go create mode 100644 dynaml/wireguard/pubkey.go diff --git a/dynaml/wireguard/genkey.go b/dynaml/wireguard/genkey.go new file mode 100644 index 0000000..2356f54 --- /dev/null +++ b/dynaml/wireguard/genkey.go @@ -0,0 +1,41 @@ +package wireguard + +import ( + . "github.com/mandelsoft/spiff/dynaml" +) + +const F_GenKey = "wggenkey" + +func init() { + RegisterFunction(F_GenKey, func_genkey) +} + +func func_genkey(arguments []interface{}, binding Binding) (interface{}, EvaluationInfo, bool) { + info := DefaultInfo() + + if len(arguments) > 1 { + return info.Error("a maximum of one argument expected for %s", F_GenKey) + } + ktype := "private" + if len(arguments) == 1 { + str, ok := arguments[0].(string) + if !ok { + return info.Error("argument for %s must be a string (private or preshared)", F_GenKey) + } + ktype = str + } + var key Key + var err error + switch ktype { + case "private": + key, err = GeneratePrivateKey() + case "preshared": + key, err = GenerateKey() + default: + return info.Error("ainvalid key type %q, use private or preshared", ktype) + } + if err != nil { + return info.Error("error generating key: %s", err) + } + return key.String(), info, true +} diff --git a/dynaml/wireguard/key.go b/dynaml/wireguard/key.go new file mode 100644 index 0000000..5274843 --- /dev/null +++ b/dynaml/wireguard/key.go @@ -0,0 +1,91 @@ +package wireguard + +import ( + "encoding/base64" + "fmt" + "math/rand" + + "golang.org/x/crypto/curve25519" +) + +// KeyLen is the expected key length for a WireGuard key. +const KeyLen = 32 // wgh.KeyLen + +// A Key is a public, private, or pre-shared secret key. The Key constructor +// functions in this package can be used to create Keys suitable for each of +// these applications. +type Key [KeyLen]byte + +// GenerateKey generates a Key suitable for use as a pre-shared secret key from +// a cryptographically safe source. +// +// The output Key should not be used as a private key; use GeneratePrivateKey +// instead. +func GenerateKey() (Key, error) { + b := make([]byte, KeyLen) + if _, err := rand.Read(b); err != nil { + return Key{}, fmt.Errorf("wgtypes: failed to read random bytes: %v", err) + } + + return NewKey(b) +} + +// GeneratePrivateKey generates a Key suitable for use as a private key from a +// cryptographically safe source. +func GeneratePrivateKey() (Key, error) { + key, err := GenerateKey() + if err != nil { + return Key{}, err + } + + // Modify random bytes using algorithm described at: + // https://cr.yp.to/ecdh.html. + key[0] &= 248 + key[31] &= 127 + key[31] |= 64 + + return key, nil +} + +// NewKey creates a Key from an existing byte slice. +// The byte slice must be exactly 32 bytes in length. +func NewKey(b []byte) (Key, error) { + if len(b) != KeyLen { + return Key{}, fmt.Errorf("incorrect key size: %d", len(b)) + } + + var k Key + copy(k[:], b) + + return k, nil +} + +// ParseKey parses a Key from a base64-encoded string, as produced by the +// Key.String method. +func ParseKey(s string) (Key, error) { + b, err := base64.StdEncoding.DecodeString(s) + if err != nil { + return Key{}, fmt.Errorf("failed to parse base64-encoded key: %v", err) + } + return NewKey(b) +} + +// PublicKey computes a public key from the private key k. +func (k Key) PublicKey() Key { + var ( + pub [KeyLen]byte + priv = [KeyLen]byte(k) + ) + + // ScalarBaseMult uses the correct base value per https://cr.yp.to/ecdh.html, + // so no need to specify it. + curve25519.ScalarBaseMult(&pub, &priv) + + return Key(pub) +} + +// String returns the base64-encoded string representation of a Key. +// Result can be parsed by ParseKey, again +func (k Key) String() string { + return base64.StdEncoding.EncodeToString(k[:]) +} diff --git a/dynaml/wireguard/pubkey.go b/dynaml/wireguard/pubkey.go new file mode 100644 index 0000000..96e2340 --- /dev/null +++ b/dynaml/wireguard/pubkey.go @@ -0,0 +1,29 @@ +package wireguard + +import ( + . "github.com/mandelsoft/spiff/dynaml" +) + +const F_PubKey = "wgpublickey" + +func init() { + RegisterFunction(F_PubKey, func_pubkey) +} + +func func_pubkey(arguments []interface{}, binding Binding) (interface{}, EvaluationInfo, bool) { + info := DefaultInfo() + + if len(arguments) != 1 { + return info.Error("one argument required for %q", F_PubKey) + } + + str, ok := arguments[0].(string) + if !ok { + return info.Error("argument for %s must be a provate wireguard key (string)", F_PubKey) + } + key, err := ParseKey(str) + if err != nil { + return info.Error("error parsing key %q: %s", str, err) + } + return key.PublicKey().String(), info, true +} diff --git a/flow/flow.go b/flow/flow.go index 087ad99..5a6ad73 100644 --- a/flow/flow.go +++ b/flow/flow.go @@ -10,6 +10,7 @@ import ( "github.com/mandelsoft/spiff/yaml" _ "github.com/mandelsoft/spiff/dynaml/passwd" + _ "github.com/mandelsoft/spiff/dynaml/wireguard" _ "github.com/mandelsoft/spiff/dynaml/x509" ) From ef36d4c5bf7667392ba3e267d6c7edba0ccbc2a8 Mon Sep 17 00:00:00 2001 From: Uwe Krueger Date: Tue, 2 Mar 2021 15:17:29 +0100 Subject: [PATCH 09/10] public key extractio from certificate --- README.md | 8 ++++---- dynaml/x509/publickey.go | 9 +++++++-- 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 42f0216..e29f66b 100644 --- a/README.md +++ b/README.md @@ -3223,7 +3223,7 @@ key: |+ #### `(( x509publickey(key) ))` -For a given key in PEM format (for example generated with the [x509genkey](#-x509genkeyspec-) +For a given key or certificate in PEM format (for example generated with the [x509genkey](#-x509genkeyspec-) function) this function extracts the public key and returns it again in PEM format as a multi-line string. @@ -3262,7 +3262,7 @@ to `ssh`. The result will then be a regular public key format usable for ssh. The default format is `pem` providing the pem output format shown above. RSA keys are by default marshalled in PKCS#1 format(`RSA PUBLIC KEY`) in pem. -If the the generic *PKIX* format (`PUBLIC KEY`) is required the format +If the generic *PKIX* format (`PUBLIC KEY`) is required the format argument `pkix` must be given. Using the format `ssh` this function can also be used to convert a pem formatted @@ -3271,7 +3271,7 @@ public key into an ssh key, #### `(( x509cert(spec) ))` The function `x509cert` creates locally signed certificates, either a self signed -one or a certificate signed by a given ca. It returns PEM encoded certificate +one or a certificate signed by a given ca. It returns a PEM encoded certificate as a multi-line string value. The single _spec_ parameter take a map with some optional and non optional @@ -3291,7 +3291,7 @@ The following map fields are observed: | `validity` | integer | optional | validity interval in hours | | `validFrom` | string | optional | start time in the format "Jan 1 01:22:31 2019" | | `hosts` | string or string list | optional | List of DNS names or IP addresses | -| `privateKey` | string | required or publicKey | private key to geberate the certificate for | +| `privateKey` | string | required or publicKey | private key to generate the certificate for | | `publicKey` | string | required or privateKey| public key to generate the certificate for | | `caCert` | string | optional| certificate to sign with | | `caPrivateKey` | string | optional| priavte key for `caCert` | diff --git a/dynaml/x509/publickey.go b/dynaml/x509/publickey.go index fbdb1b0..760baba 100644 --- a/dynaml/x509/publickey.go +++ b/dynaml/x509/publickey.go @@ -5,9 +5,10 @@ import ( "bytes" "encoding/pem" "fmt" - . "github.com/mandelsoft/spiff/dynaml" "strings" + . "github.com/mandelsoft/spiff/dynaml" + "golang.org/x/crypto/ssh" ) @@ -54,7 +55,11 @@ func func_x509publickey(arguments []interface{}, binding Binding) (interface{}, if err != nil { k, e := ParsePublicKey(str) if e != nil { - return info.Error("argument for %s must be a private key in pem format: %s", F_PublicKey, err) + cert, e := ParseCertificate(str) + if e != nil { + return info.Error("argument for %s must be a private/public key or certificate in pem format: %s", F_PublicKey, err) + } + k = publicKey(cert) } key = k } From 005a5fe59fe6cf37a5e2e2bfbb93f97025fabbd6 Mon Sep 17 00:00:00 2001 From: Uwe Krueger Date: Thu, 18 Mar 2021 22:48:22 +0100 Subject: [PATCH 10/10] binding access in dynaml --- README.md | 46 +++++++++++++++++++++++ flow/environment.go | 78 ++++++++++++++++++++++++++++----------- spiffing/spiffing_test.go | 42 +++++++++++++++++++++ yaml/node.go | 1 + 4 files changed, 146 insertions(+), 21 deletions(-) diff --git a/README.md b/README.md index e29f66b..8ded37e 100644 --- a/README.md +++ b/README.md @@ -168,6 +168,7 @@ Contents: - [Scope References](#scope-references) - [_](#_) - [__](#__) + - [___](#___) - [__ctx.OUTER](#__ctxouter) - [Special Literals](#special-literals) - [Access to evaluation context](#access-to-evaluation-context) @@ -4779,6 +4780,48 @@ result: bob: static ``` + +### `___` + +The special reference `___` can be used to lookup references in the outer most +scope. It can therefore be used to access processing bindings specified for a +document processing via command line or API. If no bindings are specified +the document root is used. + +Calling `spiff merge template.yaml --bindings bindings.yaml` with a binding of + +**bindings.yaml** +```yaml +input1: binding1 +input2: binding2 +``` + +and the template + +**template.yaml** +```yaml +input1: top1 +map: + input: map + input1: map1 + + results: + frommap: (( input1 )) + fromroot: (( .input1 )) + frombinding1: (( ___.input1 )) + frombinding2: (( input2 )) +``` + +evaluates `map.results` to + +```yaml + results: + frombinding1: binding1 + frombinding2: binding2 + frommap: map1 + fromroot: top1 +``` + ### `__ctx.OUTER` The context field `OUTER` is used for nested [merges](#-mergemap1-map2-). @@ -4831,6 +4874,9 @@ The following fields are supported: | `PATHNAME` | string | path name of actually processed field | | `PATH` | list[string] | path name as component list | | `OUTER` | yaml doc | outer documents for nested [merges](#-mergemap1-map2-), index 0 is the next outer document | +| `BINDINGS` | yaml doc | the external bindings for the actual processing (see also [___](#___)) | + +If external bindings are specified they are the last elements in `OUTER`. e.g.: diff --git a/flow/environment.go b/flow/environment.go index 56b355e..640dfa9 100644 --- a/flow/environment.go +++ b/flow/environment.go @@ -49,7 +49,8 @@ type DefaultEnvironment struct { static map[string]yaml.Node outer dynaml.Binding - active bool + active bool + binding bool } func keys(s map[string]yaml.Node) string { @@ -146,24 +147,35 @@ func (e DefaultEnvironment) FindFromRoot(path []string) (yaml.Node, bool) { return yaml.FindR(true, yaml.NewNode(e.scope.root.local, "scope"), path...) } +func FindInScopes(nodescope *Scope, path []string) (yaml.Node, bool) { + if len(path) > 0 { + scope := nodescope + for scope != nil { + val := scope.local[path[0]] + if val != nil { + return yaml.FindR(true, val, path[1:]...) + } + scope = scope.next + } + return nil, false + } + return yaml.FindR(true, node(nodescope.local), path...) +} + func (e DefaultEnvironment) FindReference(path []string) (yaml.Node, bool) { root, found, nodescope := resolveSymbol(&e, path[0], e.scope) if !found { + if path[0] == yaml.ROOT { + var outer dynaml.Binding = e + for outer.Outer() != nil { + outer = outer.Outer() + } + return yaml.FindR(true, node(outer.GetRootBinding()), path[1:]...) + } //fmt.Printf("FIND %s: %s\n", strings.Join(path,"."), e) //fmt.Printf("FOUND %s: %v\n", strings.Join(path,"."), keys(nodescope)) if path[0] == yaml.DOCNODE && nodescope != nil { - if len(path) > 1 { - scope := nodescope - for scope != nil { - val := scope.local[path[1]] - if val != nil { - return yaml.FindR(true, val, path[2:]...) - } - scope = scope.next - } - return nil, false - } - return yaml.FindR(true, node(nodescope.local), path[1:]...) + return FindInScopes(nodescope, path[1:]) } if e.outer != nil { return e.outer.FindReference(path) @@ -290,7 +302,7 @@ func NewEnvironment(stubs []yaml.Node, source string, optstate ...*State) dynaml if state == nil { state = NewState(os.Getenv("SPIFF_ENCRYPTION_KEY"), MODE_OS_ACCESS|MODE_FILE_ACCESS) } - return DefaultEnvironment{state: state, stubs: stubs, sourceName: source, currentSourceName: source, outer: nil, active: true} + return DefaultEnvironment{state: state, stubs: stubs, sourceName: source, currentSourceName: source, outer: nil, active: true, binding: true} } func NewProcessLocalEnvironment(stubs []yaml.Node, source string) dynaml.Binding { @@ -379,6 +391,33 @@ func resolveSymbol(env *DefaultEnvironment, name string, scope *Scope) (yaml.Nod return nil, false, nodescope } +func getOuters(env *DefaultEnvironment) (yaml.Node, yaml.Node) { + var bindings dynaml.Binding + var list []yaml.Node + + if outer := env.Outer(); outer != nil { + for outer != nil { + if e, ok := outer.(DefaultEnvironment); ok && e.binding { + bindings = outer + } + list = append(list, node(outer.GetRootBinding())) + outer = outer.Outer() + } + } + if list == nil { + if bindings != nil { + return nil, node(bindings.GetRootBinding()) + } else { + return nil, yaml.NewNode([]map[string]interface{}{}, "context") + } + } + if bindings != nil { + return node(list), node(bindings.GetRootBinding()) + } else { + return node(list), yaml.NewNode([]map[string]interface{}{}, "context") + } +} + func createContext(env *DefaultEnvironment) yaml.Node { ctx := make(map[string]yaml.Node) @@ -403,14 +442,11 @@ func createContext(env *DefaultEnvironment) yaml.Node { path[i] = node(v) } ctx["STUBPATH"] = node(path) - if outer := env.Outer(); outer != nil { - list := []yaml.Node{} - for outer != nil { - list = append(list, node(outer.GetRootBinding())) - outer = outer.Outer() - } - ctx["OUTER"] = node(list) + list, bindings := getOuters(env) + if list != nil { + ctx["OUTER"] = list } + ctx["BINDINGS"] = bindings return node(ctx) } diff --git a/spiffing/spiffing_test.go b/spiffing/spiffing_test.go index d66ba96..9f1af76 100644 --- a/spiffing/spiffing_test.go +++ b/spiffing/spiffing_test.go @@ -37,4 +37,46 @@ var _ = Describe("Spiffing", func() { Expect(string(result)).To(Equal("- 25\n- 26\n")) }) }) + + Context("Bindings", func() { + ctx, err := New().WithValues(map[string]interface{}{ + "values": map[string]interface{}{ + "alice": 25, + "bob": 26, + }, + }) + Expect(err).To(Succeed()) + + It("Handles simple bindings", func() { + templ, err := ctx.Unmarshal("test", []byte(` +data: (( values.alice )) +`)) + Expect(err).To(Succeed()) + result, err := ctx.Cascade(templ, nil) + Expect(err).To(Succeed()) + data, err := ctx.Marshal(result) + Expect(err).To(Succeed()) + Expect(string(data)).To(Equal( + `data: 25 +`)) + }) + + It("Handles override bindings", func() { + templ, err := ctx.Unmarshal("test", []byte(` +values: other +data: (( ___.values.alice )) +orig: (( values )) +`)) + Expect(err).To(Succeed()) + result, err := ctx.Cascade(templ, nil) + Expect(err).To(Succeed()) + data, err := ctx.Marshal(result) + Expect(err).To(Succeed()) + Expect(string(data)).To(Equal( + `data: 25 +orig: other +values: other +`)) + }) + }) }) diff --git a/yaml/node.go b/yaml/node.go index d0ff45d..b2f09f0 100644 --- a/yaml/node.go +++ b/yaml/node.go @@ -10,6 +10,7 @@ import ( const SELF = "_" const DOCNODE = "__" +const ROOT = "___" const MERGEKEY = "<<<" type RefResolver interface {