From e5c40c0c45e4a534571f9321b964c6a7147e8bc8 Mon Sep 17 00:00:00 2001 From: Roy Kurian Varghese Date: Fri, 23 Mar 2018 20:10:46 +0530 Subject: [PATCH 1/5] Dynamic step targets from workflow inputs --- README.md | 25 +++++++ assignment.go | 13 ++++ assignment_test.go | 54 +++++++++++++++ service_template.go | 23 +++++++ .../tosca_web_application_with_wf_inputs.yaml | 66 +++++++++++++++++++ workflow.go | 2 +- 6 files changed, 182 insertions(+), 1 deletion(-) create mode 100644 tests/tosca_web_application_with_wf_inputs.yaml diff --git a/README.md b/README.md index 738f5d1..bc5c4da 100644 --- a/README.md +++ b/README.md @@ -26,6 +26,31 @@ This library is an implementation of the TOSCA definition as described in the do ## Normative Types The normative types definitions are included de facto. The files are embeded using go-bindata. +## Dynamic targets for workflow steps +When defining workflow steps, you can set the step target dynamically using the get_input property function. The library will first look in workflow inputs for the value and if it not present in the workflow inputs, will check the template inputs for the same. + +This is an additional feature on top of the spec. It does not break the standard spec functionality in any way. So you can set set target as a string as well. + +### Example +```yaml +workflows: + scaleup-app: + inputs: + increment: + type: integer + min_instances: + type: integer + max_instances: + type: integer + steps: + scale: + target: { get_input: target } + activities: + - set_state: scaling + - call_operation: Scale.scaleup + - set_state: scaled +``` + # Howto Create a `ServiceTemplateDefinition` and call `Parse(r io.Reader)` of `ParseCsar(c string)` to fill it with a YAML definition. diff --git a/assignment.go b/assignment.go index 7722d54..7f36f34 100644 --- a/assignment.go +++ b/assignment.go @@ -350,3 +350,16 @@ func (p *Assignment) Evaluate(std *ServiceTemplateDefinition, ctx string) interf return nil } + +// EvaluateForWorkflow gets the value of an Assignment, including the evaluation of expression or function +func (p *Assignment) EvaluateForWorkflow(std *ServiceTemplateDefinition, ctx string) interface{} { + // TODO(kenjones): Add support for the evaluation of ConstraintClause + if p.Function == GetInputFunc { + if len(p.Args) == 1 { + return std.GetWorkflowInputValue(p.Args[0].(string), ctx, false) + } + } + + // If its not a get_input scenario, let Evaluate handle it. + return p.Evaluate(std, ctx) +} diff --git a/assignment_test.go b/assignment_test.go index 89f6caf..f9e88c0 100644 --- a/assignment_test.go +++ b/assignment_test.go @@ -624,3 +624,57 @@ func TestEvaluateGetArtifact(t *testing.T) { } } + +func TestEvaluateWorkflowInputs(t *testing.T) { + fname := "./tests/tosca_web_application_with_wf_inputs.yaml" + var s ServiceTemplateDefinition + o, err := os.Open(fname) + if err != nil { + t.Fatal(err) + } + err = s.Parse(o) + if err != nil { + t.Log("Error in processing", fname) + t.Fatal(err) + } + + // Set and verify the input value before peforming eval + // Get the value back in raw format PropertyAssignment as the + // value itself does not require evaluation. + s.SetWorkFlowInputValue("target", "scaleup-app", "web_app") + in := s.GetWorkflowInputValue("target", "scaleup-app", true) + if inv, ok := in.(PropertyAssignment); ok { + if inv.Value != "web_app" { + t.Log("(actual) failed to properly set the input value", inv) + t.Fail() + } + } else { + t.Log("(raw) failed to properly set the input value", in) + t.Fail() + } + + wf, ok := s.TopologyTemplate.Workflows["scaleup-app"] + + if !ok { + t.Log(fname, "missing workflow definition `scaleup-app`") + t.Fail() + } + + step, ok := wf.Steps["scale"] + if !ok { + t.Log(fname, "missing step definition `scale`") + t.Fail() + } + + v := step.Target.EvaluateForWorkflow(&s, "scaleup-app") + if vstr, ok := v.(string); ok { + if vstr != "web_app" { + t.Log(fname, "input evaluation failed to get value for `target`", vstr) + t.Fail() + } + } else { + t.Log("input value returned not the correct type", v) + t.Fail() + } + +} diff --git a/service_template.go b/service_template.go index 05f4570..57d887b 100644 --- a/service_template.go +++ b/service_template.go @@ -146,6 +146,29 @@ func (s *ServiceTemplateDefinition) GetInputValue(prop string, raw bool) interfa return input.Evaluate(s, "") } +// GetWorkflowInputValue retrieves a worflow input value from Service Template Definition in +// the raw form (function evaluation not performed), or actual value after all +// function evaluation has completed. +func (s *ServiceTemplateDefinition) GetWorkflowInputValue(prop, wfname string, raw bool) interface{} { + if _, ok := s.TopologyTemplate.Workflows[wfname].Inputs[prop]; !ok { + // No workflow inputs, lets use Evaluate to find out if the its present in template inputs + return s.GetInputValue(prop, raw) + } + + // Its there in the workflow, lets fetch it. + if raw { + return s.TopologyTemplate.Workflows[wfname].Inputs[prop].Value + } + input := s.TopologyTemplate.Workflows[wfname].Inputs[prop].Value + return input.Evaluate(s, "") +} + +// SetWorkFlowInputValue sets an input value on a Service Template Definition +func (s *ServiceTemplateDefinition) SetWorkFlowInputValue(prop, wfname string, value interface{}) { + v := newPAValue(value) + s.TopologyTemplate.Workflows[wfname].Inputs[prop] = PropertyDefinition{Value: *v} +} + // SetInputValue sets an input value on a Service Template Definition func (s *ServiceTemplateDefinition) SetInputValue(prop string, value interface{}) { v := newPAValue(value) diff --git a/tests/tosca_web_application_with_wf_inputs.yaml b/tests/tosca_web_application_with_wf_inputs.yaml new file mode 100644 index 0000000..067220d --- /dev/null +++ b/tests/tosca_web_application_with_wf_inputs.yaml @@ -0,0 +1,66 @@ +tosca_definitions_version: tosca_simple_yaml_1_0 + +description: > + TOSCA simple profile with a web application. + +topology_template: + inputs: + cpus: + type: integer + description: Number of CPUs for the server. + constraints: + - valid_values: [ 1, 2, 4, 8 ] + default: 1 + context_root: app + + node_templates: + web_app: + type: tosca.nodes.WebApplication + properties: + context_root: { get_input: context_root } + requirements: + - host: web_server + interfaces: + Standard: + create: + implementation: web_app_install.sh + inputs: + context_root: { get_input: context_root } + start: web_app_start.sh + + web_server: + type: tosca.nodes.WebServer + requirements: + - host: server + interfaces: + Standard: + create: web_server_install.sh + start: web_server_start.sh + + server: + type: tosca.nodes.Compute + capabilities: + host: + properties: + disk_size: 10 GB + num_cpus: { get_input: cpus } + mem_size: 1024 MB + os: + properties: + architecture: x86_64 + type: Linux + distribution: Ubuntu + version: 14.04 + + workflows: + scaleup-app: + inputs: + target: + type: string + steps: + scale: + target: { get_input: target } + activities: + - set_state: scaling + - call_operation: Scale.scaleup + - set_state: scaled diff --git a/workflow.go b/workflow.go index 849d0c3..870abff 100644 --- a/workflow.go +++ b/workflow.go @@ -10,7 +10,7 @@ type WorkflowDefinition struct { // StepDefinition structure to handle workflow steps type StepDefinition struct { - Target string `yaml:"target,omitempty" json:"target,omitempty"` + Target PropertyAssignment `yaml:"target,omitempty" json:"target,omitempty"` OnSuccess []string `yaml:"on_success,omitempty" json:"on_success,omitempty"` Activities []ActivityDefinition `yaml:"activities,omitempty" json:"activities,omitempty"` Filter Filter `yaml:"filter,omitempty" json:"filter,omitempty"` From 3b78e8c0f6533490bb323ea41aae49c5ac6fa6bd Mon Sep 17 00:00:00 2001 From: Roy Kurian Varghese Date: Tue, 17 Apr 2018 11:33:55 +0530 Subject: [PATCH 2/5] Updating the readme --- README.md | 138 +++++++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 136 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index bc5c4da..e98d2f0 100644 --- a/README.md +++ b/README.md @@ -27,9 +27,143 @@ This library is an implementation of the TOSCA definition as described in the do The normative types definitions are included de facto. The files are embeded using go-bindata. ## Dynamic targets for workflow steps -When defining workflow steps, you can set the step target dynamically using the get_input property function. The library will first look in workflow inputs for the value and if it not present in the workflow inputs, will check the template inputs for the same. +### Huh? +When defining workflow steps, you can set the step target dynamically using the get_input property function. The library will first look in workflow inputs for the value and if it is not present in the workflow inputs, will check the template inputs for the same. -This is an additional feature on top of the spec. It does not break the standard spec functionality in any way. So you can set set target as a string as well. +Please note that this functionationlity is not as per the specification. This is an additional feature on top of the spec. However, this feature is fully backward compatible with the specification, meaning taht you can go ahead and use a plain string as the step target as before. + +### But why? +While working with the TOSCA spec for our applications, we realized that since the workflow step target is a static value, we had to define multiple imperative workflows in order to essentially do the same steps for different nodes. Basically, workflows were not reusable across nodes. With small applications with 2 or 3 nodes this wouldn't be much of an issue, but with large applications with many more nodes, this would lead to extremely verbose profiles with a lot of redundancy. + +Lets take an example to make things clearer. + +Assume we define our application as per the below node templates: +```yaml + node_templates: + ui-ctnr: + type: cisco.nodes.ScalableContainer + properties: + username: { get_input: uname } + namespace: { get_input: name_space } + password: { get_input: pword } + openshift_config: { get_input: oshift_url} + image_name: containers.cisco.com/royvargh/cabdemoui + image_tag: latest + avail_zone: { get_input: avail_zone } + run_as_root: true + ports: + - + name: web + port: 80 + protocol: TCP + environment: + CISCO_LC: dev + requirements: + - endpoint: + node: cabdemoapi + capability: tosca.capabilities.Endpoint + relationship: tosca.relationships.ConnectsTo + - endpoint: + node: cabdemoapi2 + capability: tosca.capabilities.Endpoint + relationship: tosca.relationships.ConnectsTo + + api-ctnr: + type: cisco.nodes.ScalableContainer + properties: + replicas: 1 + username: { get_input: uname } + namespace: { get_input: name_space } + password: { get_input: pword } + openshift_config: { get_input: oshift_url} + image_name: containers.cisco.com/coenglan/cabdemoapi + image_tag: latest + avail_zone: { get_input: avail_zone } + ports: + - + name: web + port: 8080 + protocol: TCP + environment: + CISCO_LC: dev +``` +This would result in 2 nodes being deployed for our application. Now we expect that the ui-ctnr will see a lot of load, so we define a scale-up workflow so that we can bump up the number of replicas if we need to. +```yaml +workflows: + scaleup-app-ui: + inputs: + increment: + type: integer + min_instances: + type: integer + max_instances: + type: integer + steps: + scale: + target: ui-ctnr + activities: + - set_state: scaling + - call_operation: Scale.scaleup + - set_state: scaled +``` +Now we also know that the api-ctnr might also get subjected to a high load during peak hours, so we would want to have a scale-up workflow for the api-ctnr as well. But as per the current TOSCA spec, we will need to define a seperate workflow for api-ctnr. +```yaml +workflows: + scaleup-app-ui: + inputs: + increment: + type: integer + min_instances: + type: integer + max_instances: + type: integer + steps: + scale: + target: ui-ctnr + activities: + - set_state: scaling + - call_operation: Scale.scaleup + - set_state: scaled + scaleup-app-api: + inputs: + increment: + type: integer + min_instances: + type: integer + max_instances: + type: integer + steps: + scale: + target: api-ctnr + activities: + - set_state: scaling + - call_operation: Scale.scaleup + - set_state: scaled +``` +Even though, both workflows are essentially the same, we are still forced to define one workflow of each target. In a profile for a complex application with perhaps 10 or 20 nodes, this would result in a large number of workflow definitions. + +By changing the target type from string to a PropertyAssignment, we can use a static string value as the target, same as before, or if need we can assign the target value from a workflow/topology input via the get_input function. This provides flexibility and can simplify the above workflow definitions to: +```yaml +workflows: + scaleup-app: + inputs: + increment: + type: integer + min_instances: + type: integer + max_instances: + type: integer + target: + type: string + steps: + scale: + target: { get_input: target } + activities: + - set_state: scaling + - call_operation: Scale.scaleup + - set_state: scaled +``` +Now any node that supports the Scale.scaleup call_operation can be scaled up using the scaleup-app workflow. All that would need to be done is to pass in the target you want to scale up as an input to the workflow. ### Example ```yaml From ee8c6f322f1dde8afd9996477f3733ead5b22fd2 Mon Sep 17 00:00:00 2001 From: Roy Kurian Varghese Date: Tue, 17 Apr 2018 12:20:51 +0530 Subject: [PATCH 3/5] Spelling fixes and minor tweaks --- README.md | 27 +++------------------------ 1 file changed, 3 insertions(+), 24 deletions(-) diff --git a/README.md b/README.md index e98d2f0..1137c64 100644 --- a/README.md +++ b/README.md @@ -27,10 +27,9 @@ This library is an implementation of the TOSCA definition as described in the do The normative types definitions are included de facto. The files are embeded using go-bindata. ## Dynamic targets for workflow steps -### Huh? -When defining workflow steps, you can set the step target dynamically using the get_input property function. The library will first look in workflow inputs for the value and if it is not present in the workflow inputs, will check the template inputs for the same. +We made a few tweaks to this library that we thought would be useful. When defining workflow steps, you can now set the step target dynamically using the get_input property function. The library will first look in workflow inputs for the value and if it is not present in the workflow inputs, will check the template inputs for the same. -Please note that this functionationlity is not as per the specification. This is an additional feature on top of the spec. However, this feature is fully backward compatible with the specification, meaning taht you can go ahead and use a plain string as the step target as before. +Please note that this functionationlity is not as per the specification. This is an additional feature on top of the spec. However, this feature is fully backward compatible with the specification, meaning that you can go ahead and use a plain string as the step target, same as before. ### But why? While working with the TOSCA spec for our applications, we realized that since the workflow step target is a static value, we had to define multiple imperative workflows in order to essentially do the same steps for different nodes. Basically, workflows were not reusable across nodes. With small applications with 2 or 3 nodes this wouldn't be much of an issue, but with large applications with many more nodes, this would lead to extremely verbose profiles with a lot of redundancy. @@ -142,7 +141,7 @@ workflows: ``` Even though, both workflows are essentially the same, we are still forced to define one workflow of each target. In a profile for a complex application with perhaps 10 or 20 nodes, this would result in a large number of workflow definitions. -By changing the target type from string to a PropertyAssignment, we can use a static string value as the target, same as before, or if need we can assign the target value from a workflow/topology input via the get_input function. This provides flexibility and can simplify the above workflow definitions to: +By changing the target type from string to a PropertyAssignment, we can use a static string value as the target, same as before, or if needed we can assign the target value from a workflow/topology input via the get_input function. This provides flexibility and can simplify the above workflow definitions to: ```yaml workflows: scaleup-app: @@ -165,26 +164,6 @@ workflows: ``` Now any node that supports the Scale.scaleup call_operation can be scaled up using the scaleup-app workflow. All that would need to be done is to pass in the target you want to scale up as an input to the workflow. -### Example -```yaml -workflows: - scaleup-app: - inputs: - increment: - type: integer - min_instances: - type: integer - max_instances: - type: integer - steps: - scale: - target: { get_input: target } - activities: - - set_state: scaling - - call_operation: Scale.scaleup - - set_state: scaled -``` - # Howto Create a `ServiceTemplateDefinition` and call `Parse(r io.Reader)` of `ParseCsar(c string)` to fill it with a YAML definition. From ec8124ddc6e01a47d3a2e1e33ba012fd85925062 Mon Sep 17 00:00:00 2001 From: Roy Kurian Varghese Date: Tue, 17 Apr 2018 12:29:50 +0530 Subject: [PATCH 4/5] Additional section --- README.md | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 1137c64..ca2da9d 100644 --- a/README.md +++ b/README.md @@ -29,7 +29,7 @@ The normative types definitions are included de facto. The files are embeded usi ## Dynamic targets for workflow steps We made a few tweaks to this library that we thought would be useful. When defining workflow steps, you can now set the step target dynamically using the get_input property function. The library will first look in workflow inputs for the value and if it is not present in the workflow inputs, will check the template inputs for the same. -Please note that this functionationlity is not as per the specification. This is an additional feature on top of the spec. However, this feature is fully backward compatible with the specification, meaning that you can go ahead and use a plain string as the step target, same as before. +Please note that this functionationlity is not as per the specification. This is an additional feature on top of the spec. However, this feature is fully backward compatible with the specification, meaning that you can go ahead and use a plain string as the step target, as per the TOSCA specification. ### But why? While working with the TOSCA spec for our applications, we realized that since the workflow step target is a static value, we had to define multiple imperative workflows in order to essentially do the same steps for different nodes. Basically, workflows were not reusable across nodes. With small applications with 2 or 3 nodes this wouldn't be much of an issue, but with large applications with many more nodes, this would lead to extremely verbose profiles with a lot of redundancy. @@ -164,6 +164,10 @@ workflows: ``` Now any node that supports the Scale.scaleup call_operation can be scaled up using the scaleup-app workflow. All that would need to be done is to pass in the target you want to scale up as an input to the workflow. +##Do we want to make this part of the spec? + +We do believe this feature will improve reusability with regard to TOSCA workflows. We intend to get in touch with the folks who defined the TOSCA specification to both understand the reasoning behind their current specfications for step definition targets, as well as to discuss whether the above feature should/could be included in the specficiations. + # Howto Create a `ServiceTemplateDefinition` and call `Parse(r io.Reader)` of `ParseCsar(c string)` to fill it with a YAML definition. From 4122f6402720cb122a0f972f74badfeebe8151ab Mon Sep 17 00:00:00 2001 From: Roy Kurian Varghese Date: Tue, 17 Apr 2018 12:36:32 +0530 Subject: [PATCH 5/5] Cleaning up examples --- README.md | 21 +++++++-------------- 1 file changed, 7 insertions(+), 14 deletions(-) diff --git a/README.md b/README.md index ca2da9d..254eb61 100644 --- a/README.md +++ b/README.md @@ -40,51 +40,44 @@ Assume we define our application as per the below node templates: ```yaml node_templates: ui-ctnr: - type: cisco.nodes.ScalableContainer + type: tosca.nodes.ScalableContainer properties: username: { get_input: uname } namespace: { get_input: name_space } password: { get_input: pword } - openshift_config: { get_input: oshift_url} - image_name: containers.cisco.com/royvargh/cabdemoui + image_name: dummy/path/to/image image_tag: latest - avail_zone: { get_input: avail_zone } run_as_root: true ports: - name: web port: 80 protocol: TCP - environment: - CISCO_LC: dev requirements: - endpoint: - node: cabdemoapi + node: api-svc-1 capability: tosca.capabilities.Endpoint relationship: tosca.relationships.ConnectsTo - endpoint: - node: cabdemoapi2 + node: api-svc-2 capability: tosca.capabilities.Endpoint relationship: tosca.relationships.ConnectsTo api-ctnr: - type: cisco.nodes.ScalableContainer + type: tosca.nodes.ScalableContainer properties: replicas: 1 username: { get_input: uname } namespace: { get_input: name_space } password: { get_input: pword } - openshift_config: { get_input: oshift_url} - image_name: containers.cisco.com/coenglan/cabdemoapi + image_name: dummy/path/to/image image_tag: latest - avail_zone: { get_input: avail_zone } ports: - name: web port: 8080 protocol: TCP - environment: - CISCO_LC: dev + ``` This would result in 2 nodes being deployed for our application. Now we expect that the ui-ctnr will see a lot of load, so we define a scale-up workflow so that we can bump up the number of replicas if we need to. ```yaml