Merge pull request #129 from kanisterio/sync

Add initial support for per-phase templating

pkg/apis/cr/v1alpha1/types.go

@@ -210,9 +210,10 @@ type BlueprintAction struct {
 
 // BlueprintPhase is a an individual unit of execution.
 type BlueprintPhase struct {
-	Func string                 `json:"func"`
-	Name string                 `json:"name"`
-	Args map[string]interface{} `json:"args"`
+	Func       string                     `json:"func"`
+	Name       string                     `json:"name"`
+	ObjectRefs map[string]ObjectReference `json:"objects"`
+	Args       map[string]interface{}     `json:"args"`
 }
 
 // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object

pkg/controller/controller.go

@@ -360,7 +360,10 @@ func (c *Controller) runAction(ctx context.Context, as *crv1alpha1.ActionSet, aI
 	go func() {
 		for i, p := range phases {
 			c.logAndSuccessEvent(fmt.Sprintf("Executing phase %s", p.Name()), "Started Phase", as)
-			err = p.Exec(ctx, *tp)
+			err = param.InitPhaseParams(ctx, c.clientset, tp, p.Name(), p.Objects())
+			if err == nil {
+				err = p.Exec(ctx, *tp)
+			}
 			var rf func(*crv1alpha1.ActionSet) error
 			if err != nil {
 				rf = func(ras *crv1alpha1.ActionSet) error {
@@ -387,6 +390,7 @@ func (c *Controller) runAction(ctx context.Context, as *crv1alpha1.ActionSet, aI
 				c.logAndErrorEvent(msg, reason, err, as, bp)
 				return
 			}
+			param.UpdatePhaseParams(ctx, tp, p.Name(), nil)
 			c.logAndSuccessEvent(fmt.Sprintf("Completed phase %s", p.Name()), "Ended Phase", as)
 		}
 	}()

pkg/param/param.go

@@ -31,7 +31,8 @@ type TemplateParams struct {
 	Time         string
 	Profile      *Profile
 	Options      map[string]string
-	Unstructured map[string]interface{}
+	Object       map[string]interface{}
+	Phases       map[string]*Phase
 }
 
 // StatefulSetParams are params for stateful sets.
@@ -89,11 +90,18 @@ type KeyPair struct {
 	Secret string
 }
 
+// Phase represents a Blueprint phase and contains the phase output
+type Phase struct {
+	Secrets map[string]v1.Secret
+	Output  map[string]interface{}
+}
+
 const (
 	DeploymentKind  = "deployment"
 	StatefulSetKind = "statefulset"
 	PVCKind         = "pvc"
 	NamespaceKind   = "namespace"
+	SecretKind      = "secret"
 )
 
 // New function fetches and returns the desired params
@@ -150,7 +158,8 @@ func New(ctx context.Context, cli kubernetes.Interface, crCli versioned.Interfac
 		if err != nil {
 			return nil, errors.Wrapf(err, "could not fetch object name: %s, namespace: %s, group: %s, version: %s, resource: %s", as.Object.Name, as.Object.Namespace, gvr.Group, gvr.Version, gvr.Resource)
 		}
-		tp.Unstructured = u.UnstructuredContent()
+		// TODO: We should set `Object` for all other kinds as well.
+		tp.Object = u.UnstructuredContent()
 	}
 	return &tp, nil
 }
@@ -209,6 +218,9 @@ func fetchKeyPairCredential(ctx context.Context, cli kubernetes.Interface, c *cr
 func fetchSecrets(ctx context.Context, cli kubernetes.Interface, refs map[string]crv1alpha1.ObjectReference) (map[string]v1.Secret, error) {
 	secrets := make(map[string]v1.Secret, len(refs))
 	for name, ref := range refs {
+		if strings.ToLower(ref.Kind) != SecretKind {
+			continue
+		}
 		s, err := cli.CoreV1().Secrets(ref.Namespace).Get(ref.Name, metav1.GetOptions{})
 		if err != nil {
 			return nil, errors.WithStack(err)
@@ -321,3 +333,23 @@ func fetchPVCParams(ctx context.Context, cli kubernetes.Interface, namespace, na
 		Namespace: namespace,
 	}, nil
 }
+
+// UpdatePhaseParams updates the TemplateParams with Phase information
+func UpdatePhaseParams(ctx context.Context, tp *TemplateParams, phaseName string, output map[string]interface{}) {
+	tp.Phases[phaseName].Output = output
+}
+
+// InitPhaseParams initializes the TemplateParams with Phase information
+func InitPhaseParams(ctx context.Context, cli kubernetes.Interface, tp *TemplateParams, phaseName string, objects map[string]crv1alpha1.ObjectReference) error {
+	if tp.Phases == nil {
+		tp.Phases = make(map[string]*Phase)
+	}
+	secrets, err := fetchSecrets(ctx, cli, objects)
+	if err != nil {
+		return err
+	}
+	tp.Phases[phaseName] = &Phase{
+		Secrets: secrets,
+	}
+	return nil
+}

pkg/param/param_test.go

@@ -1,12 +1,15 @@
 package param
 
 import (
+	"bytes"
 	"context"
 	"fmt"
 	"strings"
 	"testing"
+	"text/template"
 	"time"
 
+	"github.com/Masterminds/sprig"
 	. "gopkg.in/check.v1"
 	appsv1 "k8s.io/api/apps/v1"
 	"k8s.io/api/core/v1"
@@ -323,7 +326,7 @@ func (s *ParamsSuite) testNewTemplateParams(ctx context.Context, c *C, object cr
 	case NamespaceKind:
 		template = "{{ .Namespace.Name }}"
 	default:
-		template = "{{ .Unstructured.metadata.name }}"
+		template = "{{ .Object.metadata.name }}"
 	}
 
 	artsTpl := map[string]crv1alpha1.Artifact{
@@ -487,3 +490,116 @@ func (s *ParamsSuite) TestProfile(c *C) {
 		},
 	})
 }
+
+func (s *ParamsSuite) TestPhaseParams(c *C) {
+	ctx := context.Background()
+	secret := &v1.Secret{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      "secret-name",
+			Namespace: s.namespace,
+			Labels:    map[string]string{"app": "fake-app"},
+		},
+		Data: map[string][]byte{
+			"key":   []byte("myKey"),
+			"value": []byte("myValue"),
+		},
+	}
+	prof := &crv1alpha1.Profile{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      "profName",
+			Namespace: s.namespace,
+		},
+		Credential: crv1alpha1.Credential{
+			Type: crv1alpha1.CredentialTypeKeyPair,
+			KeyPair: &crv1alpha1.KeyPair{
+				IDField:     "key",
+				SecretField: "value",
+				Secret: crv1alpha1.ObjectReference{
+					Name:      "secret-name",
+					Namespace: s.namespace,
+				},
+			},
+		},
+	}
+	_, err := s.cli.CoreV1().Secrets(s.namespace).Create(secret)
+	c.Assert(err, IsNil)
+	defer s.cli.CoreV1().Secrets(s.namespace).Delete("secret-name", &metav1.DeleteOptions{})
+
+	_, err = s.cli.CoreV1().Secrets(s.namespace).Get("secret-name", metav1.GetOptions{})
+	c.Assert(err, IsNil)
+
+	crCli := crfake.NewSimpleClientset()
+	_, err = crCli.CrV1alpha1().Profiles(s.namespace).Create(prof)
+	c.Assert(err, IsNil)
+	_, err = crCli.CrV1alpha1().Profiles(s.namespace).Get("profName", metav1.GetOptions{})
+	c.Assert(err, IsNil)
+	as := crv1alpha1.ActionSpec{
+		Object: crv1alpha1.ObjectReference{
+			Name:      s.pvc,
+			Namespace: s.namespace,
+			Kind:      PVCKind,
+		},
+		Profile: &crv1alpha1.ObjectReference{
+			Name:      "profName",
+			Namespace: s.namespace,
+		},
+	}
+	tp, err := New(ctx, s.cli, crCli, as)
+	c.Assert(err, IsNil)
+	c.Assert(tp.Phases, IsNil)
+	err = InitPhaseParams(ctx, s.cli, tp, "backup", nil)
+	c.Assert(err, IsNil)
+	UpdatePhaseParams(ctx, tp, "backup", map[string]interface{}{"version": "0.11.0"})
+	c.Assert(tp.Phases, HasLen, 1)
+	c.Assert(tp.Phases["backup"], NotNil)
+}
+
+func (s *ParamsSuite) TestRenderingPhaseParams(c *C) {
+	ctx := context.Background()
+	secret := &v1.Secret{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      "secret-dfss",
+			Namespace: "ns1",
+		},
+		StringData: map[string]string{
+			"myKey":   "foo",
+			"myValue": "bar",
+		},
+	}
+	cli := fake.NewSimpleClientset(secret)
+	secretRef := map[string]crv1alpha1.ObjectReference{
+		"authSecret": crv1alpha1.ObjectReference{
+			Kind:      SecretKind,
+			Name:      secret.Name,
+			Namespace: secret.Namespace,
+		},
+	}
+	tp := TemplateParams{}
+	err := InitPhaseParams(ctx, cli, &tp, "backup", secretRef)
+	c.Assert(err, IsNil)
+	UpdatePhaseParams(ctx, &tp, "backup", map[string]interface{}{"replicas": 2})
+	for _, tc := range []struct {
+		arg      string
+		expected string
+	}{
+		{
+			"{{ .Phases.backup.Output.replicas }}",
+			"2",
+		},
+		{
+			"{{ .Phases.backup.Secrets.authSecret.Namespace }}",
+			"ns1",
+		},
+		{
+			"{{ .Phases.backup.Secrets.authSecret.StringData.myValue }}",
+			"bar",
+		},
+	} {
+		t, err := template.New("config").Option("missingkey=error").Funcs(sprig.TxtFuncMap()).Parse(tc.arg)
+		c.Assert(err, IsNil)
+		buf := bytes.NewBuffer(nil)
+		err = t.Execute(buf, tp)
+		c.Assert(err, IsNil)
+		c.Assert(buf.String(), Equals, tc.expected)
+	}
+}

pkg/param/render.go

@@ -54,6 +54,18 @@ func render(arg interface{}, tp TemplateParams) (interface{}, error) {
 			ras[rk] = rv
 		}
 		return ras, nil
+	case reflect.Struct:
+		ras := reflect.New(val.Type())
+		for i := 0; i < val.NumField(); i++ {
+			r, err := render(val.Field(i).Interface(), tp)
+			if err != nil {
+				return nil, err
+			}
+			// set the field to the rendered value
+			rv := reflect.Indirect(reflect.ValueOf(r))
+			ras.Elem().Field(i).Set(rv)
+		}
+		return ras.Elem().Interface(), nil
 	default:
 		return arg, nil
 	}
@@ -90,3 +102,16 @@ func renderStringArg(arg string, tp TemplateParams) (string, error) {
 	}
 	return buf.String(), nil
 }
+
+// RenderObjectRefs function renders object refs from TemplateParams
+func RenderObjectRefs(in map[string]crv1alpha1.ObjectReference, tp TemplateParams) (map[string]crv1alpha1.ObjectReference, error) {
+	out := make(map[string]crv1alpha1.ObjectReference, len(in))
+	for k, v := range in {
+		rv, err := render(v, tp)
+		if err != nil {
+			return nil, errors.Wrapf(err, "could not render object reference {%s}", k)
+		}
+		out[k] = rv.(crv1alpha1.ObjectReference)
+	}
+	return out, nil
+}

pkg/param/render_test.go

@@ -135,3 +135,20 @@ func (s *RenderSuite) TestRender(c *C) {
 		}
 	}
 }
+
+func (s *RenderSuite) TestRenderObjects(c *C) {
+	tp := TemplateParams{
+		Object: map[string]interface{}{
+			"spec": map[string]string{"authSecret": "secret-name"},
+		},
+	}
+	in := map[string]crv1alpha1.ObjectReference{
+		"authSecret": crv1alpha1.ObjectReference{
+			Kind: SecretKind,
+			Name: "{{ .Object.spec.authSecret }}",
+		},
+	}
+	out, err := RenderObjectRefs(in, tp)
+	c.Assert(err, IsNil)
+	c.Assert(out["authSecret"].Name, Equals, "secret-name")
+}

pkg/phase.go

@@ -11,16 +11,22 @@ import (
 
 // Phase is an atomic unit of execution.
 type Phase struct {
-	name string
-	args map[string]interface{}
-	f    Func
+	name    string
+	args    map[string]interface{}
+	objects map[string]crv1alpha1.ObjectReference
+	f       Func
 }
 
 // Name returns the name of this phase.
 func (p *Phase) Name() string {
 	return p.name
 }
 
+// Objects returns the phase object references
+func (p *Phase) Objects() map[string]crv1alpha1.ObjectReference {
+	return p.objects
+}
+
 // Exec renders the argument templates in this Phase's Func and executes with
 // those arguments.
 func (p *Phase) Exec(ctx context.Context, tp param.TemplateParams) error {
@@ -43,6 +49,10 @@ func GetPhases(bp crv1alpha1.Blueprint, action string, tp param.TemplateParams)
 	}
 	phases := make([]*Phase, 0, len(a.Phases))
 	for _, p := range a.Phases {
+		objs, err := param.RenderObjectRefs(p.ObjectRefs, tp)
+		if err != nil {
+			return nil, err
+		}
 		args, err := param.RenderArgs(p.Args, tp)
 		if err != nil {
 			return nil, err
@@ -51,9 +61,10 @@ func GetPhases(bp crv1alpha1.Blueprint, action string, tp param.TemplateParams)
 			return nil, errors.Wrapf(err, "Reqired args missing for function %s", funcs[p.Func].Name())
 		}
 		phases = append(phases, &Phase{
-			name: p.Name,
-			args: args,
-			f:    funcs[p.Func],
+			name:    p.Name,
+			args:    args,
+			objects: objs,
+			f:       funcs[p.Func],
 		})
 	}
 	return phases, nil

pkg/validate/validate.go

@@ -52,7 +52,7 @@ func actionSpec(s crv1alpha1.ActionSpec) error {
 		// Known types
 	default:
 		// Not a known type. ActionSet must specify API group and resource
-		// name in order to populate `Unstructured` TemplateParam
+		// name in order to populate `Object` TemplateParam
 		if s.Object.APIVersion == "" || s.Object.Resource == "" {
 			return errorf("Not a known object Kind %s. Action %s must specify Resource name and API version", s.Object.Kind, s.Name)
 		}