handle environment variable for yaml file (#69)

* handle environment variable for yaml file

* panic

* add permissive mode

client/client_test.go

@@ -367,7 +367,7 @@ func TestDeleteResourceShouldWork(t *testing.T) {
 		responder,
 	)
 
-	resource, err := resource.FromYamlByte([]byte(`{"apiVersion":"v2","kind":"Topic","metadata":{"name":"toto","cluster":"local"},"spec":{}}`))
+	resource, err := resource.FromYamlByte([]byte(`{"apiVersion":"v2","kind":"Topic","metadata":{"name":"toto","cluster":"local"},"spec":{}}`), true)
 	if err != nil {
 		t.Error(err)
 	}

cmd/apply.go

@@ -11,28 +11,28 @@ import (
 
 var dryRun *bool
 
-func resourceForPath(path string) ([]resource.Resource, error) {
+func resourceForPath(path string, strict bool) ([]resource.Resource, error) {
 	directory, err := isDirectory(path)
 	if err != nil {
 		fmt.Fprintf(os.Stderr, "%s\n", err)
 		os.Exit(1)
 	}
 	if directory {
-		return resource.FromFolder(path)
+		return resource.FromFolder(path, strict)
 	} else {
-		return resource.FromFile(path)
+		return resource.FromFile(path, strict)
 	}
 }
 
-func initApply(kinds schema.KindCatalog) {
+func initApply(kinds schema.KindCatalog, strict bool) {
 	// applyCmd represents the apply command
 	var filePath *[]string
 	var applyCmd = &cobra.Command{
 		Use:   "apply",
 		Short: "Upsert a resource on Conduktor",
 		Long:  ``,
 		Run: func(cmd *cobra.Command, args []string) {
-			resources := loadResourceFromFileFlag(*filePath)
+			resources := loadResourceFromFileFlag(*filePath, strict)
 			schema.SortResourcesForApply(kinds, resources, *debug)
 			allSuccess := true
 			for _, resource := range resources {

cmd/delete.go

@@ -8,7 +8,7 @@ import (
 	"github.com/spf13/cobra"
 )
 
-func initDelete(kinds schema.KindCatalog) {
+func initDelete(kinds schema.KindCatalog, strict bool) {
 	var filePath *[]string
 	var deleteCmd = &cobra.Command{
 		Use:   "delete",
@@ -17,7 +17,7 @@ func initDelete(kinds schema.KindCatalog) {
 		Args:  cobra.NoArgs,
 		Run: func(cmd *cobra.Command, args []string) {
 			// Root command does nothing
-			resources := loadResourceFromFileFlag(*filePath)
+			resources := loadResourceFromFileFlag(*filePath, strict)
 			schema.SortResourcesForDelete(kinds, resources, *debug)
 			allSuccess := true
 			for _, resource := range resources {

cmd/load_resource_utils.go

@@ -7,10 +7,10 @@ import (
 	"github.com/conduktor/ctl/resource"
 )
 
-func loadResourceFromFileFlag(filePath []string) []resource.Resource {
+func loadResourceFromFileFlag(filePath []string, strict bool) []resource.Resource {
 	var resources = make([]resource.Resource, 0)
 	for _, path := range filePath {
-		r, err := resourceForPath(path)
+		r, err := resourceForPath(path, strict)
 		if err != nil {
 			fmt.Fprintf(os.Stderr, "%s\n", err)
 			os.Exit(1)

cmd/root.go

@@ -78,9 +78,10 @@ func init() {
 		kinds[k] = v
 	}
 	debug = rootCmd.PersistentFlags().BoolP("verbose", "v", false, "show more information for debugging")
+	var permissive = rootCmd.PersistentFlags().Bool("permissive", false, "permissive mode, allow undefined environment variables")
 	initGet(kinds)
-	initDelete(kinds)
-	initApply(kinds)
+	initDelete(kinds, !*permissive)
+	initApply(kinds, !*permissive)
 	initConsoleMkKind()
 	initGatewayMkKind()
 	initPrintKind(kinds)

resource/resource.go

@@ -7,6 +7,7 @@ import (
 	"io"
 	"os"
 	"path/filepath"
+	"regexp"
 	"strings"
 
 	gabs "github.com/Jeffail/gabs/v2"
@@ -65,24 +66,24 @@ type forParsingStruct struct {
 	Spec       map[string]interface{}
 }
 
-func FromFile(path string) ([]Resource, error) {
+func FromFile(path string, strict bool) ([]Resource, error) {
 	data, err := os.ReadFile(path)
 	if err != nil {
 		return nil, err
 	}
 
-	return FromYamlByte(data)
+	return FromYamlByte(data, strict)
 }
 
-func FromFolder(path string) ([]Resource, error) {
+func FromFolder(path string, strict bool) ([]Resource, error) {
 	dirEntry, err := os.ReadDir(path)
 	if err != nil {
 		return nil, err
 	}
 	var result = make([]Resource, 0)
 	for _, entry := range dirEntry {
 		if !entry.IsDir() && (strings.HasSuffix(entry.Name(), ".yml") || strings.HasSuffix(entry.Name(), ".yaml")) {
-			resources, err := FromFile(filepath.Join(path, entry.Name()))
+			resources, err := FromFile(filepath.Join(path, entry.Name()), strict)
 			result = append(result, resources...)
 			if err != nil {
 				return nil, err
@@ -93,7 +94,8 @@ func FromFolder(path string) ([]Resource, error) {
 	return result, nil
 }
 
-func FromYamlByte(data []byte) ([]Resource, error) {
+func FromYamlByte(data []byte, strict bool) ([]Resource, error) {
+	data = expandEnvVars(data, strict)
 	reader := bytes.NewReader(data)
 	var yamlData interface{}
 	results := make([]Resource, 0, 2)
@@ -118,6 +120,47 @@ func FromYamlByte(data []byte) ([]Resource, error) {
 	return results, nil
 }
 
+var envVarRegex = regexp.MustCompile(`\$\{([^}]+)\}`)
+
+// expandEnv replaces ${var} or $var in config according to the values of the current environment variables.
+// The replacement is case-sensitive. References to undefined variables are replaced by the empty string.
+// A default value can be given by using the form ${var:-default value}.
+func expandEnvVars(input []byte, strict bool) []byte {
+	missingEnvVars := make([]string, 0)
+	result := envVarRegex.ReplaceAllFunc(input, func(match []byte) []byte {
+		varName := string(match[2 : len(match)-1])
+		defaultValue := ""
+		if strings.Contains(varName, ":-") {
+			parts := strings.SplitN(varName, ":-", 2)
+			varName = parts[0]
+			defaultValue = parts[1]
+		}
+		value, isFound := os.LookupEnv(varName)
+
+		// use default value
+		if (!isFound || value == "") && defaultValue != "" {
+			return []byte(defaultValue)
+		}
+
+		if strict {
+			if (!isFound || value == "") && defaultValue == "" {
+				missingEnvVars = append(missingEnvVars, varName)
+				return []byte("")
+			}
+		} else {
+			if !isFound && defaultValue == "" {
+				missingEnvVars = append(missingEnvVars, varName)
+				return []byte("")
+			}
+		}
+		return []byte(value)
+	})
+	if len(missingEnvVars) > 0 {
+		panic(fmt.Sprintf("Missing environment variables: %s", strings.Join(missingEnvVars, ", ")))
+	}
+	return result
+}
+
 func extractKeyFromMetadataMap(m map[string]interface{}, key string) (string, error) {
 	if val, ok := m[key]; ok {
 		if str, ok := val.(string); ok {

resource/resource_test.go

@@ -76,7 +76,7 @@ metadata:
   name: cg1
 `)
 
-	results, err := FromYamlByte(yamlByte)
+	results, err := FromYamlByte(yamlByte, true)
 	spew.Dump(results)
 	if err != nil {
 		t.Error(err)
@@ -104,7 +104,7 @@ metadata:
 }
 
 func TestFromFolder(t *testing.T) {
-	resources, err := FromFolder("yamls")
+	resources, err := FromFolder("yamls", true)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -149,6 +149,53 @@ func TestFromFolder(t *testing.T) {
 	})
 }
 
+func TestResourceExpansionVariableEnv(t *testing.T) {
+	topicDesc, err := os.CreateTemp("/tmp", "topic.md")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer topicDesc.Close()
+	defer os.Remove(topicDesc.Name())
+	if _, err := topicDesc.Write([]byte(`This topic is awesome`)); err != nil {
+		log.Fatal(err)
+	}
+
+	yamlByte := []byte(`
+# comment
+---
+apiVersion: v1
+kind: Topic
+metadata:
+  cluster: ${CLUSTER_NAME}
+  name: ${TOPIC_NAME:-toto}
+  labels:
+    conduktor.io/descriptionFile: ` + topicDesc.Name() + `
+spec:
+  replicationFactor: 2
+  partition: 3
+`)
+	os.Setenv("CLUSTER_NAME", "cluster-a")
+
+	results, err := FromYamlByte(yamlByte, true)
+	spew.Dump(results)
+	if err != nil {
+		t.Error(err)
+	}
+
+	if len(results) != 1 {
+		t.Errorf("results expected of length 1, got length %d", len(results))
+	}
+
+	checkResourceWithoutJsonOrder(t, results[0], Resource{
+		Version:  "v1",
+		Kind:     "Topic",
+		Name:     "toto",
+		Metadata: map[string]interface{}{"cluster": "cluster-a", "name": "toto", "labels": map[string]interface{}{"conduktor.io/description": "This topic is awesome"}},
+		Spec:     map[string]interface{}{"replicationFactor": 2.0, "partition": 3.0},
+		Json:     []byte(`{"apiVersion":"v1","kind":"Topic","metadata":{"cluster":"cluster-a","name":"toto","labels":{"conduktor.io/description":"This topic is awesome"}},"spec":{"replicationFactor":2,"partition":3}}`),
+	})
+}
+
 func TestResourceExpansionForTopic(t *testing.T) {
 	topicDesc, err := os.CreateTemp("/tmp", "topic.md")
 	if err != nil {
@@ -175,7 +222,7 @@ spec:
   partition: 3
 `)
 
-	results, err := FromYamlByte(yamlByte)
+	results, err := FromYamlByte(yamlByte, true)
 	spew.Dump(results)
 	if err != nil {
 		t.Error(err)
@@ -266,7 +313,7 @@ spec:
   schemaFile: ` + jsonSchema.Name() + `
 `)
 
-	results, err := FromYamlByte(yamlByte)
+	results, err := FromYamlByte(yamlByte, true)
 	spew.Dump(results)
 	if err != nil {
 		t.Error(err)