Hide sensitive configuration from log (#34)

beatlabs · Oct 2, 2019 · 4fedcb6 · 4fedcb6
1 parent 1e78afc
commit 4fedcb6
Show file tree

Hide file tree

Showing 8 changed files with 234 additions and 14 deletions.
diff --git a/README.md b/README.md
@@ -16,10 +16,11 @@ The order is applied as it is listed above. Consul seeder and monitor are option
 
 ```go
 type Config struct {
-    IndexName      sync.String  `seed:"customers-v1"`
-    CacheRetention sync.Int64   `seed:"86400" env:"ENV_CACHE_RETENTION_SECONDS"`
-    LogLevel       sync.String  `seed:"DEBUG" flag:"loglevel"`
-    Sandbox        sync.Bool    `seed:"true" env:"ENV_SANDBOX" consul:"/config/sandbox-mode"`
+    IndexName      sync.String          `seed:"customers-v1"`
+    CacheRetention sync.Int64           `seed:"86400" env:"ENV_CACHE_RETENTION_SECONDS"`
+    LogLevel       sync.String          `seed:"DEBUG" flag:"loglevel"`
+    Sandbox        sync.Bool            `seed:"true" env:"ENV_SANDBOX" consul:"/config/sandbox-mode"`
+    AccessToken    sync.Secret          `seed:"defaultaccesstoken" env:"ENV_ACCESS_TOKEN" consul:"/config/access-token"`
 }
 ```
 
@@ -36,6 +37,9 @@ The fields have to be one of the types that the sync package supports in order t
 - sync.Int64, allows for concurrent int64 manipulation
 - sync.Float64, allows for concurrent float64 manipulation
 - sync.Bool, allows for concurrent bool manipulation
+- sync.Secret, allows for concurrent secret manipulation. Secrets can only be strings
+
+For sensitive configuration (passwords, tokens, etc.) that shouldn't be printed in log, you can use the `Secret` flavor of `sync` types. If one of these is selected, then at harvester log instead of the real value the text `***` will be displayed.
 
 `Harvester` has a seeding phase and an optional monitoring phase.
 

diff --git a/config/config.go b/config/config.go
@@ -29,6 +29,7 @@ type Field struct {
 	tp      string
 	version uint64
 	setter  reflect.Value
+	printer reflect.Value
 	sources map[Source]string
 }
 
@@ -42,6 +43,7 @@ func NewField(fld *reflect.StructField, val *reflect.Value) (*Field, error) {
 		tp:      fld.Type.Name(),
 		version: 0,
 		setter:  val.FieldByName(fld.Name).Addr().MethodByName("Set"),
+		printer: val.FieldByName(fld.Name).Addr().MethodByName("String"),
 		sources: make(map[Source]string),
 	}
 	value, ok := fld.Tag.Lookup(string(SourceSeed))
@@ -78,6 +80,15 @@ func (f *Field) Sources() map[Source]string {
 	return f.sources
 }
 
+// String returns string representation of field's value.
+func (f *Field) String() string {
+	vv := f.printer.Call([]reflect.Value{})
+	if len(vv) > 0 {
+		return vv[0].String()
+	}
+	return ""
+}
+
 // Set the value of the field.
 func (f *Field) Set(value string, version uint64) error {
 	if version != 0 && version <= f.version {
@@ -92,7 +103,7 @@ func (f *Field) Set(value string, version uint64) error {
 			return err
 		}
 		arg = v
-	case "String":
+	case "String", "Secret":
 		arg = value
 	case "Int64":
 		v, err := strconv.ParseInt(value, 10, 64)
@@ -112,7 +123,7 @@ func (f *Field) Set(value string, version uint64) error {
 		return fmt.Errorf("the set call returned %d values: %v", len(rr), rr)
 	}
 	f.version = version
-	log.Infof("field %s updated with value %s, version: %d", f.name, value, version)
+	log.Infof("field %s updated with value %v, version: %d", f.name, f, version)
 	return nil
 }
 
@@ -150,7 +161,7 @@ func getFields(tp reflect.Type, val *reflect.Value) ([]*Field, error) {
 		value, ok := fld.Sources()[SourceConsul]
 		if ok {
 			if isKeyValueDuplicate(dup, SourceConsul, value) {
-				return nil, fmt.Errorf("duplicate value %s for source %s", value, SourceConsul)
+				return nil, fmt.Errorf("duplicate value %v for source %s", fld, SourceConsul)
 			}
 		}
 		ff = append(ff, fld)
@@ -166,7 +177,7 @@ func isTypeSupported(t reflect.Type) bool {
 		return false
 	}
 	switch t.Name() {
-	case "Bool", "Int64", "Float64", "String":
+	case "Bool", "Int64", "Float64", "String", "Secret":
 		return true
 	default:
 		return false

diff --git a/examples/04_secrets/main.go b/examples/04_secrets/main.go
@@ -0,0 +1,71 @@
+package main
+
+import (
+	"context"
+	"log"
+	"os"
+	"time"
+
+	"github.com/beatlabs/harvester"
+	"github.com/beatlabs/harvester/monitor/consul"
+	"github.com/beatlabs/harvester/sync"
+	"github.com/hashicorp/consul/api"
+)
+
+type config struct {
+	IndexName      sync.String `seed:"customers-v1"`
+	CacheRetention sync.Int64  `seed:"43200" env:"ENV_CACHE_RETENTION_SECONDS"`
+	LogLevel       sync.String `seed:"DEBUG" flag:"loglevel"`
+	AccessToken    sync.Secret `seed:"defaultaccesstoken" consul:"harvester/example_04/accesstoken"`
+}
+
+func main() {
+	ctx, cnl := context.WithCancel(context.Background())
+	defer cnl()
+
+	err := os.Setenv("ENV_CACHE_RETENTION_SECONDS", "86400")
+	if err != nil {
+		log.Fatalf("failed to set env var: %v", err)
+	}
+
+	seedConsulAccessToken("currentaccesstoken")
+
+	cfg := config{}
+
+	ii := []consul.Item{
+		consul.NewKeyItem("harvester/example_04/accesstoken"),
+	}
+
+	h, err := harvester.New(&cfg).
+		WithConsulSeed("127.0.0.1:8500", "", "", 0).
+		WithConsulMonitor("127.0.0.1:8500", "", "", 0, ii...).
+		Create()
+	if err != nil {
+		log.Fatalf("failed to create harvester: %v", err)
+	}
+
+	err = h.Harvest(ctx)
+	if err != nil {
+		log.Fatalf("failed to harvest configuration: %v", err)
+	}
+
+	log.Printf("Config: IndexName: %s, CacheRetention: %d, LogLevel: %s, AccessToken: %s\n", cfg.IndexName.Get(), cfg.CacheRetention.Get(), cfg.LogLevel.Get(), cfg.AccessToken.Get())
+
+	time.Sleep(time.Second)
+	seedConsulAccessToken("newaccesstoken")
+
+	time.Sleep(time.Second)
+	log.Printf("Config: IndexName: %s, CacheRetention: %d, LogLevel: %s, AccessToken: %s\n", cfg.IndexName.Get(), cfg.CacheRetention.Get(), cfg.LogLevel.Get(), cfg.AccessToken.Get())
+}
+
+func seedConsulAccessToken(accessToken string) {
+	cl, err := api.NewClient(api.DefaultConfig())
+	if err != nil {
+		log.Fatalf("failed to create consul client: %v", err)
+	}
+	p := &api.KVPair{Key: "harvester/example_04/accesstoken", Value: []byte(accessToken)}
+	_, err = cl.KV().Put(p, nil)
+	if err != nil {
+		log.Fatalf("failed to put key value pair to consul: %v", err)
+	}
+}
diff --git a/examples/README.md b/examples/README.md
@@ -67,3 +67,24 @@ A fast way to get consul is the following:
     2019/09/19 11:31:13 WARN: version 31 is older or same as the field's OpeningBalance
     2019/09/19 11:31:14 INFO: field OpeningBalance updated with value 999.99, version: 33
     2019/09/19 11:31:15 Config: IndexName: customers-v1, CacheRetention: 86400, LogLevel: DEBUG, OpeningBalance: 999.990000
+
+## 04 Monitor Consul for live changes with secrets
+
+    2019/09/24 16:40:14 INFO: field IndexName updated with value customers-v1, version: 0
+    2019/09/24 16:40:14 INFO: seed value customers-v1 applied on field IndexName
+    2019/09/24 16:40:14 INFO: field CacheRetention updated with value 43200, version: 0
+    2019/09/24 16:40:14 INFO: seed value 43200 applied on field CacheRetention
+    2019/09/24 16:40:14 INFO: field CacheRetention updated with value 86400, version: 0
+    2019/09/24 16:40:14 INFO: env var value 86400 applied on field CacheRetention
+    2019/09/24 16:40:14 INFO: field LogLevel updated with value DEBUG, version: 0
+    2019/09/24 16:40:14 INFO: seed value DEBUG applied on field LogLevel
+    2019/09/24 16:40:14 INFO: field AccessToken updated with value ***, version: 0
+    2019/09/24 16:40:14 INFO: seed value *** applied on field AccessToken
+    2019/09/24 16:40:14 INFO: field AccessToken updated with value ***, version: 135
+    2019/09/24 16:40:14 INFO: consul value *** applied on field AccessToken
+    2019/09/24 16:40:14 WARN: flag var loglevel did not exist for field LogLevel
+    2019/09/24 16:40:14 INFO: plan for key harvester/example_04/accesstoken created
+    2019/09/24 16:40:14 Config: IndexName: customers-v1, CacheRetention: 86400, LogLevel: DEBUG, AccessToken: currentaccesstoken
+    2019/09/24 16:40:14 WARN: version 135 is older or same as the field's AccessToken
+    2019/09/24 16:40:15 INFO: field AccessToken updated with value ***, version: 136
+    2019/09/24 16:40:16 Config: IndexName: customers-v1, CacheRetention: 86400, LogLevel: DEBUG, AccessToken: newaccesstoken
diff --git a/harvester_integration_test.go b/harvester_integration_test.go
@@ -3,22 +3,52 @@
 package harvester
 
 import (
+	"bytes"
 	"context"
 	"log"
 	"os"
 	"testing"
 	"time"
 
+	"github.com/beatlabs/harvester/sync"
+
 	"github.com/beatlabs/harvester/monitor/consul"
 	"github.com/hashicorp/consul/api"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
 
 var (
-	csl *api.KV
+	csl       *api.KV
+	secretLog = []string{
+		"INFO: field Name updated with value ***, version: ",
+		"INFO: seed value *** applied on field Name",
+		"INFO: field Name updated with value ***, version: ",
+		"INFO: consul value *** applied on field Name",
+		"INFO: field Age updated with value 18, version: ",
+		"INFO: seed value 18 applied on field Age",
+		"INFO: field Age updated with value 99, version: ",
+		"INFO: consul value 99 applied on field Age",
+		"INFO: field Balance updated with value 99.900000, version: ",
+		"INFO: seed value 99.900000 applied on field Balance",
+		"INFO: field Balance updated with value 111.100000, version: ",
+		"INFO: consul value 111.100000 applied on field Balance",
+		"INFO: field HasJob updated with value true, version: ",
+		"INFO: seed value true applied on field HasJob",
+		"INFO: field HasJob updated with value false, version: ",
+		"INFO: consul value false applied on field HasJob",
+		"INFO: plan for key harvester1/name created",
+		"INFO: plan for keyprefix harvester created",
+	}
 )
 
+type testConfigWithSecret struct {
+	Name    sync.Secret  `seed:"John Doe" consul:"harvester1/name"`
+	Age     sync.Int64   `seed:"18"  consul:"harvester/age"`
+	Balance sync.Float64 `seed:"99.9"  consul:"harvester/balance"`
+	HasJob  sync.Bool    `seed:"true"  consul:"harvester/has-job"`
+}
+
 func TestMain(m *testing.M) {
 	config := api.DefaultConfig()
 	config.Address = addr
@@ -44,7 +74,9 @@ func TestMain(m *testing.M) {
 }
 
 func Test_harvester_Harvest(t *testing.T) {
-	cfg := testConfig{}
+	buf := bytes.NewBuffer(make([]byte, 0))
+	log.SetOutput(buf)
+	cfg := testConfigWithSecret{}
 	ii := []consul.Item{consul.NewKeyItem("harvester1/name"), consul.NewPrefixItem("harvester")}
 	h, err := New(&cfg).
 		WithConsulSeed(addr, "", "", 0).
@@ -55,6 +87,7 @@ func Test_harvester_Harvest(t *testing.T) {
 	ctx, cnl := context.WithCancel(context.Background())
 	defer cnl()
 	err = h.Harvest(ctx)
+	testLogOutput(buf, t)
 	assert.NoError(t, err)
 	assert.Equal(t, "Mr. Smith", cfg.Name.Get())
 	assert.Equal(t, int64(99), cfg.Age.Get())
@@ -66,6 +99,13 @@ func Test_harvester_Harvest(t *testing.T) {
 	assert.Equal(t, "Mr. Anderson", cfg.Name.Get())
 }
 
+func testLogOutput(buf *bytes.Buffer, t *testing.T) {
+	log := buf.String()
+	for _, logLine := range secretLog {
+		assert.Contains(t, log, logLine)
+	}
+}
+
 func cleanup() error {
 	_, err := csl.Delete("harvester1/name", nil)
 	if err != nil {

diff --git a/seed/seed.go b/seed/seed.go
@@ -64,7 +64,7 @@ func (s *Seeder) Seed(cfg *config.Config) error {
 			if err != nil {
 				return err
 			}
-			log.Infof("seed value %s applied on field %s", val, f.Name())
+			log.Infof("seed value %v applied on field %s", f, f.Name())
 			seedMap[f] = true
 		}
 		key, ok := ss[config.SourceEnv]
@@ -75,7 +75,7 @@ func (s *Seeder) Seed(cfg *config.Config) error {
 				if err != nil {
 					return err
 				}
-				log.Infof("env var value %s applied on field %s", val, f.Name())
+				log.Infof("env var value %v applied on field %s", f, f.Name())
 				seedMap[f] = true
 			} else {
 				log.Warnf("env var %s did not exist for field %s", key, f.Name())
@@ -106,7 +106,7 @@ func (s *Seeder) Seed(cfg *config.Config) error {
 			if err != nil {
 				return err
 			}
-			log.Infof("consul value %s applied on field %s", *value, f.Name())
+			log.Infof("consul value %v applied on field %s", f, f.Name())
 			seedMap[f] = true
 		}
 	}
@@ -140,7 +140,7 @@ func (s *Seeder) Seed(cfg *config.Config) error {
 				if err != nil {
 					return err
 				}
-				log.Infof("flag value %s applied on field %s", *flagInfo.value, flagInfo.field.Name())
+				log.Infof("flag value %v applied on field %s", flagInfo.field, flagInfo.field.Name())
 				seedMap[flagInfo.field] = true
 			} else {
 				log.Warnf("flag var %s did not exist for field %s", flagInfo.key, flagInfo.field.Name())