Skip to content

Commit

Permalink
Improve capabilities of planned data redaction
Browse files Browse the repository at this point in the history 
Support more complex data shapes in terraform plans.
  • Loading branch information
@DavidS-ovm
DavidS-ovm committed Nov 10, 2023
1 parent 2b2d5dd commit 0975d17
Show file tree
Hide file tree
Showing 4 changed files with 136 additions and 28 deletions.
39 changes: 27 additions & 12 deletions cmd/submitplan.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -81,9 +81,9 @@ type TfData struct {
}

// maskAllData masks every entry in attributes as redacted
func maskAllData(attributes map[string]interface{}) map[string]interface{} {
func maskAllData(attributes map[string]any) map[string]any {
for k, v := range attributes {
if mv, ok := v.(map[string]interface{}); ok {
if mv, ok := v.(map[string]any); ok {
attributes[k] = maskAllData(mv)
} else {
attributes[k] = "REDACTED"
Expand All @@ -93,35 +93,50 @@ func maskAllData(attributes map[string]interface{}) map[string]interface{} {
}

// maskSensitiveData masks every entry in attributes that is set to true in sensitive. returns the redacted attributes
func maskSensitiveData(attributes, sensitive map[string]interface{}) map[string]interface{} {
func maskSensitiveData(attributes, sensitive map[string]any) map[string]any {
for k, s := range sensitive {
log.Debugf("checking %v", k)
if mv, ok := s.(map[string]interface{}); ok {
if sub, ok := attributes[k].(map[string]interface{}); ok {
log.Debugf("checking %v: %v", k, s)
if mv, ok := s.(map[string]any); ok {
if sub, ok := attributes[k].(map[string]any); ok {
attributes[k] = maskSensitiveData(sub, mv)
}
} else if arr, ok := s.([]any); ok {
if sub, ok := attributes[k].([]any); ok {
if len(arr) != len(sub) {
attributes[k] = "REDACTED (len mismatch)"
continue
}
for i, v := range arr {
if v == true {
sub[i] = "REDACTED"
}
}
attributes[k] = sub
}
} else {
attributes[k] = "REDACTED"
if _, ok := attributes[k]; ok {
attributes[k] = "REDACTED"
}
}
}
return attributes
}

func itemAttributesFromResourceChangeData(attributesMsg, sensitiveMsg json.RawMessage) (*sdp.ItemAttributes, error) {
var attributes map[string]interface{}
var attributes map[string]any
err := json.Unmarshal(attributesMsg, &attributes)
if err != nil {
return nil, fmt.Errorf("failed to parse attributes: %w", err)
}

// sensitiveMsg can be a bool or a map[string]interface{}
// sensitiveMsg can be a bool or a map[string]any
var isSensitive bool
err = json.Unmarshal(sensitiveMsg, &isSensitive)
if err == nil && isSensitive {
attributes = maskAllData(attributes)
} else if err != nil {
// only try parsing as map if parsing as bool failed
var sensitive map[string]interface{}
var sensitive map[string]any
err = json.Unmarshal(sensitiveMsg, &sensitive)
if err != nil {
return nil, fmt.Errorf("failed to parse sensitive: %w", err)
Expand Down Expand Up @@ -164,7 +179,7 @@ func itemDiffFromResourceChange(resourceChange ResourceChange) (*sdp.ItemDiff, e
}

if beforeAttributes != nil {
result.Before = &sdp.Item{ // DODO: check for empty before/after attributes and set to nil
result.Before = &sdp.Item{
Type: resourceChange.Type,
UniqueAttribute: "terraform_address",
Attributes: beforeAttributes,
Expand Down Expand Up @@ -326,7 +341,7 @@ func mappedItemDiffsFromPlan(ctx context.Context, fileName string, lf log.Fields
}

// Create the map that variables will pull data from
dataMap := make(map[string]interface{})
dataMap := make(map[string]any)

// Populate resource values
dataMap["values"] = currentResource.AttributeValues
Expand Down
105 changes: 105 additions & 0 deletions cmd/submitplan_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ import (

"github.com/overmindtech/sdp-go"
"github.com/sirupsen/logrus"
"github.com/stretchr/testify/require"
)

func TestMappedItemDiffsFromPlan(t *testing.T) {
Expand Down Expand Up @@ -136,3 +137,107 @@ func TestMappedItemDiffsFromPlan(t *testing.T) {
t.Errorf("Expected aws_iam_policy query query to be 'arn:aws:iam::123456789012:policy/test-alb-ingress', got '%v'", aws_iam_policy.MappingQuery.Query)
}
}

// note that these tests need to allocate the input map for every test as
// maskSensitiveData mutates its inputs
func TestMaskSensitiveData(t *testing.T) {
t.Parallel()

t.Run("empty", func(t *testing.T) {
t.Parallel()
got := maskSensitiveData(map[string]any{}, map[string]any{})
require.Equal(t, got, map[string]any{})
})

t.Run("easy", func(t *testing.T) {
t.Parallel()
require.Equal(t,
map[string]any{
"foo": "bar",
},
maskSensitiveData(
map[string]any{
"foo": "bar",
},
map[string]any{}))

require.Equal(t,
map[string]any{
"foo": "REDACTED",
},
maskSensitiveData(
map[string]any{
"foo": "bar",
},
map[string]any{"foo": true}))

})

t.Run("deep", func(t *testing.T) {
t.Parallel()
require.Equal(t,
map[string]any{
"foo": map[string]any{"key": "bar"},
},
maskSensitiveData(
map[string]any{
"foo": map[string]any{"key": "bar"},
},
map[string]any{}))

require.Equal(t,
map[string]any{
"foo": "REDACTED",
},
maskSensitiveData(
map[string]any{
"foo": map[string]any{"key": "bar"},
},
map[string]any{"foo": true}))

require.Equal(t,
map[string]any{
"foo": map[string]any{"key": "REDACTED"},
},
maskSensitiveData(
map[string]any{
"foo": map[string]any{"key": "bar"},
},
map[string]any{"foo": map[string]any{"key": true}}))

})

t.Run("arrays", func(t *testing.T) {
t.Parallel()
require.Equal(t,
map[string]any{
"foo": []any{"one", "two"},
},
maskSensitiveData(
map[string]any{
"foo": []any{"one", "two"},
},
map[string]any{}))

require.Equal(t,
map[string]any{
"foo": "REDACTED",
},
maskSensitiveData(
map[string]any{
"foo": []any{"one", "two"},
},
map[string]any{"foo": true}))

require.Equal(t,
map[string]any{
"foo": []any{"one", "REDACTED"},
},
maskSensitiveData(
map[string]any{
"foo": []any{"one", "two"},
},
map[string]any{"foo": []any{false, true}}))

})
}
5 changes: 4 additions & 1 deletion go.mod
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,9 +10,9 @@ require (
github.com/mattn/go-isatty v0.0.20
github.com/overmindtech/sdp-go v0.57.0
github.com/sirupsen/logrus v1.9.3
github.com/sourcegraph/conc v0.3.0
github.com/spf13/cobra v1.8.0
github.com/spf13/viper v1.17.0
github.com/stretchr/testify v1.8.4
github.com/ttacon/chalk v0.0.0-20160626202418-22c06c80ed31
github.com/uptrace/opentelemetry-go-extra/otellogrus v0.2.3
github.com/xiam/dig v0.0.0-20191116195832-893b5fb5093b
Expand All @@ -32,6 +32,7 @@ require (
github.com/auth0/go-jwt-middleware/v2 v2.1.0 // indirect
github.com/aws/aws-sdk-go v1.45.16 // indirect
github.com/cenkalti/backoff/v4 v4.2.1 // indirect
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
github.com/felixge/httpsnoop v1.0.3 // indirect
github.com/fsnotify/fsnotify v1.6.0 // indirect
github.com/go-logr/logr v1.2.4 // indirect
Expand All @@ -50,9 +51,11 @@ require (
github.com/nats-io/nuid v1.0.1 // indirect
github.com/pelletier/go-toml/v2 v2.1.0 // indirect
github.com/pkg/errors v0.9.1 // indirect
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
github.com/rivo/uniseg v0.4.4 // indirect
github.com/sagikazarmark/locafero v0.3.0 // indirect
github.com/sagikazarmark/slog-shim v0.1.0 // indirect
github.com/sourcegraph/conc v0.3.0 // indirect
github.com/spf13/afero v1.10.0 // indirect
github.com/spf13/cast v1.5.1 // indirect
github.com/spf13/pflag v1.0.5 // indirect
Expand Down
15 changes: 0 additions & 15 deletions go.sum
View file
Original file line number Diff line number Diff line change
Expand Up @@ -54,7 +54,6 @@ github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDk
github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=
github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk=
github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk=
github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
github.com/cpuguy83/go-md2man/v2 v2.0.3/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
Expand Down Expand Up @@ -192,8 +191,6 @@ github.com/nats-io/nkeys v0.4.6 h1:IzVe95ru2CT6ta874rt9saQRkWfe2nFj1NtvYSLqMzY=
github.com/nats-io/nkeys v0.4.6/go.mod h1:4DxZNzenSVd1cYQoAa8948QY3QDjrHfcfVADymtkpts=
github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw=
github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c=
github.com/overmindtech/sdp-go v0.56.0 h1:1VFF6kulsNIVVctBClZbeZxOsHKRQDPzfZPVmwcWGSo=
github.com/overmindtech/sdp-go v0.56.0/go.mod h1:WZ/CsRATgtF0KZpgrlJdzLieBO3Gk9mk5XcdrSVOkbQ=
github.com/overmindtech/sdp-go v0.57.0 h1:hUqBij/gmc11TaQF4OOKj15kZH6SZJsObQgeCah0vRc=
github.com/overmindtech/sdp-go v0.57.0/go.mod h1:G01shsRPrtJBMQb2HizIG1BIst7jG8Q6BZ+G8/9YPHM=
github.com/pelletier/go-toml/v2 v2.1.0 h1:FnwAJ4oYMvbT/34k9zzHuZNrhlz48GB3/s6at6/MHO4=
Expand Down Expand Up @@ -227,8 +224,6 @@ github.com/spf13/afero v1.10.0 h1:EaGW2JJh15aKOejeuJ+wpFSHnbd7GE6Wvp3TsNhb6LY=
github.com/spf13/afero v1.10.0/go.mod h1:UBogFpq8E9Hx+xc5CNTTEpTnuHVmXDwZcZcE1eb/UhQ=
github.com/spf13/cast v1.5.1 h1:R+kOtfhWQE6TVQzY+4D7wJLBgkdVasCEFxSUBYBYIlA=
github.com/spf13/cast v1.5.1/go.mod h1:b9PdjNptOpzXr7Rq1q9gJML/2cdGQAo69NKzQ10KN48=
github.com/spf13/cobra v1.7.0 h1:hyqWnYt1ZQShIddO5kBpj3vu05/++x6tJ6dg8EC572I=
github.com/spf13/cobra v1.7.0/go.mod h1:uLxZILRyS/50WlhOIKD7W6V5bgeIt+4sICxh6uRMrb0=
github.com/spf13/cobra v1.8.0 h1:7aJaZx1B85qltLMc546zn58BxxfZdR/W22ej9CFoEf0=
github.com/spf13/cobra v1.8.0/go.mod h1:WXLWApfZ71AjXPya3WOlMsY9yMs7YeiHhFVlvLyhcho=
github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
Expand Down Expand Up @@ -302,8 +297,6 @@ golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPh
golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4=
golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
golang.org/x/crypto v0.14.0 h1:wBqGXzWJW6m1XrIKlAH0Hs1JJ7+9KBwnIO8v66Q9cHc=
golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4=
golang.org/x/crypto v0.15.0 h1:frVn1TEaCEaZcn3Tmd7Y2b5KKPaZ+I32Q2OA3kYp5TA=
golang.org/x/crypto v0.15.0/go.mod h1:4ChreQoLWfG3xLDer1WdlH5NdlQ3+mwnQq1YTKY+72g=
golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
Expand Down Expand Up @@ -376,8 +369,6 @@ golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v
golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco=
golang.org/x/net v0.17.0 h1:pVaXccu2ozPjCXewfr1S7xza/zcXTity9cCdXQYSjIM=
golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE=
golang.org/x/net v0.18.0 h1:mIYleuAkSbHh0tCv7RvjL3F6ZVbLjq4+R7zbOn3Kokg=
golang.org/x/net v0.18.0/go.mod h1:/czyP5RqHAH4odGYxBJ1qz0+CE5WZ+2j1YgoEo8F2jQ=
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
Expand All @@ -389,8 +380,6 @@ golang.org/x/oauth2 v0.0.0-20200902213428-5d25da1a8d43/go.mod h1:KelEdhl1UZF7XfJ
golang.org/x/oauth2 v0.0.0-20201109201403-9fd604954f58/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
golang.org/x/oauth2 v0.0.0-20201208152858-08078c50e5b5/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
golang.org/x/oauth2 v0.0.0-20210218202405-ba52d332ba99/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
golang.org/x/oauth2 v0.13.0 h1:jDDenyj+WgFtmV3zYVoi8aE2BwtXFLWOA67ZfNWftiY=
golang.org/x/oauth2 v0.13.0/go.mod h1:/JMhi4ZRXAf4HG9LiNmxvk+45+96RUlVThiH8FzNBn0=
golang.org/x/oauth2 v0.14.0 h1:P0Vrf/2538nmC0H+pEQ3MNFRRnVR7RlqyVw+bvm26z0=
golang.org/x/oauth2 v0.14.0/go.mod h1:lAtNWgaWfL4cm7j2OV8TxGi9Qb7ECORx8DktCY74OwM=
golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
Expand Down Expand Up @@ -445,8 +434,6 @@ golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBc
golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.13.0 h1:Af8nKPmuFypiUBjVoU9V20FiaFXOcuZI21p0ycVYYGE=
golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.14.0 h1:Vz7Qs629MkJkGyHxUlRHizWJRG2j8fbQKjELVSNhy7Q=
golang.org/x/sys v0.14.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
Expand All @@ -461,8 +448,6 @@ golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
golang.org/x/text v0.13.0 h1:ablQoSUd0tRdKxZewP80B+BaqeKJuVhuRxj/dkrun3k=
golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ=
golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
Expand Down

0 comments on commit 0975d17

Please sign in to comment.