From 093ad39393e8ddc162aafff81717fdaa56cdae4f Mon Sep 17 00:00:00 2001 From: Isaiah Vita Date: Mon, 9 Oct 2023 14:32:22 -0700 Subject: [PATCH 1/8] works on test file, unit tests failing --- internal/ini/ini_lexer.go | 5 +++++ internal/ini/literal_tokens.go | 24 ++++++++++++++++++++++++ 2 files changed, 29 insertions(+) diff --git a/internal/ini/ini_lexer.go b/internal/ini/ini_lexer.go index abf1fb03626..f8e4c809880 100644 --- a/internal/ini/ini_lexer.go +++ b/internal/ini/ini_lexer.go @@ -67,6 +67,8 @@ func (l *iniLexer) tokenize(b []byte) ([]Token, error) { for len(runes) > 0 && count < tokenAmount { switch { + case isSubProperty(runes): + tokens[count], n, err = newLitToken(runes) case isWhitespace(runes[0]): tokens[count], n, err = newWSToken(runes) case isComma(runes[0]): @@ -101,6 +103,8 @@ func countTokens(runes []rune) int { for len(runes) > 0 { switch { + case isSubProperty(runes): + _, n, err = newLitToken(runes) case isWhitespace(runes[0]): _, n, err = newWSToken(runes) case isComma(runes[0]): @@ -128,6 +132,7 @@ func countTokens(runes []rune) int { return count + 1 } + // Token indicates a metadata about a given value. type Token struct { t TokenType diff --git a/internal/ini/literal_tokens.go b/internal/ini/literal_tokens.go index efcd2e6c7da..6790c620c81 100644 --- a/internal/ini/literal_tokens.go +++ b/internal/ini/literal_tokens.go @@ -116,6 +116,10 @@ func newLitToken(b []rune) (Token, int, error) { } token = newToken(TokenLit, b[:n], QuotedStringType) + } else if isSubProperty(b) { + n, err = getSubProperty(b) + offset := 3 // start after newline + ws + token = newToken(TokenLit, b[offset:n], StringType) } else { n, err = getValue(b) token = newToken(TokenLit, b[:n], StringType) @@ -124,6 +128,26 @@ func newLitToken(b []rune) (Token, int, error) { return token, n, err } + +func isSubProperty(runes []rune) bool { + if len(runes) < 3 { + return false + } + return isNewline(runes) && isWhitespace(runes[2]) +} + +func getSubProperty(runes []rune) (int, error) { + offset := 3 // start after newline + ws + for idx, val := range runes[offset:] { + if val == '\n' { + return idx, nil + } + } + return 0, fmt.Errorf("no sub property") +} + + + // IntValue returns an integer value func (v Value) IntValue() (int64, bool) { i, err := strconv.ParseInt(string(v.raw), 0, 64) From 828047e98e9aacd3b1d56826fd2ac46a7cef80ea Mon Sep 17 00:00:00 2001 From: Isaiah Vita Date: Tue, 10 Oct 2023 08:55:16 -0700 Subject: [PATCH 2/8] ini and config tests passing --- internal/ini/ini_parser_test.go | 31 ++++++++++++------- internal/ini/literal_tokens.go | 31 ++++++++++++++----- .../testdata/valid/op_sep_in_values_expected | 2 +- 3 files changed, 43 insertions(+), 21 deletions(-) diff --git a/internal/ini/ini_parser_test.go b/internal/ini/ini_parser_test.go index b0bc1a136a3..50369655ef2 100644 --- a/internal/ini/ini_parser_test.go +++ b/internal/ini/ini_parser_test.go @@ -17,6 +17,9 @@ func TestParser(t *testing.T) { regionLit, _, _ := newLitToken([]rune(`"us-west-2"`)) regionNoQuotesLit, _, _ := newLitToken([]rune("us-west-2")) + s3ServiceId, _, _ := newLitToken([]rune("s3")) + nestedParamsLit, _, _ := newLitToken([]rune("\n\tfoo=bar\n\tbar=baz\n")) + credentialID, _, _ := newLitToken([]rune("credential_source")) ec2MetadataLit, _, _ := newLitToken([]rune("Ec2InstanceMetadata")) @@ -57,6 +60,9 @@ func TestParser(t *testing.T) { sepInValueExpr := newEqualExpr(newExpression(sepInValueID), equalOp) sepInValueExpr.AppendChild(newExpression(sepInValueLit)) + nestedEQExpr := newEqualExpr(newExpression(s3ServiceId), equalOp) + nestedEQExpr.AppendChild(newExpression(nestedParamsLit)) + cases := []struct { name string r io.Reader @@ -206,8 +212,9 @@ func TestParser(t *testing.T) { }, { name: "section statement", - r: bytes.NewBuffer([]byte(`[default] - region="us-west-2"`)), + r: bytes.NewBuffer([]byte( +`[default] +region="us-west-2"`)), expectedStack: []AST{ newCompletedSectionStatement( defaultProfileStmt, @@ -217,15 +224,15 @@ func TestParser(t *testing.T) { }, { name: "complex section statement", - r: bytes.NewBuffer([]byte(`[default] - region = us-west-2 - credential_source = Ec2InstanceMetadata - output = json + r: bytes.NewBuffer([]byte( +`[default] +region = us-west-2 +credential_source = Ec2InstanceMetadata +output = json - [assumerole] - output = json - region = us-west-2 - `)), +[assumerole] +output = json +region = us-west-2`)), expectedStack: []AST{ newCompletedSectionStatement( defaultProfileStmt, @@ -258,7 +265,7 @@ region = us-west-2 newCompletedSectionStatement( defaultProfileStmt, ), - newSkipStatement(newEqualExpr(newExpression(s3ID), equalOp)), + newExprStatement(nestedEQExpr), newExprStatement(noQuotesRegionEQRegion), newExprStatement(credEQExpr), newExprStatement(outputEQExpr), @@ -289,7 +296,7 @@ region = us-west-2 ), newExprStatement(noQuotesRegionEQRegion), newExprStatement(credEQExpr), - newSkipStatement(newEqualExpr(newExpression(s3ID), equalOp)), + newExprStatement(nestedEQExpr), newExprStatement(outputEQExpr), newCompletedSectionStatement( assumeProfileStmt, diff --git a/internal/ini/literal_tokens.go b/internal/ini/literal_tokens.go index 6790c620c81..74230e7c664 100644 --- a/internal/ini/literal_tokens.go +++ b/internal/ini/literal_tokens.go @@ -117,9 +117,14 @@ func newLitToken(b []rune) (Token, int, error) { token = newToken(TokenLit, b[:n], QuotedStringType) } else if isSubProperty(b) { - n, err = getSubProperty(b) - offset := 3 // start after newline + ws - token = newToken(TokenLit, b[offset:n], StringType) + // _, start, _ := newNewlineToken(b) + // removeWhitespace := start + 1 + end, err := getSubProperty(b, 0) + if err != nil { + return token, n, err + } + token = newToken(TokenLit, b[0:end], StringType) + n = end } else { n, err = getValue(b) token = newToken(TokenLit, b[:n], StringType) @@ -133,14 +138,24 @@ func isSubProperty(runes []rune) bool { if len(runes) < 3 { return false } - return isNewline(runes) && isWhitespace(runes[2]) + split := strings.Split(string(runes), "=") + if len(split) < 2 { + return false + } + if !isNewline(runes) { + return false + } + _, n, err := newNewlineToken(runes) + if err != nil { + return false + } + return isWhitespace(runes[n]) } -func getSubProperty(runes []rune) (int, error) { - offset := 3 // start after newline + ws +func getSubProperty(runes []rune, offset int) (int, error) { for idx, val := range runes[offset:] { - if val == '\n' { - return idx, nil + if val == '\n' && !isSubProperty(runes[offset+idx:]) { + return offset + idx, nil } } return 0, fmt.Errorf("no sub property") diff --git a/internal/ini/testdata/valid/op_sep_in_values_expected b/internal/ini/testdata/valid/op_sep_in_values_expected index a2609269eb4..8d33b62a26a 100644 --- a/internal/ini/testdata/valid/op_sep_in_values_expected +++ b/internal/ini/testdata/valid/op_sep_in_values_expected @@ -19,7 +19,7 @@ "key": "value5" }, "case6": { - "s3": "", + "s3": "[nested6]\n key = valuen6", "key": "=value6" }, "case7": { From ace1cfe96e01e42072bf022b0fe3ae577f9c50ff Mon Sep 17 00:00:00 2001 From: Isaiah Vita Date: Tue, 10 Oct 2023 09:21:56 -0700 Subject: [PATCH 3/8] add comments, add extra op --- internal/ini/ini_lexer.go | 2 +- internal/ini/literal_tokens.go | 26 ++++++++++++++++++++------ internal/ini/op_tokens.go | 1 + 3 files changed, 22 insertions(+), 7 deletions(-) diff --git a/internal/ini/ini_lexer.go b/internal/ini/ini_lexer.go index f8e4c809880..637306a5f56 100644 --- a/internal/ini/ini_lexer.go +++ b/internal/ini/ini_lexer.go @@ -104,7 +104,7 @@ func countTokens(runes []rune) int { for len(runes) > 0 { switch { case isSubProperty(runes): - _, n, err = newLitToken(runes) + _, n, err = newLitToken(runes) case isWhitespace(runes[0]): _, n, err = newWSToken(runes) case isComma(runes[0]): diff --git a/internal/ini/literal_tokens.go b/internal/ini/literal_tokens.go index 74230e7c664..77ff78e56f6 100644 --- a/internal/ini/literal_tokens.go +++ b/internal/ini/literal_tokens.go @@ -114,16 +114,14 @@ func newLitToken(b []rune) (Token, int, error) { if err != nil { return token, n, err } - token = newToken(TokenLit, b[:n], QuotedStringType) } else if isSubProperty(b) { - // _, start, _ := newNewlineToken(b) - // removeWhitespace := start + 1 - end, err := getSubProperty(b, 0) + offset := 0 // include leading newline and whitespace + end, err := getSubProperty(b, offset) if err != nil { return token, n, err } - token = newToken(TokenLit, b[0:end], StringType) + token = newToken(TokenLit, b[offset:end], StringType) n = end } else { n, err = getValue(b) @@ -134,14 +132,24 @@ func newLitToken(b []rune) (Token, int, error) { } + + func isSubProperty(runes []rune) bool { + // needs at least + // (1) newline (2) whitespace (3) literal if len(runes) < 3 { return false } - split := strings.Split(string(runes), "=") + + // must have an equal expression + split := strings.FieldsFunc(string(runes), func(r rune)(bool){ + return isOp([]rune{r}) + }) if len(split) < 2 { return false } + + // must start with a new line if !isNewline(runes) { return false } @@ -149,9 +157,15 @@ func isSubProperty(runes []rune) bool { if err != nil { return false } + + // whitespace must follow newline return isWhitespace(runes[n]) } +// getSubProperty pulls all subproperties and terminates when +// it hits a newline that is not the start of another subproperty. +// offset allows for removal of leading newline and whitespace +// characters func getSubProperty(runes []rune, offset int) (int, error) { for idx, val := range runes[offset:] { if val == '\n' && !isSubProperty(runes[offset+idx:]) { diff --git a/internal/ini/op_tokens.go b/internal/ini/op_tokens.go index 8a84c7cbe08..0abaa98bff7 100644 --- a/internal/ini/op_tokens.go +++ b/internal/ini/op_tokens.go @@ -24,6 +24,7 @@ func isOp(b []rune) bool { } } + func newOpToken(b []rune) (Token, int, error) { tok := Token{} From ad797eab284416b8e22f1592902e7c106b2535f2 Mon Sep 17 00:00:00 2001 From: Isaiah Vita Date: Tue, 10 Oct 2023 09:23:14 -0700 Subject: [PATCH 4/8] remove added newline --- internal/ini/op_tokens.go | 1 - 1 file changed, 1 deletion(-) diff --git a/internal/ini/op_tokens.go b/internal/ini/op_tokens.go index 0abaa98bff7..8a84c7cbe08 100644 --- a/internal/ini/op_tokens.go +++ b/internal/ini/op_tokens.go @@ -24,7 +24,6 @@ func isOp(b []rune) bool { } } - func newOpToken(b []rune) (Token, int, error) { tok := Token{} From c75bc0f5dbbca28b37c7ff2d963bb7a422d380d0 Mon Sep 17 00:00:00 2001 From: Isaiah Vita Date: Tue, 10 Oct 2023 09:47:41 -0700 Subject: [PATCH 5/8] fix casing --- internal/ini/ini_parser_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/ini/ini_parser_test.go b/internal/ini/ini_parser_test.go index 50369655ef2..c262dfb1d67 100644 --- a/internal/ini/ini_parser_test.go +++ b/internal/ini/ini_parser_test.go @@ -17,7 +17,7 @@ func TestParser(t *testing.T) { regionLit, _, _ := newLitToken([]rune(`"us-west-2"`)) regionNoQuotesLit, _, _ := newLitToken([]rune("us-west-2")) - s3ServiceId, _, _ := newLitToken([]rune("s3")) + s3ServiceID, _, _ := newLitToken([]rune("s3")) nestedParamsLit, _, _ := newLitToken([]rune("\n\tfoo=bar\n\tbar=baz\n")) credentialID, _, _ := newLitToken([]rune("credential_source")) From 1bbd8a569f6e0d83a1fa34a0ea232529f3361aac Mon Sep 17 00:00:00 2001 From: Isaiah Vita Date: Tue, 10 Oct 2023 09:54:46 -0700 Subject: [PATCH 6/8] fix casing 2 --- internal/ini/ini_parser_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/ini/ini_parser_test.go b/internal/ini/ini_parser_test.go index c262dfb1d67..fac3cb752f5 100644 --- a/internal/ini/ini_parser_test.go +++ b/internal/ini/ini_parser_test.go @@ -60,7 +60,7 @@ func TestParser(t *testing.T) { sepInValueExpr := newEqualExpr(newExpression(sepInValueID), equalOp) sepInValueExpr.AppendChild(newExpression(sepInValueLit)) - nestedEQExpr := newEqualExpr(newExpression(s3ServiceId), equalOp) + nestedEQExpr := newEqualExpr(newExpression(s3ServiceID), equalOp) nestedEQExpr.AppendChild(newExpression(nestedParamsLit)) cases := []struct { From 1bc1c8fd5a6b7b1665d9acf67faf3810235b02dd Mon Sep 17 00:00:00 2001 From: Isaiah Vita Date: Tue, 10 Oct 2023 20:09:02 -0700 Subject: [PATCH 7/8] add new literal value map type --- internal/ini/literal_tokens.go | 22 ++++++++++++++----- .../ini/testdata/valid/nested_fields_expected | 1 + .../testdata/valid/op_sep_in_values_expected | 2 +- internal/ini/walker_test.go | 7 +++++- 4 files changed, 25 insertions(+), 7 deletions(-) diff --git a/internal/ini/literal_tokens.go b/internal/ini/literal_tokens.go index 77ff78e56f6..6692f42a98d 100644 --- a/internal/ini/literal_tokens.go +++ b/internal/ini/literal_tokens.go @@ -60,7 +60,6 @@ const ( NoneType = ValueType(iota) StringType QuotedStringType - // FUTURE(2226) MapType ) // Value is a union container @@ -69,7 +68,7 @@ type Value struct { raw []rune str string - // FUTURE(2226) mp map[string]string + mp map[string]string } func newValue(t ValueType, base int, raw []rune) (Value, error) { @@ -81,6 +80,21 @@ func newValue(t ValueType, base int, raw []rune) (Value, error) { switch t { case StringType: v.str = string(raw) + if isSubProperty(raw) { + newlineParts := strings.Split(string(raw), "\n") + for _, part := range newlineParts { + operandParts := strings.Split(part, "=") + if len(operandParts) < 2 { + continue + } + key := strings.TrimSpace(operandParts[0]) + val := strings.TrimSpace(operandParts[1]) + if v.mp == nil { + v.mp = make(map[string]string) + } + v.mp[key] = val + } + } case QuotedStringType: v.str = string(raw[1 : len(raw)-1]) } @@ -131,9 +145,6 @@ func newLitToken(b []rune) (Token, int, error) { return token, n, err } - - - func isSubProperty(runes []rune) bool { // needs at least // (1) newline (2) whitespace (3) literal @@ -218,6 +229,7 @@ func isTrimmable(r rune) bool { // StringValue returns the string value func (v Value) StringValue() string { switch v.Type { + case StringType: return strings.TrimFunc(string(v.raw), isTrimmable) case QuotedStringType: diff --git a/internal/ini/testdata/valid/nested_fields_expected b/internal/ini/testdata/valid/nested_fields_expected index 0f1ae73acfc..c51a24cbdea 100644 --- a/internal/ini/testdata/valid/nested_fields_expected +++ b/internal/ini/testdata/valid/nested_fields_expected @@ -1,5 +1,6 @@ { "foo": { + "aws_access_key_id": "map[aws_secret_access_key:valid]", "aws_session_token": "valid" }, "bar": { diff --git a/internal/ini/testdata/valid/op_sep_in_values_expected b/internal/ini/testdata/valid/op_sep_in_values_expected index 8d33b62a26a..7a85944b1ad 100644 --- a/internal/ini/testdata/valid/op_sep_in_values_expected +++ b/internal/ini/testdata/valid/op_sep_in_values_expected @@ -19,7 +19,7 @@ "key": "value5" }, "case6": { - "s3": "[nested6]\n key = valuen6", + "s3": "map[key:valuen6]", "key": "=value6" }, "case7": { diff --git a/internal/ini/walker_test.go b/internal/ini/walker_test.go index 02946be782b..87cef0b5419 100644 --- a/internal/ini/walker_test.go +++ b/internal/ini/walker_test.go @@ -71,7 +71,12 @@ func TestValidDataFiles(t *testing.T) { for k, v := range table { switch e := v.(type) { case string: - a := p.String(k) + var a string + if p.values[k].mp != nil { + a = fmt.Sprintf("%v", p.values[k].mp) + } else { + a = p.String(k) + } if e != a { t.Errorf("%s: expected %v, but received %v for profile %v", path, e, a, profile) } From 0968d2a61625853178897c1243d66bad3136b4ac Mon Sep 17 00:00:00 2001 From: Isaiah Vita Date: Wed, 11 Oct 2023 07:32:32 -0700 Subject: [PATCH 8/8] remove unnecessary comment --- internal/ini/literal_tokens.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/ini/literal_tokens.go b/internal/ini/literal_tokens.go index 6692f42a98d..ee2e0cc5a46 100644 --- a/internal/ini/literal_tokens.go +++ b/internal/ini/literal_tokens.go @@ -130,7 +130,7 @@ func newLitToken(b []rune) (Token, int, error) { } token = newToken(TokenLit, b[:n], QuotedStringType) } else if isSubProperty(b) { - offset := 0 // include leading newline and whitespace + offset := 0 end, err := getSubProperty(b, offset) if err != nil { return token, n, err