Support Op and Sep Tokens in Expression Values

internal/ini/ini_parser.go

@@ -5,9 +5,12 @@ import (
 	"io"
 )
 
+// ParseState represents the current state of the parser.
+type ParseState uint
+
 // State enums for the parse table
 const (
-	InvalidState = iota
+	InvalidState ParseState = iota
 	// stmt -> value stmt'
 	StatementState
 	// stmt' -> MarkComplete | op stmt
@@ -36,24 +39,24 @@ const (
 )
 
 // parseTable is a state machine to dictate the grammar above.
-var parseTable = map[ASTKind]map[TokenType]int{
-	ASTKindStart: map[TokenType]int{
+var parseTable = map[ASTKind]map[TokenType]ParseState{
+	ASTKindStart: {
 		TokenLit:     StatementState,
 		TokenSep:     OpenScopeState,
 		TokenWS:      SkipTokenState,
 		TokenNL:      SkipTokenState,
 		TokenComment: CommentState,
 		TokenNone:    TerminalState,
 	},
-	ASTKindCommentStatement: map[TokenType]int{
+	ASTKindCommentStatement: {
 		TokenLit:     StatementState,
 		TokenSep:     OpenScopeState,
 		TokenWS:      SkipTokenState,
 		TokenNL:      SkipTokenState,
 		TokenComment: CommentState,
 		TokenNone:    MarkCompleteState,
 	},
-	ASTKindExpr: map[TokenType]int{
+	ASTKindExpr: {
 		TokenOp:      StatementPrimeState,
 		TokenLit:     ValueState,
 		TokenSep:     OpenScopeState,
@@ -62,46 +65,48 @@ var parseTable = map[ASTKind]map[TokenType]int{
 		TokenComment: CommentState,
 		TokenNone:    MarkCompleteState,
 	},
-	ASTKindEqualExpr: map[TokenType]int{
-		TokenLit:     ValueState,
-		TokenWS:      SkipTokenState,
-		TokenNL:      SkipState,
-		TokenNone:    SkipState,
+	ASTKindEqualExpr: {
+		TokenLit:  ValueState,
+		TokenSep:  ValueState,
+		TokenOp:   ValueState,
+		TokenWS:   SkipTokenState,
+		TokenNL:   SkipState,
+		TokenNone: SkipState,
 	},
-	ASTKindStatement: map[TokenType]int{
+	ASTKindStatement: {
 		TokenLit:     SectionState,
 		TokenSep:     CloseScopeState,
 		TokenWS:      SkipTokenState,
 		TokenNL:      SkipTokenState,
 		TokenComment: CommentState,
 		TokenNone:    MarkCompleteState,
 	},
-	ASTKindExprStatement: map[TokenType]int{
+	ASTKindExprStatement: {
 		TokenLit:     ValueState,
-		TokenSep:     OpenScopeState,
+		TokenSep:     ValueState,
 		TokenOp:      ValueState,
 		TokenWS:      ValueState,
 		TokenNL:      MarkCompleteState,
 		TokenComment: CommentState,
 		TokenNone:    TerminalState,
 		TokenComma:   SkipState,
 	},
-	ASTKindSectionStatement: map[TokenType]int{
+	ASTKindSectionStatement: {
 		TokenLit: SectionState,
 		TokenOp:  SectionState,
 		TokenSep: CloseScopeState,
 		TokenWS:  SectionState,
 		TokenNL:  SkipTokenState,
 	},
-	ASTKindCompletedSectionStatement: map[TokenType]int{
+	ASTKindCompletedSectionStatement: {
 		TokenWS:      SkipTokenState,
 		TokenNL:      SkipTokenState,
 		TokenLit:     StatementState,
 		TokenSep:     OpenScopeState,
 		TokenComment: CommentState,
 		TokenNone:    MarkCompleteState,
 	},
-	ASTKindSkipStatement: map[TokenType]int{
+	ASTKindSkipStatement: {
 		TokenLit:     StatementState,
 		TokenSep:     OpenScopeState,
 		TokenWS:      SkipTokenState,
@@ -243,7 +248,7 @@ loop:
 				}
 
 				children[len(children)-1] = rhs
-				k.SetChildren(children)
+				root.SetChildren(children)
 
 				stack.Push(k)
 			}

internal/ini/ini_parser_test.go

@@ -25,6 +25,9 @@ func TestParser(t *testing.T) {
 	outputID, _, _ := newLitToken([]rune("output"))
 	outputLit, _, _ := newLitToken([]rune("json"))
 
+	sepInValueID, _, _ := newLitToken([]rune("sepInValue"))
+	sepInValueLit := newToken(TokenOp, []rune("=:[foo]]bar["), StringType)
+
 	equalOp, _, _ := newOpToken([]rune("= 1234"))
 	equalColonOp, _, _ := newOpToken([]rune(": 1234"))
 	numLit, _, _ := newLitToken([]rune("1234"))
@@ -53,6 +56,9 @@ func TestParser(t *testing.T) {
 	outputEQExpr := newEqualExpr(newExpression(outputID), equalOp)
 	outputEQExpr.AppendChild(newExpression(outputLit))
 
+	sepInValueExpr := newEqualExpr(newExpression(sepInValueID), equalOp)
+	sepInValueExpr.AppendChild(newExpression(sepInValueLit))
+
 	cases := []struct {
 		name          string
 		r             io.Reader
@@ -67,24 +73,48 @@ func TestParser(t *testing.T) {
 			},
 		},
 		{
-			name:          "0==0",
-			r:             bytes.NewBuffer([]byte(`0==0`)),
-			expectedError: true,
+			name: "0==0",
+			r:    bytes.NewBuffer([]byte(`0==0`)),
+			expectedStack: []AST{
+				func() AST {
+					equalExpr := newEqualExpr(newExpression(newToken(TokenLit, []rune("0"), StringType)), equalOp)
+					equalExpr.AppendChild(newExpression(newToken(TokenOp, []rune("=0"), StringType)))
+					return newExprStatement(equalExpr)
+				}(),
+			},
 		},
 		{
-			name:          "0=:0",
-			r:             bytes.NewBuffer([]byte(`0=:0`)),
-			expectedError: true,
+			name: "0=:0",
+			r:    bytes.NewBuffer([]byte(`0=:0`)),
+			expectedStack: []AST{
+				func() AST {
+					equalExpr := newEqualExpr(newExpression(newToken(TokenLit, []rune("0"), StringType)), equalOp)
+					equalExpr.AppendChild(newExpression(newToken(TokenOp, []rune(":0"), StringType)))
+					return newExprStatement(equalExpr)
+				}(),
+			},
 		},
 		{
-			name:          "0:=0",
-			r:             bytes.NewBuffer([]byte(`0:=0`)),
-			expectedError: true,
+			name: "0:=0",
+			r:    bytes.NewBuffer([]byte(`0:=0`)),
+			expectedStack: []AST{
+				func() AST {
+					equalExpr := newEqualExpr(newExpression(newToken(TokenLit, []rune("0"), StringType)), equalColonOp)
+					equalExpr.AppendChild(newExpression(newToken(TokenOp, []rune("=0"), StringType)))
+					return newExprStatement(equalExpr)
+				}(),
+			},
 		},
 		{
-			name:          "0::0",
-			r:             bytes.NewBuffer([]byte(`0::0`)),
-			expectedError: true,
+			name: "0::0",
+			r:    bytes.NewBuffer([]byte(`0::0`)),
+			expectedStack: []AST{
+				func() AST {
+					equalExpr := newEqualExpr(newExpression(newToken(TokenLit, []rune("0"), StringType)), equalColonOp)
+					equalExpr.AppendChild(newExpression(newToken(TokenOp, []rune(":0"), StringType)))
+					return newExprStatement(equalExpr)
+				}(),
+			},
 		},
 		{
 			name: "section with variable",
@@ -302,6 +332,25 @@ s3 =`)),
 				newExprStatement(noQuotesRegionEQRegion),
 			},
 		},
+		{
+			name: "token seperators [ and  ] in values",
+			r: bytes.NewBuffer([]byte(
+				`[default]
+sepInValue = =:[foo]]bar[
+output = json
+[assumerole]
+sepInValue==:[foo]]bar[
+output = json
+`)),
+			expectedStack: []AST{
+				newCompletedSectionStatement(defaultProfileStmt),
+				newExprStatement(sepInValueExpr),
+				newExprStatement(outputEQExpr),
+				newCompletedSectionStatement(assumeProfileStmt),
+				newExprStatement(sepInValueExpr),
+				newExprStatement(outputEQExpr),
+			},
+		},
 	}
 
 	for i, c := range cases {

internal/ini/testdata/valid/op_sep_in_values

@@ -0,0 +1,6 @@
+[default]
+sepInValue = =:[foo]]bar[
+output = json
+[assumerole]
+sepInValue==:[foo]]bar[
+output = json

internal/ini/testdata/valid/op_sep_in_values_expected

@@ -0,0 +1,10 @@
+{
+  "default": {
+    "sepInValue": "=:[foo]]bar[",
+    "output": "json"
+  },
+  "assumerole": {
+    "sepInValue": "=:[foo]]bar[",
+    "output": "json"
+  }
+}

internal/ini/visitor.go

@@ -50,7 +50,7 @@ func (v *DefaultVisitor) VisitExpr(expr AST) error {
 
 			rhs := children[1]
 
-			if rhs.Root.Type() != TokenLit {
+			if !(rhs.Root.Type() == TokenLit || rhs.Root.Type() == TokenOp || rhs.Root.Type() == TokenSep) {
 				return NewParseError("unexpected token type")
 			}