From 77c1a05b3b5de41ec8da937a5515316210ee3d2a Mon Sep 17 00:00:00 2001 From: shes50103 Date: Sat, 23 Jun 2018 19:43:55 +0800 Subject: [PATCH 1/2] parse underscore --- compiler/lexer/lexer.go | 2 ++ compiler/parser/data_type_parsing.go | 31 ++++++++++++++++++++++++ compiler/parser/parser.go | 1 + compiler/parser/precedence/precedence.go | 1 + compiler/token/token.go | 1 + 5 files changed, 36 insertions(+) diff --git a/compiler/lexer/lexer.go b/compiler/lexer/lexer.go index a41d96133..505946bc8 100644 --- a/compiler/lexer/lexer.go +++ b/compiler/lexer/lexer.go @@ -110,6 +110,8 @@ func (l *Lexer) NextToken() token.Token { } case ';': tok = newToken(token.Semicolon, l.ch, l.line) + case '_': + tok = newToken(token.UnderScore, l.ch, l.line) case '(': tok = newToken(token.LParen, l.ch, l.line) case ')': diff --git a/compiler/parser/data_type_parsing.go b/compiler/parser/data_type_parsing.go index d8f953c19..746aec96b 100644 --- a/compiler/parser/data_type_parsing.go +++ b/compiler/parser/data_type_parsing.go @@ -23,6 +23,34 @@ func (p *Parser) parseIntegerLiteral() ast.Expression { return lit } +func (p *Parser) parseUnderScoreLiteral(integerPart ast.Expression) ast.Expression { + p.nextToken() + + lit := &ast.IntegerLiteral{BaseNode: &ast.BaseNode{Token: p.curToken}} + + firstValue, err := strconv.ParseInt(integerPart.String(), 0, 64) + + + len := len(integerPart.String()) + + for ; len > 1; len-- { + firstValue *= 10 + } + + secondValue, err := strconv.ParseInt(lit.TokenLiteral(), 0, 64) + + if err != nil { + p.error = errors.NewTypeParsingError(lit.TokenLiteral(), "integer", p.curToken.Line) + return nil + } + + lit.Value = int(firstValue + secondValue) + + + + return lit +} + func (p *Parser) parseFloatLiteral(integerPart ast.Expression) ast.Expression { // Get the fractional part of the token p.nextToken() @@ -38,7 +66,10 @@ func (p *Parser) parseFloatLiteral(integerPart ast.Expression) ast.Expression { p.error = errors.NewTypeParsingError(lit.TokenLiteral(), "float", p.curToken.Line) return nil } + fmt.Println("lit1:", lit) lit.Value = float64(value) + fmt.Println("lit2:", lit) + return lit } diff --git a/compiler/parser/parser.go b/compiler/parser/parser.go index a60de2fc8..a8abc4633 100644 --- a/compiler/parser/parser.go +++ b/compiler/parser/parser.go @@ -102,6 +102,7 @@ func New(l *lexer.Lexer) *Parser { p.registerInfix(token.Assign, p.parseAssignExpression) p.registerInfix(token.Range, p.parseRangeExpression) p.registerInfix(token.Dot, p.parseCallExpressionWithReceiver) + p.registerInfix(token.UnderScore, p.parseUnderScoreLiteral) p.registerInfix(token.LParen, p.parseCallExpressionWithoutReceiver) p.registerInfix(token.LBracket, p.parseIndexExpression) p.registerInfix(token.Colon, p.parseArgumentPairExpression) diff --git a/compiler/parser/precedence/precedence.go b/compiler/parser/precedence/precedence.go index 5928d02b7..bf3613cb4 100644 --- a/compiler/parser/precedence/precedence.go +++ b/compiler/parser/precedence/precedence.go @@ -22,6 +22,7 @@ const ( // LookupTable maps token to its corresponding precedence var LookupTable = map[token.Type]int{ + token.UnderScore: Equals, token.Eq: Equals, token.NotEq: Equals, token.Match: Compare, diff --git a/compiler/token/token.go b/compiler/token/token.go index d9ac90e41..1eaacc97a 100644 --- a/compiler/token/token.go +++ b/compiler/token/token.go @@ -23,6 +23,7 @@ const ( String = "STRING" Comment = "COMMENT" + UnderScore = "_" Assign = "=" Plus = "+" PlusEq = "+=" From 48682b2e3c181bb70129dab27e3e15b9d1612c66 Mon Sep 17 00:00:00 2001 From: shes50103 Date: Thu, 30 Aug 2018 20:22:52 +0800 Subject: [PATCH 2/2] test --- compiler/parser/data_type_parsing.go | 20 ++++++++++------ compiler/parser/expression_parsing.go | 6 +++++ compiler/parser/parser.go | 2 ++ compiler/parser/precedence/precedence.go | 2 +- compiler/token/token.go | 30 ++++++++++++------------ 5 files changed, 37 insertions(+), 23 deletions(-) diff --git a/compiler/parser/data_type_parsing.go b/compiler/parser/data_type_parsing.go index 746aec96b..ca7504082 100644 --- a/compiler/parser/data_type_parsing.go +++ b/compiler/parser/data_type_parsing.go @@ -26,14 +26,15 @@ func (p *Parser) parseIntegerLiteral() ast.Expression { func (p *Parser) parseUnderScoreLiteral(integerPart ast.Expression) ast.Expression { p.nextToken() + fmt.Println("!!!") + lit := &ast.IntegerLiteral{BaseNode: &ast.BaseNode{Token: p.curToken}} firstValue, err := strconv.ParseInt(integerPart.String(), 0, 64) + len := len(p.curToken.Literal) - len := len(integerPart.String()) - - for ; len > 1; len-- { + for ; len > 0; len-- { firstValue *= 10 } @@ -46,7 +47,7 @@ func (p *Parser) parseUnderScoreLiteral(integerPart ast.Expression) ast.Expressi lit.Value = int(firstValue + secondValue) - + lit.Token.Literal = strconv.Itoa(lit.Value) return lit } @@ -54,10 +55,17 @@ func (p *Parser) parseUnderScoreLiteral(integerPart ast.Expression) ast.Expressi func (p *Parser) parseFloatLiteral(integerPart ast.Expression) ast.Expression { // Get the fractional part of the token p.nextToken() + lit2 := p.parseIntegerLiteral() + + for p.peekTokenIs(token.UnderScore) { + p.nextToken() + + lit2 = p.parseUnderScoreLiteral(lit2) + } floatTok := token.Token{ Type: token.Float, - Literal: fmt.Sprintf("%s.%s", integerPart.String(), p.curToken.Literal), + Literal: fmt.Sprintf("%s.%s", integerPart.String(), lit2.String()), Line: p.curToken.Line, } lit := &ast.FloatLiteral{BaseNode: &ast.BaseNode{Token: floatTok}} @@ -66,9 +74,7 @@ func (p *Parser) parseFloatLiteral(integerPart ast.Expression) ast.Expression { p.error = errors.NewTypeParsingError(lit.TokenLiteral(), "float", p.curToken.Line) return nil } - fmt.Println("lit1:", lit) lit.Value = float64(value) - fmt.Println("lit2:", lit) return lit } diff --git a/compiler/parser/expression_parsing.go b/compiler/parser/expression_parsing.go index b556140e1..06f26b887 100644 --- a/compiler/parser/expression_parsing.go +++ b/compiler/parser/expression_parsing.go @@ -192,6 +192,12 @@ func (p *Parser) parseExpression(precedence int) ast.Expression { } leftExp = infixFn(leftExp) + + for p.peekTokenIs(token.UnderScore) { + p.nextToken() + leftExp = infixFn(leftExp) + } + } if p.peekTokenIs(token.Semicolon) { diff --git a/compiler/parser/parser.go b/compiler/parser/parser.go index a8abc4633..28d2e7c4e 100644 --- a/compiler/parser/parser.go +++ b/compiler/parser/parser.go @@ -114,6 +114,8 @@ func New(l *lexer.Lexer) *Parser { // ParseProgram update program statements and return program func (p *Parser) ParseProgram() (program *ast.Program, err *errors.Error) { + fmt.Println("aa") + defer func() { if recover() != nil { err = p.error diff --git a/compiler/parser/precedence/precedence.go b/compiler/parser/precedence/precedence.go index bf3613cb4..56467181c 100644 --- a/compiler/parser/precedence/precedence.go +++ b/compiler/parser/precedence/precedence.go @@ -22,7 +22,7 @@ const ( // LookupTable maps token to its corresponding precedence var LookupTable = map[token.Type]int{ - token.UnderScore: Equals, + token.UnderScore: Equals, token.Eq: Equals, token.NotEq: Equals, token.Match: Compare, diff --git a/compiler/token/token.go b/compiler/token/token.go index 1eaacc97a..0eca24d71 100644 --- a/compiler/token/token.go +++ b/compiler/token/token.go @@ -23,21 +23,21 @@ const ( String = "STRING" Comment = "COMMENT" - UnderScore = "_" - Assign = "=" - Plus = "+" - PlusEq = "+=" - Minus = "-" - MinusEq = "-=" - Bang = "!" - Asterisk = "*" - Pow = "**" - Slash = "/" - Dot = "." - And = "&&" - Or = "||" - OrEq = "||=" - Modulo = "%" + UnderScore = "_" + Assign = "=" + Plus = "+" + PlusEq = "+=" + Minus = "-" + MinusEq = "-=" + Bang = "!" + Asterisk = "*" + Pow = "**" + Slash = "/" + Dot = "." + And = "&&" + Or = "||" + OrEq = "||=" + Modulo = "%" Match = "=~" LT = "<"