Add support for $ symbol in identifiers

expr-lang · Mar 4, 2020 · 13d13fd · 13d13fd
1 parent 0cf3b1f
commit 13d13fd
Show file tree

Hide file tree

Showing 4 changed files with 22 additions and 26 deletions.
diff --git a/parser/lexer/lexer_test.go b/parser/lexer/lexer_test.go
@@ -88,6 +88,15 @@ var lexTests = []lexTest{
 			{Kind: EOF},
 		},
 	},
+	{
+		`$i _0 früh`,
+		[]Token{
+			{Kind: Identifier, Value: "$i"},
+			{Kind: Identifier, Value: "_0"},
+			{Kind: Identifier, Value: "früh"},
+			{Kind: EOF},
+		},
+	},
 }
 
 func compareTokens(i1, i2 []Token) bool {

diff --git a/parser/lexer/state.go b/parser/lexer/state.go
@@ -11,7 +11,7 @@ func root(l *lexer) stateFn {
 	case r == eof:
 		l.emitEOF()
 		return nil
-	case isSpace(r):
+	case IsSpace(r):
 		l.ignore()
 		return root
 	case r == '\'' || r == '"':
@@ -36,7 +36,7 @@ func root(l *lexer) stateFn {
 	case r == '.':
 		l.backup()
 		return dot
-	case isAlphaNumeric(r):
+	case IsAlphaNumeric(r):
 		l.backup()
 		return identifier
 	default:
@@ -67,18 +67,14 @@ func (l *lexer) scanNumber() bool {
 		}
 	}
 	l.acceptRun(digits)
-	end := l.end
-	loc := l.loc
-	prev := l.prev
+	loc, prev, end := l.loc, l.prev, l.end
 	if l.accept(".") {
 		// Lookup for .. operator: if after dot there is another dot (1..2), it maybe a range operator.
 		if l.peek() == '.' {
 			// We can't backup() here, as it would require two backups,
 			// and backup() func supports only one for now. So, save and
 			// restore it here.
-			l.end = end
-			l.loc = loc
-			l.prev = prev
+			l.loc, l.prev, l.end = loc, prev, end
 			return true
 		}
 		l.acceptRun(digits)
@@ -88,7 +84,7 @@ func (l *lexer) scanNumber() bool {
 		l.acceptRun(digits)
 	}
 	// Next thing mustn't be alphanumeric.
-	if isAlphaNumeric(l.peek()) {
+	if IsAlphaNumeric(l.peek()) {
 		l.next()
 		return false
 	}
@@ -110,7 +106,7 @@ func identifier(l *lexer) stateFn {
 loop:
 	for {
 		switch r := l.next(); {
-		case isAlphaNumeric(r):
+		case IsAlphaNumeric(r):
 			// absorb
 		default:
 			l.backup()

diff --git a/parser/lexer/utils.go b/parser/lexer/utils.go
@@ -7,16 +7,16 @@ import (
 	"unicode/utf8"
 )
 
-func isSpace(r rune) bool {
+func IsSpace(r rune) bool {
 	return unicode.IsSpace(r)
 }
 
-func isAlphaNumeric(r rune) bool {
-	return isAlphabetic(r) || unicode.IsDigit(r)
+func IsAlphaNumeric(r rune) bool {
+	return IsAlphabetic(r) || unicode.IsDigit(r)
 }
 
-func isAlphabetic(r rune) bool {
-	return r == '_' || unicode.IsLetter(r)
+func IsAlphabetic(r rune) bool {
+	return r == '_' || r == '$' || unicode.IsLetter(r)
 }
 
 var (

diff --git a/parser/parser.go b/parser/parser.go
@@ -5,7 +5,6 @@ import (
 	"regexp"
 	"strconv"
 	"strings"
-	"unicode"
 	"unicode/utf8"
 
 	. "github.com/antonmedv/expr/ast"
@@ -528,25 +527,17 @@ func isValidIdentifier(str string) bool {
 		return false
 	}
 	h, w := utf8.DecodeRuneInString(str)
-	if !isAlphabetic(h) {
+	if !IsAlphabetic(h) {
 		return false
 	}
 	for _, r := range str[w:] {
-		if !isAlphaNumeric(r) {
+		if !IsAlphaNumeric(r) {
 			return false
 		}
 	}
 	return true
 }
 
-func isAlphaNumeric(r rune) bool {
-	return isAlphabetic(r) || unicode.IsDigit(r)
-}
-
-func isAlphabetic(r rune) bool {
-	return r == '_' || unicode.IsLetter(r)
-}
-
 func (p *parser) parseArguments() []Node {
 	p.expect(Bracket, "(")
 	nodes := make([]Node, 0)