diff --git a/lexer_test.go b/lexer_test.go index 08cf695..376467c 100644 --- a/lexer_test.go +++ b/lexer_test.go @@ -252,3 +252,43 @@ func TestPartialLexer(x *testing.T) { t.AssertNil(lexer.CompileDFA()) scan(lexer) } + +func TestRegression(t *testing.T) { + skip := func(*Scanner, *machines.Match) (interface{}, error) { + return nil, nil + } + token := func(id int, name string) Action { + return func(s *Scanner, m *machines.Match) (interface{}, error) { + return string(m.Bytes), nil + } + } + + data := "true" // This input fails. + // data := "true " // this with a trailing space does not. + + lexer := NewLexer() + lexer.Add([]byte("true"), token(0, "TRUE")) + lexer.Add([]byte("( |\t|\n|\r)+"), skip) + + if err := lexer.CompileDFA(); err != nil { + t.Fatal(err) + } + + var scanner *Scanner + + scanner, err := lexer.Scanner([]byte(data)) + if err != nil { + t.Fatal(err) + } + + found := 0 + tok, err, eos := scanner.Next() + for ; !eos; tok, err, eos = scanner.Next() { + fmt.Printf("Token: %v\n", tok) + found++ + } + if found != 1 { + t.Errorf("Expected exactly 1 tokens got %v, ===\nErr: %v\nEOS: %v\nTC: %d\n", found, err, eos, scanner.TC) + + } +} diff --git a/machines/dfa_machine.go b/machines/dfa_machine.go index ca814ec..c4b7dde 100644 --- a/machines/dfa_machine.go +++ b/machines/dfa_machine.go @@ -79,6 +79,19 @@ func DFALexerEngine(startState, errorState int, trans DFATrans, accepting DFAAcc if match, has := accepting[state]; has { matchID = match matchTC = tc + startLC := lineCols[startTC] + endLC := lineCols[matchTC-1] + match := &Match{ + PC: matchID, + TC: startTC, + StartLine: startLC.line, + StartColumn: startLC.col, + EndLine: endLC.line, + EndColumn: endLC.col, + Bytes: text[startTC:matchTC], + } + matchID = -1 + return tc, match, nil, scan } if matchTC != len(text) && startTC >= len(text) { // the user has moved us farther than the text. Assume that was