Codegen for stateful lexer.

This uses a fairly naive recursive solution but even with that caveat gives
significant gains. Roughly 10x performance with constant GC overhead per
tokenisation call. The latter is possible because the tokens return
slices into the input substring.

lexer/lexer.go

@@ -63,7 +63,7 @@ func Must(def Definition, err error) Definition {
 
 // ConsumeAll reads all tokens from a Lexer.
 func ConsumeAll(lexer Lexer) ([]Token, error) {
-	tokens := []Token{}
+	tokens := make([]Token, 0, 1024)
 	for {
 		token, err := lexer.Next()
 		if err != nil {