EOF token is zero length

ruby · Sep 6, 2024 · 16006c2 · 16006c2
1 parent 86c34f9
commit 16006c2
Show file tree

Hide file tree

Showing 7 changed files with 26 additions and 5 deletions.
diff --git a/ext/rbs_extension/lexer.c b/ext/rbs_extension/lexer.c
@@ -116,7 +116,7 @@ token rbsparser_next_token(lexstate *state) {
 yy1:
 	rbs_skip(state);
 #line 144 "ext/rbs_extension/lexer.re"
-	{ return next_token(state, pEOF); }
+	{ return next_eof_token(state); }
 #line 121 "ext/rbs_extension/lexer.c"
 yy2:
 	rbs_skip(state);

diff --git a/ext/rbs_extension/lexer.h b/ext/rbs_extension/lexer.h
@@ -167,6 +167,11 @@ void skipn(lexstate *state, size_t size);
  * */
 token next_token(lexstate *state, enum TokenType type);
 
+/**
+ * Return new token with EOF type.
+ * */
+token next_eof_token(lexstate *state);
+
 token rbsparser_next_token(lexstate *state);
 
 void print_token(token tok);

diff --git a/ext/rbs_extension/lexer.re b/ext/rbs_extension/lexer.re
@@ -141,7 +141,7 @@ token rbsparser_next_token(lexstate *state) {
       skip = ([ \t]+|[\r\n]);
 
       skip     { return next_token(state, tTRIVIA); }
-      "\x00"   { return next_token(state, pEOF); }
+      "\x00"   { return next_eof_token(state); }
       *        { return next_token(state, ErrorToken); }
   */
 }
diff --git a/ext/rbs_extension/lexstate.c b/ext/rbs_extension/lexstate.c
@@ -129,6 +129,22 @@ token next_token(lexstate *state, enum TokenType type) {
   return t;
 }
 
+token next_eof_token(lexstate *state) {
+  if (state->current.byte_pos == RSTRING_LEN(state->string)+1) {
+    // End of String
+    token t;
+    t.type = pEOF;
+    t.range.start = state->start;
+    t.range.end = state->start;
+    state->start = state->current;
+
+    return t;
+  } else {
+    // NULL byte in the middle of the string
+    return next_token(state, pEOF);
+  }
+}
+
 void rbs_skip(lexstate *state) {
   if (!state->last_char) {
     peek(state);

diff --git a/lib/rbs/errors.rb b/lib/rbs/errors.rb
@@ -35,7 +35,7 @@ def detailed_message(highlight: false, **)
       return msg unless location.start_line == location.end_line
 
       indent = " " * location.start_column
-      marker = "^" * (location.end_column - location.start_column)
+      marker = "^" * ([location.end_column - location.start_column, 1].max or raise)
 
       io = StringIO.new
       io.puts msg

diff --git a/test/rbs/cli_test.rb b/test/rbs/cli_test.rb
@@ -912,7 +912,7 @@ def test_parse_method_type
 
       assert_raises(SystemExit) { cli.run(['parse', '--method-type', '-e', '()']) }
       assert_equal [
-        "-e:1:2...1:3: Syntax error: expected a token `pARROW`, token=`` (pEOF) (RBS::ParsingError)",
+        "-e:1:2...1:2: Syntax error: expected a token `pARROW`, token=`` (pEOF) (RBS::ParsingError)",
         "",
         "  ()",
         "    ^"

diff --git a/test/rbs/parser_test.rb b/test/rbs/parser_test.rb
@@ -834,6 +834,6 @@ class Foo[T < Integer] < Bar # Comment
     assert_equal [:tTRIVIA, "\n", 52...53], tokens.shift.then { |t| [t[0], t[1].source, t[1].range] }
     assert_equal [:kEND, 'end', 53...56], tokens.shift.then { |t| [t[0], t[1].source, t[1].range] }
     assert_equal [:tTRIVIA, "\n", 56...57], tokens.shift.then { |t| [t[0], t[1].source, t[1].range] }
-    assert_equal [:pEOF, '', 57...58], tokens.shift.then { |t| [t[0], t[1].source, t[1].range] }
+    assert_equal [:pEOF, '', 57...57], tokens.shift.then { |t| [t[0], t[1].source, t[1].range] }
   end
 end