diff --git a/ext/rbs_extension/lexer.c b/ext/rbs_extension/lexer.c index 81a9e36a4..73984cf84 100644 --- a/ext/rbs_extension/lexer.c +++ b/ext/rbs_extension/lexer.c @@ -116,7 +116,7 @@ token rbsparser_next_token(lexstate *state) { yy1: rbs_skip(state); #line 144 "ext/rbs_extension/lexer.re" - { return next_token(state, pEOF); } + { return next_eof_token(state); } #line 121 "ext/rbs_extension/lexer.c" yy2: rbs_skip(state); diff --git a/ext/rbs_extension/lexer.h b/ext/rbs_extension/lexer.h index c426647f2..55a8a3994 100644 --- a/ext/rbs_extension/lexer.h +++ b/ext/rbs_extension/lexer.h @@ -167,6 +167,11 @@ void skipn(lexstate *state, size_t size); * */ token next_token(lexstate *state, enum TokenType type); +/** + * Return new token with EOF type. + * */ +token next_eof_token(lexstate *state); + token rbsparser_next_token(lexstate *state); void print_token(token tok); diff --git a/ext/rbs_extension/lexer.re b/ext/rbs_extension/lexer.re index cfd84f8e8..aa1b94746 100644 --- a/ext/rbs_extension/lexer.re +++ b/ext/rbs_extension/lexer.re @@ -141,7 +141,7 @@ token rbsparser_next_token(lexstate *state) { skip = ([ \t]+|[\r\n]); skip { return next_token(state, tTRIVIA); } - "\x00" { return next_token(state, pEOF); } + "\x00" { return next_eof_token(state); } * { return next_token(state, ErrorToken); } */ } diff --git a/ext/rbs_extension/lexstate.c b/ext/rbs_extension/lexstate.c index b6cb95a07..ed32fd06a 100644 --- a/ext/rbs_extension/lexstate.c +++ b/ext/rbs_extension/lexstate.c @@ -129,6 +129,22 @@ token next_token(lexstate *state, enum TokenType type) { return t; } +token next_eof_token(lexstate *state) { + if (state->current.byte_pos == RSTRING_LEN(state->string)+1) { + // End of String + token t; + t.type = pEOF; + t.range.start = state->start; + t.range.end = state->start; + state->start = state->current; + + return t; + } else { + // NULL byte in the middle of the string + return next_token(state, pEOF); + } +} + void rbs_skip(lexstate *state) { if (!state->last_char) { peek(state); diff --git a/lib/rbs/errors.rb b/lib/rbs/errors.rb index 32bdba46f..ea0373699 100644 --- a/lib/rbs/errors.rb +++ b/lib/rbs/errors.rb @@ -35,7 +35,7 @@ def detailed_message(highlight: false, **) return msg unless location.start_line == location.end_line indent = " " * location.start_column - marker = "^" * (location.end_column - location.start_column) + marker = "^" * ([location.end_column - location.start_column, 1].max or raise) io = StringIO.new io.puts msg diff --git a/test/rbs/cli_test.rb b/test/rbs/cli_test.rb index 815c624c3..f8701045c 100644 --- a/test/rbs/cli_test.rb +++ b/test/rbs/cli_test.rb @@ -912,7 +912,7 @@ def test_parse_method_type assert_raises(SystemExit) { cli.run(['parse', '--method-type', '-e', '()']) } assert_equal [ - "-e:1:2...1:3: Syntax error: expected a token `pARROW`, token=`` (pEOF) (RBS::ParsingError)", + "-e:1:2...1:2: Syntax error: expected a token `pARROW`, token=`` (pEOF) (RBS::ParsingError)", "", " ()", " ^" diff --git a/test/rbs/parser_test.rb b/test/rbs/parser_test.rb index ea0e9bcd2..adfde5f74 100644 --- a/test/rbs/parser_test.rb +++ b/test/rbs/parser_test.rb @@ -834,6 +834,6 @@ class Foo[T < Integer] < Bar # Comment assert_equal [:tTRIVIA, "\n", 52...53], tokens.shift.then { |t| [t[0], t[1].source, t[1].range] } assert_equal [:kEND, 'end', 53...56], tokens.shift.then { |t| [t[0], t[1].source, t[1].range] } assert_equal [:tTRIVIA, "\n", 56...57], tokens.shift.then { |t| [t[0], t[1].source, t[1].range] } - assert_equal [:pEOF, '', 57...58], tokens.shift.then { |t| [t[0], t[1].source, t[1].range] } + assert_equal [:pEOF, '', 57...57], tokens.shift.then { |t| [t[0], t[1].source, t[1].range] } end end