Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Save/restore the ANTLR RustLexer specific internals. #5624

Merged
merged 2 commits into from
Mar 8, 2023
Merged

Save/restore the ANTLR RustLexer specific internals. #5624

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
bc36f9b
Save/restore the ANTLR RustLexer specific internals.
lkishalmi Mar 7, 2023
68fa9ba
Use just the token type in RustLexerBase
lkishalmi Mar 7, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
27 changes: 25 additions & 2 deletions rust/rust.grammar/src/org/netbeans/modules/rust/grammar/RustLanguageLexer.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,6 @@
import java.util.BitSet;
import java.util.logging.Level;
import java.util.logging.Logger;
import org.antlr.v4.runtime.ANTLRErrorListener;
import org.antlr.v4.runtime.BaseErrorListener;
import org.antlr.v4.runtime.CharStream;
import org.antlr.v4.runtime.Parser;
Expand Down Expand Up @@ -75,7 +74,7 @@ public void reportContextSensitivity(Parser parser, DFA dfa, int line, int charP

}

private static final RustLexer createLexer(CharStream input) {
private static RustLexer createLexer(CharStream input) {
RustLexer lexer = new RustLexer(input);
lexer.removeErrorListeners();
lexer.addErrorListener(new RustLanguageLexerErrorListener());
Expand All @@ -91,4 +90,28 @@ protected Token<RustTokenID> mapToken(org.antlr.v4.runtime.Token antlrToken) {
return token(RustTokenID.from(antlrToken));
}

@Override
public Object state() {
return new LexerState(lexer);
}

private static final class LexerState extends AbstractAntlrLexerBridge.LexerState<RustLexer> {
final Integer lt1;
final Integer lt2;

LexerState(RustLexer lexer) {
super(lexer);

this.lt1 = lexer.lt1;
this.lt2 = lexer.lt2;
}

@Override
public void restore(RustLexer lexer) {
super.restore(lexer);

lexer.lt1 = lt1;
lexer.lt2 = lt2;
}
}
}
10 changes: 5 additions & 5 deletions rust/rust.grammar/src/org/netbeans/modules/rust/grammar/antlr4/RustLexerBase.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -54,8 +54,8 @@ public RustLexerBase(CharStream input) {
super(input);
}

Token lt1;
Token lt2;
public Integer lt1;
public Integer lt2;

@Override
public Token nextToken() {
Expand Down Expand Up @@ -85,7 +85,7 @@ public Token nextToken() {
if (next.getChannel() == Token.DEFAULT_CHANNEL) {
// Keep track of the last token on the default channel.
this.lt2 = this.lt1;
this.lt1 = next;
this.lt1 = next.getType();
}

return next;
Expand Down Expand Up @@ -130,10 +130,10 @@ public boolean floatLiteralPossible() {
if (this.lt1 == null || this.lt2 == null) {
return true;
}
if (this.lt1.getType() != RustLexer.DOT) {
if (this.lt1 != RustLexer.DOT) {
return true;
}
switch (this.lt2.getType()) {
switch (this.lt2) {
case RustLexer.CHAR_LITERAL:
case RustLexer.STRING_LITERAL:
case RustLexer.RAW_STRING_LITERAL:
Expand Down