Skip to content

Commit

Permalink
Merge pull request #5765 from matthiasblaesing/js_improvements3
Browse files Browse the repository at this point in the history 
JS: Improve JSX lexing (support fragment syntax and embedded comments)
  • Loading branch information
@matthiasblaesing
matthiasblaesing authored Apr 7, 2023
2 parents 7c9c4bc + 1eb0cfb commit c7847c6
Show file tree
Hide file tree
Showing 11 changed files with 368 additions and 291 deletions.
559 changes: 278 additions & 281 deletions webcommon/javascript2.lexer/src/org/netbeans/modules/javascript2/lexer/JsColoringLexer.java
View file

Large diffs are not rendered by default.

1 change: 1 addition & 0 deletions webcommon/javascript2.lexer/test/unit/data/testfiles/jsx/jsxComment.js
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
const a = <table>{/* Test */ /* Test */ /* Test */}{ a = 3 }</table>;
28 changes: 28 additions & 0 deletions webcommon/javascript2.lexer/test/unit/data/testfiles/jsx/jsxComment.js.tokens.txt
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,28 @@
<Unnamed test>
KEYWORD_CONST "const", la=1, st=LexerState{canFollowLiteral=false, canFollowKeyword=true, braceBalances=[], jsxBalances=[]}
WHITESPACE " ", la=1, st=LexerState{canFollowLiteral=false, canFollowKeyword=true, braceBalances=[], jsxBalances=[]}
IDENTIFIER "a", la=1, st=LexerState{canFollowLiteral=false, canFollowKeyword=true, braceBalances=[], jsxBalances=[]}
WHITESPACE " ", la=1, st=LexerState{canFollowLiteral=false, canFollowKeyword=true, braceBalances=[], jsxBalances=[]}
OPERATOR_ASSIGNMENT "=", la=1, st=LexerState{canFollowLiteral=true, canFollowKeyword=true, braceBalances=[], jsxBalances=[]}
WHITESPACE " ", la=1, st=LexerState{canFollowLiteral=true, canFollowKeyword=true, braceBalances=[], jsxBalances=[]}
JSX_TEXT "<table>", la=1, st=LexerState{canFollowLiteral=false, canFollowKeyword=true, braceBalances=[], jsxBalances=[1]}
JSX_EXP_BEGIN "{", st=LexerState{canFollowLiteral=false, canFollowKeyword=true, braceBalances=[JSX], jsxBalances=[1]}
BLOCK_COMMENT "/* Test */", st=LexerState{canFollowLiteral=false, canFollowKeyword=true, braceBalances=[JSX], jsxBalances=[1]}
WHITESPACE " ", la=1, st=LexerState{canFollowLiteral=false, canFollowKeyword=true, braceBalances=[JSX], jsxBalances=[1]}
BLOCK_COMMENT "/* Test */", st=LexerState{canFollowLiteral=false, canFollowKeyword=true, braceBalances=[JSX], jsxBalances=[1]}
WHITESPACE " ", la=1, st=LexerState{canFollowLiteral=false, canFollowKeyword=true, braceBalances=[JSX], jsxBalances=[1]}
BLOCK_COMMENT "/* Test */", st=LexerState{canFollowLiteral=false, canFollowKeyword=true, braceBalances=[JSX], jsxBalances=[1]}
JSX_EXP_END "}", st=LexerState{canFollowLiteral=false, canFollowKeyword=true, braceBalances=[], jsxBalances=[1]}
JSX_EXP_BEGIN "{", st=LexerState{canFollowLiteral=false, canFollowKeyword=true, braceBalances=[JSX], jsxBalances=[1]}
WHITESPACE " ", la=1, st=LexerState{canFollowLiteral=false, canFollowKeyword=true, braceBalances=[JSX], jsxBalances=[1]}
IDENTIFIER "a", la=1, st=LexerState{canFollowLiteral=false, canFollowKeyword=true, braceBalances=[JSX], jsxBalances=[1]}
WHITESPACE " ", la=1, st=LexerState{canFollowLiteral=false, canFollowKeyword=true, braceBalances=[JSX], jsxBalances=[1]}
OPERATOR_ASSIGNMENT "=", la=1, st=LexerState{canFollowLiteral=true, canFollowKeyword=true, braceBalances=[JSX], jsxBalances=[1]}
WHITESPACE " ", la=1, st=LexerState{canFollowLiteral=true, canFollowKeyword=true, braceBalances=[JSX], jsxBalances=[1]}
NUMBER "3", la=1, st=LexerState{canFollowLiteral=false, canFollowKeyword=true, braceBalances=[JSX], jsxBalances=[1]}
WHITESPACE " ", la=1, st=LexerState{canFollowLiteral=false, canFollowKeyword=true, braceBalances=[JSX], jsxBalances=[1]}
JSX_EXP_END "}", st=LexerState{canFollowLiteral=false, canFollowKeyword=true, braceBalances=[], jsxBalances=[1]}
JSX_TEXT "</table>", st=LexerState{canFollowLiteral=false, canFollowKeyword=true, braceBalances=[], jsxBalances=[]}
OPERATOR_SEMICOLON ";", st=LexerState{canFollowLiteral=true, canFollowKeyword=true, braceBalances=[], jsxBalances=[]}
----- EOF -----

1 change: 1 addition & 0 deletions webcommon/javascript2.lexer/test/unit/data/testfiles/jsx/jsxFragment.js
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
const a = <></>;
11 changes: 11 additions & 0 deletions webcommon/javascript2.lexer/test/unit/data/testfiles/jsx/jsxFragment.js.tokens.txt
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,11 @@
<Unnamed test>
KEYWORD_CONST "const", la=1, st=LexerState{canFollowLiteral=false, canFollowKeyword=true, braceBalances=[], jsxBalances=[]}
WHITESPACE " ", la=1, st=LexerState{canFollowLiteral=false, canFollowKeyword=true, braceBalances=[], jsxBalances=[]}
IDENTIFIER "a", la=1, st=LexerState{canFollowLiteral=false, canFollowKeyword=true, braceBalances=[], jsxBalances=[]}
WHITESPACE " ", la=1, st=LexerState{canFollowLiteral=false, canFollowKeyword=true, braceBalances=[], jsxBalances=[]}
OPERATOR_ASSIGNMENT "=", la=1, st=LexerState{canFollowLiteral=true, canFollowKeyword=true, braceBalances=[], jsxBalances=[]}
WHITESPACE " ", la=1, st=LexerState{canFollowLiteral=true, canFollowKeyword=true, braceBalances=[], jsxBalances=[]}
JSX_TEXT "<></>", st=LexerState{canFollowLiteral=false, canFollowKeyword=true, braceBalances=[], jsxBalances=[]}
OPERATOR_SEMICOLON ";", st=LexerState{canFollowLiteral=true, canFollowKeyword=true, braceBalances=[], jsxBalances=[]}
----- EOF -----

10 changes: 10 additions & 0 deletions ...avascript2.lexer/test/unit/src/org/netbeans/modules/javascript2/lexer/JsLexerJsxTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -93,4 +93,14 @@ public void testIncLess() throws Exception {
LexerTestUtilities.checkTokenDump(this, "testfiles/jsx/incLess.js",
JsTokenId.javascriptLanguage());
}

public void testJsxFragment() throws Exception {
LexerTestUtilities.checkTokenDump(this, "testfiles/jsx/jsxFragment.js",
JsTokenId.javascriptLanguage());
}

public void testJsxComment() throws Exception {
LexerTestUtilities.checkTokenDump(this, "testfiles/jsx/jsxComment.js",
JsTokenId.javascriptLanguage());
}
}
2 changes: 1 addition & 1 deletion webcommon/javascript2.lexer/tools/JsColoringScanner.flex
View file
Original file line number Diff line number Diff line change
Expand Up @@ -712,7 +712,7 @@ RegexpFirstCharacter = [^*\x5b/\r\n\\] | {RegexpBackslashSequence} | {RegexpClas
}

<JSX> {
"/>" | "</"{JSXCharacter}+">"
"/>" | "</"{JSXCharacter}*">"
{
Integer balance = jsxBalances.isEmpty() ? 0 : jsxBalances.pop() - 1;
if (balance <= 0) {
Expand Down
18 changes: 17 additions & 1 deletion webcommon/libs.nashorn/src/com/oracle/js/parser/Parser.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5896,7 +5896,7 @@ private Expression jsxElement(long startToken) {
}
break;
case LBRACE:
if (T(k + 1) == TokenType.RBRACE) {
if (! lookaheadIsJsxAssignment()) {
next();
next();
} else {
Expand All @@ -5912,6 +5912,19 @@ private Expression jsxElement(long startToken) {
return new JsxElementNode(name, attributes, children, realStart, finish);
}

private boolean lookaheadIsJsxAssignment() {
assert type == LBRACE;
for (int i = 1;; i++) {
TokenType t = T(k + i);
switch (t) {
case COMMENT:
continue;
default:
return t != TokenType.RBRACE;
}
}
}

private Expression jsxAttribute() {
Expression attribute = null;
if (type == LBRACE) {
Expand Down Expand Up @@ -5958,6 +5971,9 @@ private Expression jsxAttribute() {
}

private String jsxElementName() {
if (type == TokenType.JSX_ELEM_END) {
return "";
}
expectDontAdvance(TokenType.JSX_IDENTIFIER);
StringBuilder name = new StringBuilder((String) getValue(token));
next();
Expand Down
14 changes: 12 additions & 2 deletions webcommon/libs.nashorn/test/unit/src/com/oracle/js/parser/DumpingVisitor.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@
import com.oracle.js.parser.ir.FunctionNode;
import com.oracle.js.parser.ir.IdentNode;
import com.oracle.js.parser.ir.IndexNode;
import com.oracle.js.parser.ir.JsxElementNode;
import com.oracle.js.parser.ir.LexicalContext;
import com.oracle.js.parser.ir.LiteralNode;
import com.oracle.js.parser.ir.Node;
Expand All @@ -51,7 +52,9 @@ protected Node leaveDefault(Node node) {
@Override
protected boolean enterDefault(Node node) {
if (node instanceof IdentNode) {
System.out.println(indent() + node.getClass().getName() + " [" + ((IdentNode) node).getName() + "]");
System.out.printf("%s%s [%d-%d, name=%s]%n", indent(),
node.getClass().getName(), node.getStart(),
node.getFinish(), ((IdentNode) node).getName());
} else if (node instanceof LiteralNode) {
System.out.println(indent() + node.getClass().getName() + " [" + ((LiteralNode) node).getValue() + "]");
} else if (node instanceof FunctionNode) {
Expand Down Expand Up @@ -89,8 +92,15 @@ protected boolean enterDefault(Node node) {
} else if (node instanceof PropertyNode) {
PropertyNode pn = (PropertyNode) node;
System.out.println(indent() + node.getClass().getName() + " [static=" + pn.isStatic() + "]");
} else if (node instanceof JsxElementNode) {
JsxElementNode jen = (JsxElementNode) node;
System.out.printf("%s%s [%d-%d, name=%s]%n", indent(),
node.getClass().getName(), node.getStart(),
node.getFinish(), jen.getName());
} else {
System.out.println(indent() + node.getClass().getName());
System.out.printf("%s%s [%d-%d]%n", indent(),
node.getClass().getName(), node.getStart(),
node.getFinish());
}
indent += INDENT_PER_LEVEL;
return super.enterDefault(node);
Expand Down
7 changes: 1 addition & 6 deletions webcommon/libs.nashorn/test/unit/src/com/oracle/js/parser/ManualTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -38,12 +38,7 @@ public static void main(String[] args) {
// Source source = Source.sourceFor("dummy.js", "var a = import('test');");
// Source source = Source.sourceFor("dummy.js", "try {} catch (e) {}");
// Source source = Source.sourceFor("dummy.js", "function a() {}; async function b() {}; class x { y(){} async z(){} }");
Source source = Source.sourceFor("dummy.js", "class Polygon {\n" +
" constructor(height, width) {\n" +
" this.height = height;\n" +
" this.width = width;\n" +
" }\n" +
"} ");
Source source = Source.sourceFor("dummy.js", "const a = <table>{/* Test */ /* Test */ /* Test */}{ a = 3 }</table>");
ScriptEnvironment.Builder builder = ScriptEnvironment.builder();
Parser parser = new Parser(
builder.emptyStatements(true).ecmacriptEdition(13).jsx(true).build(),
Expand Down
8 changes: 8 additions & 0 deletions webcommon/libs.nashorn/test/unit/src/com/oracle/js/parser/ParserTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -266,6 +266,14 @@ public void testFor() {
assertParses(13, "async function dummy() { for await (const num of foo()) {}}");
}

@Test
public void testJsx() {
// JSX Fragments are parsed as JSX Elements without name and attributes
assertParses(13, "const a = <></>;");
assertParses(13, "const a = <table style={{ border: '2px solid black', borderRadius: '.5em'}}></table>");
assertParses(13, "const a = <table>{/* Test */ /* Test */ /* Test */}{ a = 3 }</table>");
}

private Predicate<Node> functionNodeWithName(String name) {
return n -> n instanceof FunctionNode && name.equals(((FunctionNode) n).getName());
}
Expand Down

0 comments on commit c7847c6

Please sign in to comment.