From 0b384e32b39117a8da2829d8543e7474338ddebe Mon Sep 17 00:00:00 2001 From: Liam Miller-Cushon Date: Thu, 30 May 2024 08:56:36 -0700 Subject: [PATCH] Prepare g-j-f for some internal javac API changes * Refactor comment handling in tokenization to avoid extending javac's internal Comment. This prepares for upcoming JDK changes to the Comment API, which break the current subclass approach by changing the return type of `Comment#getPos`, see also similar changes to Error Prone in unknown commit * `JCCompilationUnit#getImports` no longer returns `JCImport`, as part of the changes for modular imports. Supporting modular imports is future work, this change just avoids a crash formatting the existing syntax on the latest JDK versions. PiperOrigin-RevId: 638669955 --- .../googlejavaformat/java/JavacTokens.java | 77 ++++++++----------- .../java/RemoveUnusedImports.java | 8 +- 2 files changed, 36 insertions(+), 49 deletions(-) diff --git a/core/src/main/java/com/google/googlejavaformat/java/JavacTokens.java b/core/src/main/java/com/google/googlejavaformat/java/JavacTokens.java index 6daac33b9..793c6220c 100644 --- a/core/src/main/java/com/google/googlejavaformat/java/JavacTokens.java +++ b/core/src/main/java/com/google/googlejavaformat/java/JavacTokens.java @@ -15,9 +15,9 @@ package com.google.googlejavaformat.java; import static com.google.common.base.Preconditions.checkArgument; +import static com.google.common.collect.ImmutableList.toImmutableList; import com.google.common.collect.ImmutableList; -import com.google.common.collect.Lists; import com.sun.tools.javac.parser.JavaTokenizer; import com.sun.tools.javac.parser.Scanner; import com.sun.tools.javac.parser.ScannerFactory; @@ -25,8 +25,9 @@ import com.sun.tools.javac.parser.Tokens.Comment.CommentStyle; import com.sun.tools.javac.parser.Tokens.Token; import com.sun.tools.javac.parser.Tokens.TokenKind; -import com.sun.tools.javac.parser.UnicodeReader; import com.sun.tools.javac.util.Context; +import java.util.HashMap; +import java.util.Map; import java.util.Set; /** A wrapper around javac's lexer. */ @@ -79,8 +80,8 @@ public static ImmutableList getTokens( } ScannerFactory fac = ScannerFactory.instance(context); char[] buffer = (source + EOF_COMMENT).toCharArray(); - Scanner scanner = - new AccessibleScanner(fac, new CommentSavingTokenizer(fac, buffer, buffer.length)); + CommentSavingTokenizer tokenizer = new CommentSavingTokenizer(fac, buffer, buffer.length); + Scanner scanner = new AccessibleScanner(fac, tokenizer); ImmutableList.Builder tokens = ImmutableList.builder(); int end = source.length(); int last = 0; @@ -88,7 +89,7 @@ public static ImmutableList getTokens( scanner.nextToken(); Token t = scanner.token(); if (t.comments != null) { - for (Comment c : Lists.reverse(t.comments)) { + for (CommentWithTextAndPosition c : getComments(t, tokenizer.comments())) { if (last < c.getSourcePos(0)) { tokens.add(new RawTok(null, null, last, c.getSourcePos(0))); } @@ -120,17 +121,36 @@ public static ImmutableList getTokens( return tokens.build(); } + private static ImmutableList getComments( + Token token, Map comments) { + if (token.comments == null) { + return ImmutableList.of(); + } + // javac stores the comments in reverse declaration order + return token.comments.stream().map(comments::get).collect(toImmutableList()).reverse(); + } + /** A {@link JavaTokenizer} that saves comments. */ static class CommentSavingTokenizer extends JavaTokenizer { + + private final Map comments = new HashMap<>(); + CommentSavingTokenizer(ScannerFactory fac, char[] buffer, int length) { super(fac, buffer, length); } + Map comments() { + return comments; + } + @Override protected Comment processComment(int pos, int endPos, CommentStyle style) { char[] buf = getRawCharactersReflectively(pos, endPos); - return new CommentWithTextAndPosition( - pos, endPos, new AccessibleReader(fac, buf, buf.length), style); + Comment comment = super.processComment(pos, endPos, style); + CommentWithTextAndPosition commentWithTextAndPosition = + new CommentWithTextAndPosition(pos, endPos, new String(buf)); + comments.put(comment, commentWithTextAndPosition); + return comment; } private char[] getRawCharactersReflectively(int beginIndex, int endIndex) { @@ -153,21 +173,16 @@ private char[] getRawCharactersReflectively(int beginIndex, int endIndex) { } /** A {@link Comment} that saves its text and start position. */ - static class CommentWithTextAndPosition implements Comment { + static class CommentWithTextAndPosition { private final int pos; private final int endPos; - private final AccessibleReader reader; - private final CommentStyle style; - - private String text = null; + private final String text; - public CommentWithTextAndPosition( - int pos, int endPos, AccessibleReader reader, CommentStyle style) { + public CommentWithTextAndPosition(int pos, int endPos, String text) { this.pos = pos; this.endPos = endPos; - this.reader = reader; - this.style = style; + this.text = text; } /** @@ -176,7 +191,6 @@ public CommentWithTextAndPosition( *

The handling of javadoc comments in javac has more logic to skip over leading whitespace * and '*' characters when indexing into doc comments, but we don't need any of that. */ - @Override public int getSourcePos(int index) { checkArgument( 0 <= index && index < (endPos - pos), @@ -186,49 +200,22 @@ public int getSourcePos(int index) { return pos + index; } - @Override - public CommentStyle getStyle() { - return style; - } - - @Override public String getText() { - String text = this.text; - if (text == null) { - this.text = text = new String(reader.getRawCharacters()); - } return text; } - /** - * We don't care about {@code @deprecated} javadoc tags (see the DepAnn check). - * - * @return false - */ - @Override - public boolean isDeprecated() { - return false; - } - @Override public String toString() { return String.format("Comment: '%s'", getText()); } } - // Scanner(ScannerFactory, JavaTokenizer) is package-private + // Scanner(ScannerFactory, JavaTokenizer) is protected static class AccessibleScanner extends Scanner { protected AccessibleScanner(ScannerFactory fac, JavaTokenizer tokenizer) { super(fac, tokenizer); } } - // UnicodeReader(ScannerFactory, char[], int) is package-private - static class AccessibleReader extends UnicodeReader { - protected AccessibleReader(ScannerFactory fac, char[] buffer, int length) { - super(fac, buffer, length); - } - } - private JavacTokens() {} } diff --git a/core/src/main/java/com/google/googlejavaformat/java/RemoveUnusedImports.java b/core/src/main/java/com/google/googlejavaformat/java/RemoveUnusedImports.java index a0fc2f54a..8c3cae319 100644 --- a/core/src/main/java/com/google/googlejavaformat/java/RemoveUnusedImports.java +++ b/core/src/main/java/com/google/googlejavaformat/java/RemoveUnusedImports.java @@ -273,7 +273,7 @@ private static RangeMap buildReplacements( Set usedNames, Multimap> usedInJavadoc) { RangeMap replacements = TreeRangeMap.create(); - for (JCImport importTree : unit.getImports()) { + for (JCTree importTree : unit.getImports()) { String simpleName = getSimpleName(importTree); if (!isUnused(unit, usedNames, usedInJavadoc, importTree, simpleName)) { continue; @@ -291,7 +291,7 @@ private static RangeMap buildReplacements( return replacements; } - private static String getSimpleName(JCImport importTree) { + private static String getSimpleName(JCTree importTree) { return getQualifiedIdentifier(importTree).getIdentifier().toString(); } @@ -299,7 +299,7 @@ private static boolean isUnused( JCCompilationUnit unit, Set usedNames, Multimap> usedInJavadoc, - JCImport importTree, + JCTree importTree, String simpleName) { JCFieldAccess qualifiedIdentifier = getQualifiedIdentifier(importTree); String qualifier = qualifiedIdentifier.getExpression().toString(); @@ -322,7 +322,7 @@ private static boolean isUnused( return true; } - private static JCFieldAccess getQualifiedIdentifier(JCImport importTree) { + private static JCFieldAccess getQualifiedIdentifier(JCTree importTree) { // Use reflection because the return type is JCTree in some versions and JCFieldAccess in others try { return (JCFieldAccess) JCImport.class.getMethod("getQualifiedIdentifier").invoke(importTree);