Skip to content

Commit

Permalink
Refactor comment handling in tokenization to use a new ErrorProneComm…
Browse files Browse the repository at this point in the history
…ent class that doesn't extend from javac's internal Comment

This is pre-work for handling upcoming JDK changes to the Comment API, which break the current subclass approach by changing the return type of `Comment#getPos`. This change encapsulates more of the internal javac APIs.

PiperOrigin-RevId: 638438532
  • Loading branch information
cushon authored and Error Prone Team committed May 29, 2024
1 parent 3fff610 commit c2df1b6
Show file tree
Hide file tree
Showing 14 changed files with 187 additions and 230 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -55,6 +55,7 @@
import com.google.errorprone.apply.SourceFile;
import com.google.errorprone.fixes.SuggestedFixes.FixCompiler.Result;
import com.google.errorprone.util.ASTHelpers;
import com.google.errorprone.util.ErrorProneComment;
import com.google.errorprone.util.ErrorProneToken;
import com.google.errorprone.util.FindIdentifiers;
import com.sun.source.doctree.DocTree;
Expand Down Expand Up @@ -96,7 +97,6 @@
import com.sun.tools.javac.code.Types.DefaultTypeVisitor;
import com.sun.tools.javac.main.Arguments;
import com.sun.tools.javac.parser.Tokens;
import com.sun.tools.javac.parser.Tokens.Comment;
import com.sun.tools.javac.parser.Tokens.TokenKind;
import com.sun.tools.javac.tree.DCTree;
import com.sun.tools.javac.tree.DCTree.DCDocComment;
Expand Down Expand Up @@ -1689,9 +1689,9 @@ public static SuggestedFix replaceIncludingComments(
if (tokens.get(0).comments().isEmpty()) {
return SuggestedFix.replace(tokens.get(0).pos(), state.getEndPosition(tree), replacement);
}
ImmutableList<Comment> comments =
ImmutableList<ErrorProneComment> comments =
ImmutableList.sortedCopyOf(
Comparator.<Comment>comparingInt(c -> c.getSourcePos(0)).reversed(),
Comparator.<ErrorProneComment>comparingInt(c -> c.getSourcePos(0)).reversed(),
tokens.get(0).comments());
int startPos = getStartPosition(tree);
// This can happen for desugared expressions like `int a, b;`.
Expand All @@ -1700,7 +1700,7 @@ public static SuggestedFix replaceIncludingComments(
}
// Delete backwards for comments which are not separated from our target by a blank line.
CharSequence sourceCode = state.getSourceCode();
for (Comment comment : comments) {
for (ErrorProneComment comment : comments) {
int endOfCommentPos = comment.getSourcePos(comment.getText().length() - 1);
CharSequence stringBetweenComments = sourceCode.subSequence(endOfCommentPos, startPos);
if (stringBetweenComments.chars().filter(c -> c == '\n').count() > 1) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,6 @@
import com.google.common.collect.ImmutableList;
import com.google.errorprone.annotations.CanIgnoreReturnValue;
import com.sun.source.tree.Tree;
import com.sun.tools.javac.parser.Tokens.Comment;

/** Class to hold AST nodes annotated with the comments that are associated with them */
@AutoValue
Expand All @@ -35,9 +34,9 @@ public enum Position {

public abstract T tree();

public abstract ImmutableList<Comment> beforeComments();
public abstract ImmutableList<ErrorProneComment> beforeComments();

public abstract ImmutableList<Comment> afterComments();
public abstract ImmutableList<ErrorProneComment> afterComments();

static <T extends Tree> Builder<T> builder() {
return new AutoValue_Commented.Builder<T>();
Expand All @@ -48,14 +47,14 @@ abstract static class Builder<T extends Tree> {

abstract Builder<T> setTree(T tree);

protected abstract ImmutableList.Builder<Comment> beforeCommentsBuilder();
protected abstract ImmutableList.Builder<ErrorProneComment> beforeCommentsBuilder();

protected abstract ImmutableList.Builder<Comment> afterCommentsBuilder();
protected abstract ImmutableList.Builder<ErrorProneComment> afterCommentsBuilder();

@CanIgnoreReturnValue
Builder<T> addComment(
Comment comment, int nodePosition, int tokenizingOffset, Position position) {
OffsetComment offsetComment = new OffsetComment(comment, tokenizingOffset);
ErrorProneComment comment, int nodePosition, int tokenizingOffset, Position position) {
ErrorProneComment offsetComment = comment.withOffset(tokenizingOffset);

if (comment.getSourcePos(0) < nodePosition) {
if (position.equals(Position.BEFORE) || position.equals(Position.ANY)) {
Expand All @@ -71,11 +70,11 @@ Builder<T> addComment(

@CanIgnoreReturnValue
Builder<T> addAllComment(
Iterable<? extends Comment> comments,
Iterable<ErrorProneComment> comments,
int nodePosition,
int tokenizingOffset,
Position position) {
for (Comment comment : comments) {
for (ErrorProneComment comment : comments) {
addComment(comment, nodePosition, tokenizingOffset, position);
}
return this;
Expand Down
24 changes: 9 additions & 15 deletions check_api/src/main/java/com/google/errorprone/util/Comments.java
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,6 @@
import com.google.common.collect.TreeRangeSet;
import com.google.errorprone.VisitorState;
import com.google.errorprone.util.Commented.Position;
import com.google.errorprone.util.ErrorProneTokens.CommentWithTextAndPosition;
import com.sun.source.tree.BlockTree;
import com.sun.source.tree.ClassTree;
import com.sun.source.tree.ExpressionTree;
Expand All @@ -34,7 +33,6 @@
import com.sun.source.tree.Tree;
import com.sun.source.util.TreePath;
import com.sun.source.util.TreeScanner;
import com.sun.tools.javac.parser.Tokens.Comment;
import com.sun.tools.javac.parser.Tokens.TokenKind;
import com.sun.tools.javac.util.Position.LineMap;
import java.util.Iterator;
Expand Down Expand Up @@ -92,7 +90,7 @@ public static ImmutableList<Commented<ExpressionTree>> findCommentsForArguments(
*
* <p>TODO(andrewrice) Update this method to handle block comments properly if we find the need
*/
public static String getTextFromComment(Comment comment) {
public static String getTextFromComment(ErrorProneComment comment) {
switch (comment.getStyle()) {
case BLOCK:
return comment.getText().replaceAll("^\\s*/\\*\\s*(.*?)\\s*\\*/\\s*", "$1");
Expand Down Expand Up @@ -152,29 +150,25 @@ private static ImmutableList<Commented<ExpressionTree>> findCommentsForArguments
if (tokenTracker.atStartOfLine() && !tokenTracker.wasPreviousLineEmpty()) {
// if the token is at the start of a line it could still have a comment attached which was
// on the previous line
for (Comment c : token.comments()) {
if (!(c instanceof CommentWithTextAndPosition)) {
continue;
}
CommentWithTextAndPosition comment = (CommentWithTextAndPosition) c;
for (ErrorProneComment comment : token.comments()) {
int commentStart = comment.getPos();
int commentEnd = comment.getEndPos();
if (exclude.intersects(Range.closedOpen(commentStart, commentEnd))) {
continue;
}
if (tokenTracker.isCommentOnPreviousLine(c)
if (tokenTracker.isCommentOnPreviousLine(comment)
&& token.pos() <= argumentTracker.currentArgumentStartPosition
&& argumentTracker.isPreviousArgumentOnPreviousLine()) {
// token was on the previous line so therefore we should add it to the previous comment
// unless the previous argument was not on the previous line with it
argumentTracker.addCommentToPreviousArgument(c, Position.ANY);
argumentTracker.addCommentToPreviousArgument(comment, Position.ANY);
} else {
// if the comment comes after the end of the invocation and it's not on the same line
// as the final argument then we need to ignore it
if (commentStart <= invocationEnd
|| lineMap.getLineNumber(commentStart)
<= lineMap.getLineNumber(argumentTracker.currentArgumentEndPosition)) {
argumentTracker.addCommentToCurrentArgument(c, Position.ANY);
argumentTracker.addCommentToCurrentArgument(comment, Position.ANY);
}
}
}
Expand Down Expand Up @@ -366,7 +360,7 @@ void advance(ErrorProneToken token) {
}
}

boolean isCommentOnPreviousLine(Comment c) {
boolean isCommentOnPreviousLine(ErrorProneComment c) {
int tokenLine = lineMap.getLineNumber(c.getSourcePos(0));
return tokenLine == currentLineNumber - 1;
}
Expand Down Expand Up @@ -447,15 +441,15 @@ boolean isPreviousArgumentOnPreviousLine() {
== lineMap.getLineNumber(currentArgumentStartPosition) - 1;
}

void addCommentToPreviousArgument(Comment c, Position position) {
void addCommentToPreviousArgument(ErrorProneComment c, Position position) {
previousCommentedResultBuilder.addComment(c, previousArgumentEndPosition, offset, position);
}

void addCommentToCurrentArgument(Comment c, Position position) {
void addCommentToCurrentArgument(ErrorProneComment c, Position position) {
currentCommentedResultBuilder.addComment(c, currentArgumentStartPosition, offset, position);
}

void addAllCommentsToCurrentArgument(Iterable<Comment> comments, Position position) {
void addAllCommentsToCurrentArgument(Iterable<ErrorProneComment> comments, Position position) {
currentCommentedResultBuilder.addAllComment(
comments, currentArgumentStartPosition, offset, position);
}
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,91 @@
/*
* Copyright 2019 The Error Prone Authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

package com.google.errorprone.util;

import static com.google.common.base.Preconditions.checkArgument;

import com.google.common.base.Supplier;
import com.google.common.base.Suppliers;
import com.sun.tools.javac.parser.Tokens.Comment;
import com.sun.tools.javac.parser.Tokens.Comment.CommentStyle;

/** Wraps a {@link Comment} to allow an offset source position to be reported. */
public final class ErrorProneComment {

private final int pos;
private final int endPos;
private final int offset;
private final Supplier<String> text;
private final CommentStyle style;

ErrorProneComment(int pos, int endPos, int offset, Supplier<String> text, CommentStyle style) {
this.pos = pos;
this.endPos = endPos;
this.offset = offset;
this.text = Suppliers.memoize(text);
this.style = style;
}

public ErrorProneComment withOffset(int offset) {
return new ErrorProneComment(pos, endPos, offset, text, style);
}

public int getPos() {
return pos + offset;
}

public int getEndPos() {
return endPos + offset;
}

/**
* Returns the source position of the character at index {@code index} in the comment text.
*
* <p>The handling of javadoc comments in javac has more logic to skip over leading whitespace and
* '*' characters when indexing into doc comments, but we don't need any of that.
*/
public int getSourcePos(int index) {
checkArgument(
0 <= index && index < (endPos - pos),
"Expected %s in the range [0, %s)",
index,
endPos - pos);
return pos + index + offset;
}

public CommentStyle getStyle() {
return style;
}

public String getText() {
return text.get();
}

/**
* We don't care about {@code @deprecated} javadoc tags (see the DepAnn check).
*
* @return false
*/
public boolean isDeprecated() {
return false;
}

@Override
public String toString() {
return String.format("Comment: '%s'", getText());
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -16,24 +16,23 @@

package com.google.errorprone.util;

import static java.util.stream.Collectors.toList;
import static com.google.common.collect.ImmutableList.toImmutableList;

import com.google.common.collect.Lists;
import com.sun.tools.javac.parser.Tokens.Comment;
import com.google.common.collect.ImmutableList;
import com.sun.tools.javac.parser.Tokens.Token;
import com.sun.tools.javac.parser.Tokens.TokenKind;
import com.sun.tools.javac.util.Name;
import java.util.Collections;
import java.util.List;

/** Wraps a javac {@link Token} to return comments in declaration order. */
public class ErrorProneToken {
private final int offset;
private final Token token;
private final int offset;
private final ImmutableList<ErrorProneComment> comments;

ErrorProneToken(Token token, int offset) {
ErrorProneToken(Token token, int offset, ImmutableList<ErrorProneComment> comments) {
this.token = token;
this.offset = offset;
this.comments = comments;
}

public TokenKind kind() {
Expand All @@ -48,17 +47,8 @@ public int endPos() {
return offset + token.endPos;
}

public List<Comment> comments() {
// javac stores the comments in reverse declaration order because appending to linked
// lists is expensive
if (token.comments == null) {
return Collections.emptyList();
}
if (offset == 0) {
return Lists.reverse(token.comments);
}
return Lists.reverse(
token.comments.stream().map(c -> new OffsetComment(c, offset)).collect(toList()));
public ImmutableList<ErrorProneComment> comments() {
return comments.stream().map(c -> c.withOffset(offset)).collect(toImmutableList());
}

public boolean hasName() {
Expand Down
Loading

0 comments on commit c2df1b6

Please sign in to comment.