diff --git a/clang-tools-extra/clangd/FindSymbols.cpp b/clang-tools-extra/clangd/FindSymbols.cpp index 5244a4e893769e..55f16b7085a6fe 100644 --- a/clang-tools-extra/clangd/FindSymbols.cpp +++ b/clang-tools-extra/clangd/FindSymbols.cpp @@ -454,7 +454,7 @@ class DocumentOutline { if (!MacroName.isValid() || !MacroName.isFileID()) continue; // All conditions satisfied, add the macro. - if (auto *Tok = AST.getTokens().spelledTokenAt(MacroName)) + if (auto *Tok = AST.getTokens().spelledTokenContaining(MacroName)) CurParent = &CurParent->inMacro( *Tok, SM, AST.getTokens().expansionStartingAt(Tok)); } diff --git a/clang-tools-extra/clangd/IncludeCleaner.cpp b/clang-tools-extra/clangd/IncludeCleaner.cpp index 01b47679790f1d..dc5b7ec95db5ff 100644 --- a/clang-tools-extra/clangd/IncludeCleaner.cpp +++ b/clang-tools-extra/clangd/IncludeCleaner.cpp @@ -303,7 +303,7 @@ collectMacroReferences(ParsedAST &AST) { for (const auto &[_, Refs] : AST.getMacros().MacroRefs) { for (const auto &Ref : Refs) { auto Loc = SM.getComposedLoc(SM.getMainFileID(), Ref.StartOffset); - const auto *Tok = AST.getTokens().spelledTokenAt(Loc); + const auto *Tok = AST.getTokens().spelledTokenContaining(Loc); if (!Tok) continue; auto Macro = locateMacroAt(*Tok, PP); diff --git a/clang-tools-extra/clangd/SemanticHighlighting.cpp b/clang-tools-extra/clangd/SemanticHighlighting.cpp index eb025f21f36161..a366f1331c2d3d 100644 --- a/clang-tools-extra/clangd/SemanticHighlighting.cpp +++ b/clang-tools-extra/clangd/SemanticHighlighting.cpp @@ -447,11 +447,10 @@ class HighlightingsBuilder { if (!RLoc.isValid()) return; - const auto *RTok = TB.spelledTokenAt(RLoc); - // Handle `>>`. RLoc is always pointing at the right location, just change - // the end to be offset by 1. - // We'll either point at the beginning of `>>`, hence get a proper spelled - // or point in the middle of `>>` hence get no spelled tok. + const auto *RTok = TB.spelledTokenContaining(RLoc); + // Handle `>>`. RLoc is either part of `>>` or a spelled token on its own + // `>`. If it's the former, slice to have length of 1, if latter use the + // token as-is. if (!RTok || RTok->kind() == tok::greatergreater) { Position Begin = sourceLocToPosition(SourceMgr, RLoc); Position End = sourceLocToPosition(SourceMgr, RLoc.getLocWithOffset(1)); @@ -577,7 +576,7 @@ class HighlightingsBuilder { return std::nullopt; // We might have offsets in the main file that don't correspond to any // spelled tokens. - const auto *Tok = TB.spelledTokenAt(Loc); + const auto *Tok = TB.spelledTokenContaining(Loc); if (!Tok) return std::nullopt; return halfOpenToRange(SourceMgr, diff --git a/clang-tools-extra/clangd/XRefs.cpp b/clang-tools-extra/clangd/XRefs.cpp index cd909266489a85..f94cadeffaa298 100644 --- a/clang-tools-extra/clangd/XRefs.cpp +++ b/clang-tools-extra/clangd/XRefs.cpp @@ -844,7 +844,7 @@ std::vector getDocumentLinks(ParsedAST &AST) { if (Inc.Resolved.empty()) continue; auto HashLoc = SM.getComposedLoc(SM.getMainFileID(), Inc.HashOffset); - const auto *HashTok = AST.getTokens().spelledTokenAt(HashLoc); + const auto *HashTok = AST.getTokens().spelledTokenContaining(HashLoc); assert(HashTok && "got inclusion at wrong offset"); const auto *IncludeTok = std::next(HashTok); const auto *FileTok = std::next(IncludeTok); @@ -938,7 +938,7 @@ class ReferenceFinder : public index::IndexDataConsumer { CollectorOpts.CollectMainFileSymbols = true; for (SourceLocation L : Locs) { L = SM.getFileLoc(L); - if (const auto *Tok = TB.spelledTokenAt(L)) + if (const auto *Tok = TB.spelledTokenContaining(L)) References.push_back( {*Tok, Roles, SymbolCollector::getRefContainer(ASTNode.Parent, CollectorOpts)}); @@ -1216,7 +1216,7 @@ DocumentHighlight toHighlight(const ReferenceFinder::Reference &Ref, std::optional toHighlight(SourceLocation Loc, const syntax::TokenBuffer &TB) { Loc = TB.sourceManager().getFileLoc(Loc); - if (const auto *Tok = TB.spelledTokenAt(Loc)) { + if (const auto *Tok = TB.spelledTokenContaining(Loc)) { DocumentHighlight Result; Result.range = halfOpenToRange( TB.sourceManager(), @@ -1353,7 +1353,8 @@ maybeFindIncludeReferences(ParsedAST &AST, Position Pos, Loc = SM.getIncludeLoc(SM.getFileID(Loc)); ReferencesResult::Reference Result; - const auto *Token = AST.getTokens().spelledTokenAt(Loc); + const auto *Token = AST.getTokens().spelledTokenContaining(Loc); + assert(Token && "references expected token here"); Result.Loc.range = Range{sourceLocToPosition(SM, Token->location()), sourceLocToPosition(SM, Token->endLocation())}; Result.Loc.uri = URIMainFile; diff --git a/clang-tools-extra/clangd/refactor/Rename.cpp b/clang-tools-extra/clangd/refactor/Rename.cpp index c0fc4453a3fccc..c85e13dbdfe97f 100644 --- a/clang-tools-extra/clangd/refactor/Rename.cpp +++ b/clang-tools-extra/clangd/refactor/Rename.cpp @@ -748,7 +748,7 @@ std::vector collectRenameIdentifierRanges( clangd::Range tokenRangeForLoc(ParsedAST &AST, SourceLocation TokLoc, const SourceManager &SM, const LangOptions &LangOpts) { - const auto *Token = AST.getTokens().spelledTokenAt(TokLoc); + const auto *Token = AST.getTokens().spelledTokenContaining(TokLoc); assert(Token && "rename expects spelled tokens"); clangd::Range Result; Result.start = sourceLocToPosition(SM, Token->location()); diff --git a/clang-tools-extra/clangd/unittests/PreambleTests.cpp b/clang-tools-extra/clangd/unittests/PreambleTests.cpp index 6420516e785576..16a2f9448b1ecf 100644 --- a/clang-tools-extra/clangd/unittests/PreambleTests.cpp +++ b/clang-tools-extra/clangd/unittests/PreambleTests.cpp @@ -417,7 +417,7 @@ TEST(PreamblePatchTest, LocateMacroAtWorks) { ASSERT_TRUE(AST); const auto &SM = AST->getSourceManager(); - auto *MacroTok = AST->getTokens().spelledTokenAt( + auto *MacroTok = AST->getTokens().spelledTokenContaining( SM.getComposedLoc(SM.getMainFileID(), Modified.point("use"))); ASSERT_TRUE(MacroTok); @@ -441,7 +441,7 @@ TEST(PreamblePatchTest, LocateMacroAtDeletion) { ASSERT_TRUE(AST); const auto &SM = AST->getSourceManager(); - auto *MacroTok = AST->getTokens().spelledTokenAt( + auto *MacroTok = AST->getTokens().spelledTokenContaining( SM.getComposedLoc(SM.getMainFileID(), Modified.point())); ASSERT_TRUE(MacroTok); @@ -512,9 +512,10 @@ TEST(PreamblePatchTest, RefsToMacros) { ExpectedLocations.push_back(referenceRangeIs(R)); for (const auto &P : Modified.points()) { - auto *MacroTok = AST->getTokens().spelledTokenAt(SM.getComposedLoc( - SM.getMainFileID(), - llvm::cantFail(positionToOffset(Modified.code(), P)))); + auto *MacroTok = + AST->getTokens().spelledTokenContaining(SM.getComposedLoc( + SM.getMainFileID(), + llvm::cantFail(positionToOffset(Modified.code(), P)))); ASSERT_TRUE(MacroTok); EXPECT_THAT(findReferences(*AST, P, 0).References, testing::ElementsAreArray(ExpectedLocations)); diff --git a/clang-tools-extra/clangd/unittests/XRefsTests.cpp b/clang-tools-extra/clangd/unittests/XRefsTests.cpp index f53cbf01b7992c..cbceb9a343f87c 100644 --- a/clang-tools-extra/clangd/unittests/XRefsTests.cpp +++ b/clang-tools-extra/clangd/unittests/XRefsTests.cpp @@ -2173,6 +2173,11 @@ TEST(FindReferences, WithinAST) { using $def[[MyTypeD^ef]] = int; enum MyEnum : $(MyEnum)[[MyTy^peDef]] { }; )cpp", + // UDL + R"cpp( + bool $decl[[operator]]"" _u^dl(unsigned long long value); + bool x = $(x)[[1_udl]]; + )cpp", }; for (const char *Test : Tests) checkFindRefs(Test); @@ -2358,7 +2363,13 @@ TEST(FindReferences, UsedSymbolsFromInclude) { R"cpp([[#in^clude ]] std::[[vector]] vec; - )cpp"}; + )cpp", + + R"cpp( + [[#include ^"udl_header.h"]] + auto x = [[1_b]]; + )cpp", + }; for (const char *Test : Tests) { Annotations T(Test); auto TU = TestTU::withCode(T.code()); @@ -2375,6 +2386,9 @@ TEST(FindReferences, UsedSymbolsFromInclude) { class vector{}; } )cpp"); + TU.AdditionalFiles["udl_header.h"] = guard(R"cpp( + bool operator"" _b(unsigned long long value); + )cpp"); TU.ExtraArgs.push_back("-isystem" + testPath("system")); auto AST = TU.build(); diff --git a/clang/include/clang/Tooling/Syntax/Tokens.h b/clang/include/clang/Tooling/Syntax/Tokens.h index b1bdefed7c97f1..f71b8d67bfea4f 100644 --- a/clang/include/clang/Tooling/Syntax/Tokens.h +++ b/clang/include/clang/Tooling/Syntax/Tokens.h @@ -292,9 +292,9 @@ class TokenBuffer { /// "DECL", "(", "a", ")", ";"} llvm::ArrayRef spelledTokens(FileID FID) const; - /// Returns the spelled Token starting at Loc, if there are no such tokens + /// Returns the spelled Token containing the Loc, if there are no such tokens /// returns nullptr. - const syntax::Token *spelledTokenAt(SourceLocation Loc) const; + const syntax::Token *spelledTokenContaining(SourceLocation Loc) const; /// Get all tokens that expand a macro in \p FID. For the following input /// #define FOO B diff --git a/clang/lib/Tooling/Syntax/Tokens.cpp b/clang/lib/Tooling/Syntax/Tokens.cpp index 8d32c45a4a70cf..0a656dff384217 100644 --- a/clang/lib/Tooling/Syntax/Tokens.cpp +++ b/clang/lib/Tooling/Syntax/Tokens.cpp @@ -383,12 +383,13 @@ llvm::ArrayRef TokenBuffer::spelledTokens(FileID FID) const { return It->second.SpelledTokens; } -const syntax::Token *TokenBuffer::spelledTokenAt(SourceLocation Loc) const { +const syntax::Token * +TokenBuffer::spelledTokenContaining(SourceLocation Loc) const { assert(Loc.isFileID()); const auto *Tok = llvm::partition_point( spelledTokens(SourceMgr->getFileID(Loc)), - [&](const syntax::Token &Tok) { return Tok.location() < Loc; }); - if (!Tok || Tok->location() != Loc) + [&](const syntax::Token &Tok) { return Tok.endLocation() <= Loc; }); + if (!Tok || Loc < Tok->location()) return nullptr; return Tok; } diff --git a/clang/unittests/Tooling/Syntax/TokensTest.cpp b/clang/unittests/Tooling/Syntax/TokensTest.cpp index 42f51697139658..dc8a11dbc345ca 100644 --- a/clang/unittests/Tooling/Syntax/TokensTest.cpp +++ b/clang/unittests/Tooling/Syntax/TokensTest.cpp @@ -374,11 +374,24 @@ TEST_F(TokenCollectorTest, Locations) { auto StartLoc = SourceMgr->getLocForStartOfFile(SourceMgr->getMainFileID()); for (auto &R : Code.ranges()) { - EXPECT_THAT(Buffer.spelledTokenAt(StartLoc.getLocWithOffset(R.Begin)), - Pointee(RangeIs(R))); + EXPECT_THAT( + Buffer.spelledTokenContaining(StartLoc.getLocWithOffset(R.Begin)), + Pointee(RangeIs(R))); } } +TEST_F(TokenCollectorTest, LocationInMiddleOfSpelledToken) { + llvm::Annotations Code(R"cpp( + int foo = [[baa^aar]]; + )cpp"); + recordTokens(Code.code()); + // Check spelled tokens. + auto StartLoc = SourceMgr->getLocForStartOfFile(SourceMgr->getMainFileID()); + EXPECT_THAT( + Buffer.spelledTokenContaining(StartLoc.getLocWithOffset(Code.point())), + Pointee(RangeIs(Code.range()))); +} + TEST_F(TokenCollectorTest, MacroDirectives) { // Macro directives are not stored anywhere at the moment. std::string Code = R"cpp(