Merge from 'master' to 'sycl-web' (intel#37)

  CONFLICT (content): Merge conflict in clang/include/clang/Basic/DiagnosticIDs.h

clang-tools-extra/clangd/ClangdLSPServer.cpp

@@ -485,8 +485,8 @@ void ClangdLSPServer::onInitialize(const InitializeParams &Params,
       }
   }
 
-  ClangdServerOpts.SemanticHighlighting =
-      Params.capabilities.SemanticHighlighting;
+  ClangdServerOpts.TheiaSemanticHighlighting =
+      Params.capabilities.TheiaSemanticHighlighting;
   if (Params.rootUri && *Params.rootUri)
     ClangdServerOpts.WorkspaceRoot = std::string(Params.rootUri->file());
   else if (Params.rootPath && !Params.rootPath->empty())
@@ -611,7 +611,7 @@ void ClangdLSPServer::onInitialize(const InitializeParams &Params,
         }}}};
   if (NegotiatedOffsetEncoding)
     Result["offsetEncoding"] = *NegotiatedOffsetEncoding;
-  if (Params.capabilities.SemanticHighlighting)
+  if (Params.capabilities.TheiaSemanticHighlighting)
     Result.getObject("capabilities")
         ->insert(
             {"semanticHighlighting",
@@ -1169,8 +1169,8 @@ void ClangdLSPServer::applyConfiguration(
   reparseOpenedFiles(ModifiedFiles);
 }
 
-void ClangdLSPServer::publishSemanticHighlighting(
-    const SemanticHighlightingParams &Params) {
+void ClangdLSPServer::publishTheiaSemanticHighlighting(
+    const TheiaSemanticHighlightingParams &Params) {
   notify("textDocument/semanticHighlighting", Params);
 }
 
@@ -1376,12 +1376,12 @@ void ClangdLSPServer::onHighlightingsReady(
   // LSP allows us to send incremental edits of highlightings. Also need to diff
   // to remove highlightings from tokens that should no longer have them.
   std::vector<LineHighlightings> Diffed = diffHighlightings(Highlightings, Old);
-  SemanticHighlightingParams Notification;
+  TheiaSemanticHighlightingParams Notification;
   Notification.TextDocument.uri =
       URIForFile::canonicalize(File, /*TUPath=*/File);
   Notification.TextDocument.version = decodeVersion(Version);
-  Notification.Lines = toSemanticHighlightingInformation(Diffed);
-  publishSemanticHighlighting(Notification);
+  Notification.Lines = toTheiaSemanticHighlightingInformation(Diffed);
+  publishTheiaSemanticHighlighting(Notification);
 }
 
 void ClangdLSPServer::onDiagnosticsReady(PathRef File, llvm::StringRef Version,

clang-tools-extra/clangd/ClangdLSPServer.h

@@ -135,7 +135,8 @@ class ClangdLSPServer : private ClangdServer::Callbacks {
   void applyConfiguration(const ConfigurationSettings &Settings);
 
   /// Sends a "publishSemanticHighlighting" notification to the LSP client.
-  void publishSemanticHighlighting(const SemanticHighlightingParams &);
+  void
+  publishTheiaSemanticHighlighting(const TheiaSemanticHighlightingParams &);
 
   /// Sends a "publishDiagnostics" notification to the LSP client.
   void publishDiagnostics(const PublishDiagnosticsParams &);

clang-tools-extra/clangd/ClangdServer.cpp

@@ -58,9 +58,9 @@ namespace {
 struct UpdateIndexCallbacks : public ParsingCallbacks {
   UpdateIndexCallbacks(FileIndex *FIndex,
                        ClangdServer::Callbacks *ServerCallbacks,
-                       bool SemanticHighlighting)
+                       bool TheiaSemanticHighlighting)
       : FIndex(FIndex), ServerCallbacks(ServerCallbacks),
-        SemanticHighlighting(SemanticHighlighting) {}
+        TheiaSemanticHighlighting(TheiaSemanticHighlighting) {}
 
   void onPreambleAST(PathRef Path, llvm::StringRef Version, ASTContext &Ctx,
                      std::shared_ptr<clang::Preprocessor> PP,
@@ -75,14 +75,14 @@ struct UpdateIndexCallbacks : public ParsingCallbacks {
 
     std::vector<Diag> Diagnostics = AST.getDiagnostics();
     std::vector<HighlightingToken> Highlightings;
-    if (SemanticHighlighting)
+    if (TheiaSemanticHighlighting)
       Highlightings = getSemanticHighlightings(AST);
 
     if (ServerCallbacks)
       Publish([&]() {
         ServerCallbacks->onDiagnosticsReady(Path, AST.version(),
                                             std::move(Diagnostics));
-        if (SemanticHighlighting)
+        if (TheiaSemanticHighlighting)
           ServerCallbacks->onHighlightingsReady(Path, AST.version(),
                                                 std::move(Highlightings));
       });
@@ -103,7 +103,7 @@ struct UpdateIndexCallbacks : public ParsingCallbacks {
 private:
   FileIndex *FIndex;
   ClangdServer::Callbacks *ServerCallbacks;
-  bool SemanticHighlighting;
+  bool TheiaSemanticHighlighting;
 };
 } // namespace
 
@@ -112,7 +112,7 @@ ClangdServer::Options ClangdServer::optsForTest() {
   Opts.UpdateDebounce = DebouncePolicy::fixed(/*zero*/ {});
   Opts.StorePreamblesInMemory = true;
   Opts.AsyncThreadsCount = 4; // Consistent!
-  Opts.SemanticHighlighting = true;
+  Opts.TheiaSemanticHighlighting = true;
   return Opts;
 }
 
@@ -142,8 +142,8 @@ ClangdServer::ClangdServer(const GlobalCompilationDatabase &CDB,
       // critical paths.
       WorkScheduler(
           CDB, TUScheduler::Options(Opts),
-          std::make_unique<UpdateIndexCallbacks>(DynamicIdx.get(), Callbacks,
-                                                 Opts.SemanticHighlighting)) {
+          std::make_unique<UpdateIndexCallbacks>(
+              DynamicIdx.get(), Callbacks, Opts.TheiaSemanticHighlighting)) {
   // Adds an index to the stack, at higher priority than existing indexes.
   auto AddIndex = [&](SymbolIndex *Idx) {
     if (this->Index != nullptr) {

clang-tools-extra/clangd/ClangdServer.h

@@ -143,7 +143,7 @@ class ClangdServer {
     std::vector<std::string> QueryDriverGlobs;
 
     /// Enable semantic highlighting features.
-    bool SemanticHighlighting = false;
+    bool TheiaSemanticHighlighting = false;
 
     /// Returns true if the tweak should be enabled.
     std::function<bool(const Tweak &)> TweakFilter = [](const Tweak &T) {

clang-tools-extra/clangd/Hover.cpp

@@ -520,6 +520,49 @@ llvm::Optional<HoverInfo> getHoverContents(const Expr *E, ParsedAST &AST) {
   }
   return llvm::None;
 }
+
+bool isParagraphLineBreak(llvm::StringRef Str, size_t LineBreakIndex) {
+  return Str.substr(LineBreakIndex + 1)
+      .drop_while([](auto C) { return C == ' ' || C == '\t'; })
+      .startswith("\n");
+};
+
+bool isPunctuationLineBreak(llvm::StringRef Str, size_t LineBreakIndex) {
+  constexpr llvm::StringLiteral Punctuation = R"txt(.:,;!?)txt";
+
+  return LineBreakIndex > 0 && Punctuation.contains(Str[LineBreakIndex - 1]);
+};
+
+bool isFollowedByHardLineBreakIndicator(llvm::StringRef Str,
+                                        size_t LineBreakIndex) {
+  // '-'/'*' md list, '@'/'\' documentation command, '>' md blockquote,
+  // '#' headings, '`' code blocks
+  constexpr llvm::StringLiteral LinbreakIdenticators = R"txt(-*@\>#`)txt";
+
+  auto NextNonSpaceCharIndex = Str.find_first_not_of(' ', LineBreakIndex + 1);
+
+  if (NextNonSpaceCharIndex == llvm::StringRef::npos) {
+    return false;
+  }
+
+  auto FollowedBySingleCharIndicator =
+      LinbreakIdenticators.find(Str[NextNonSpaceCharIndex]) !=
+      llvm::StringRef::npos;
+
+  auto FollowedByNumberedListIndicator =
+      llvm::isDigit(Str[NextNonSpaceCharIndex]) &&
+      NextNonSpaceCharIndex + 1 < Str.size() &&
+      (Str[NextNonSpaceCharIndex + 1] == '.' ||
+       Str[NextNonSpaceCharIndex + 1] == ')');
+
+  return FollowedBySingleCharIndicator || FollowedByNumberedListIndicator;
+};
+
+bool isHardLineBreak(llvm::StringRef Str, size_t LineBreakIndex) {
+  return isPunctuationLineBreak(Str, LineBreakIndex) ||
+         isFollowedByHardLineBreakIndicator(Str, LineBreakIndex);
+}
+
 } // namespace
 
 llvm::Optional<HoverInfo> getHover(ParsedAST &AST, Position Pos,
@@ -652,7 +695,7 @@ markup::Document HoverInfo::present() const {
   }
 
   if (!Documentation.empty())
-    Output.addParagraph().appendText(Documentation);
+    parseDocumentation(Documentation, Output);
 
   if (!Definition.empty()) {
     Output.addRuler();
@@ -675,6 +718,45 @@ markup::Document HoverInfo::present() const {
   return Output;
 }
 
+void parseDocumentation(llvm::StringRef Input, markup::Document &Output) {
+
+  constexpr auto WhiteSpaceChars = "\t\n\v\f\r ";
+
+  auto TrimmedInput = Input.trim();
+
+  std::string CurrentLine;
+
+  for (size_t CharIndex = 0; CharIndex < TrimmedInput.size();) {
+    if (TrimmedInput[CharIndex] == '\n') {
+      // Trim whitespace infront of linebreak
+      const auto LastNonSpaceCharIndex =
+          CurrentLine.find_last_not_of(WhiteSpaceChars) + 1;
+      CurrentLine.erase(LastNonSpaceCharIndex);
+
+      if (isParagraphLineBreak(TrimmedInput, CharIndex) ||
+          isHardLineBreak(TrimmedInput, CharIndex)) {
+        // FIXME: maybe distinguish between line breaks and paragraphs
+        Output.addParagraph().appendText(CurrentLine);
+        CurrentLine = "";
+      } else {
+        // Ommit linebreak
+        CurrentLine += ' ';
+      }
+
+      CharIndex++;
+      // After a linebreak always remove spaces to avoid 4 space markdown code
+      // blocks, also skip all additional linebreaks since they have no effect
+      CharIndex = TrimmedInput.find_first_not_of(WhiteSpaceChars, CharIndex);
+    } else {
+      CurrentLine += TrimmedInput[CharIndex];
+      CharIndex++;
+    }
+  }
+  if (!CurrentLine.empty()) {
+    Output.addParagraph().appendText(CurrentLine);
+  }
+}
+
 llvm::raw_ostream &operator<<(llvm::raw_ostream &OS,
                               const HoverInfo::Param &P) {
   std::vector<llvm::StringRef> Output;

clang-tools-extra/clangd/Hover.h

@@ -74,6 +74,10 @@ struct HoverInfo {
   /// Produce a user-readable information.
   markup::Document present() const;
 };
+
+// Try to infer structure of a documentation comment (e.g. line breaks).
+void parseDocumentation(llvm::StringRef Input, markup::Document &Output);
+
 llvm::raw_ostream &operator<<(llvm::raw_ostream &, const HoverInfo::Param &);
 inline bool operator==(const HoverInfo::Param &LHS,
                        const HoverInfo::Param &RHS) {

clang-tools-extra/clangd/Protocol.cpp

@@ -295,7 +295,7 @@ bool fromJSON(const llvm::json::Value &Params, ClientCapabilities &R) {
             TextDocument->getObject("semanticHighlightingCapabilities")) {
       if (auto SemanticHighlightingSupport =
               SemanticHighlighting->getBoolean("semanticHighlighting"))
-        R.SemanticHighlighting = *SemanticHighlightingSupport;
+        R.TheiaSemanticHighlighting = *SemanticHighlightingSupport;
     }
     if (auto *Diagnostics = TextDocument->getObject("publishDiagnostics")) {
       if (auto CategorySupport = Diagnostics->getBoolean("categorySupport"))
@@ -1131,18 +1131,19 @@ llvm::raw_ostream &operator<<(llvm::raw_ostream &OS, OffsetEncoding Enc) {
   return OS << toString(Enc);
 }
 
-bool operator==(const SemanticHighlightingInformation &Lhs,
-                const SemanticHighlightingInformation &Rhs) {
+bool operator==(const TheiaSemanticHighlightingInformation &Lhs,
+                const TheiaSemanticHighlightingInformation &Rhs) {
   return Lhs.Line == Rhs.Line && Lhs.Tokens == Rhs.Tokens;
 }
 
-llvm::json::Value toJSON(const SemanticHighlightingInformation &Highlighting) {
+llvm::json::Value
+toJSON(const TheiaSemanticHighlightingInformation &Highlighting) {
   return llvm::json::Object{{"line", Highlighting.Line},
                             {"tokens", Highlighting.Tokens},
                             {"isInactive", Highlighting.IsInactive}};
 }
 
-llvm::json::Value toJSON(const SemanticHighlightingParams &Highlighting) {
+llvm::json::Value toJSON(const TheiaSemanticHighlightingParams &Highlighting) {
   return llvm::json::Object{
       {"textDocument", Highlighting.TextDocument},
       {"lines", std::move(Highlighting.Lines)},

clang-tools-extra/clangd/Protocol.h

@@ -433,9 +433,11 @@ struct ClientCapabilities {
   /// textDocument.codeAction.codeActionLiteralSupport.
   bool CodeActionStructure = false;
 
-  /// Client supports semantic highlighting.
+  /// Client supports Theia semantic highlighting extension.
+  /// https://github.com/microsoft/vscode-languageserver-node/pull/367
   /// textDocument.semanticHighlightingCapabilities.semanticHighlighting
-  bool SemanticHighlighting = false;
+  /// FIXME: drop this support once clients support LSP 3.16 Semantic Tokens.
+  bool TheiaSemanticHighlighting = false;
 
   /// Supported encodings for LSP character offsets. (clangd extension).
   llvm::Optional<std::vector<OffsetEncoding>> offsetEncoding;
@@ -1342,7 +1344,7 @@ llvm::json::Value toJSON(const FileStatus &FStatus);
 
 /// Represents a semantic highlighting information that has to be applied on a
 /// specific line of the text document.
-struct SemanticHighlightingInformation {
+struct TheiaSemanticHighlightingInformation {
   /// The line these highlightings belong to.
   int Line = 0;
   /// The base64 encoded string of highlighting tokens.
@@ -1353,18 +1355,19 @@ struct SemanticHighlightingInformation {
   /// clients should combine line style and token style if possible.
   bool IsInactive = false;
 };
-bool operator==(const SemanticHighlightingInformation &Lhs,
-                const SemanticHighlightingInformation &Rhs);
-llvm::json::Value toJSON(const SemanticHighlightingInformation &Highlighting);
+bool operator==(const TheiaSemanticHighlightingInformation &Lhs,
+                const TheiaSemanticHighlightingInformation &Rhs);
+llvm::json::Value
+toJSON(const TheiaSemanticHighlightingInformation &Highlighting);
 
 /// Parameters for the semantic highlighting (server-side) push notification.
-struct SemanticHighlightingParams {
+struct TheiaSemanticHighlightingParams {
   /// The textdocument these highlightings belong to.
   VersionedTextDocumentIdentifier TextDocument;
   /// The lines of highlightings that should be sent.
-  std::vector<SemanticHighlightingInformation> Lines;
+  std::vector<TheiaSemanticHighlightingInformation> Lines;
 };
-llvm::json::Value toJSON(const SemanticHighlightingParams &Highlighting);
+llvm::json::Value toJSON(const TheiaSemanticHighlightingParams &Highlighting);
 
 struct SelectionRangeParams {
   /// The text document.

clang-tools-extra/clangd/SemanticHighlighting.cpp

@@ -445,14 +445,15 @@ bool operator==(const LineHighlightings &L, const LineHighlightings &R) {
   return std::tie(L.Line, L.Tokens) == std::tie(R.Line, R.Tokens);
 }
 
-std::vector<SemanticHighlightingInformation>
-toSemanticHighlightingInformation(llvm::ArrayRef<LineHighlightings> Tokens) {
+std::vector<TheiaSemanticHighlightingInformation>
+toTheiaSemanticHighlightingInformation(
+    llvm::ArrayRef<LineHighlightings> Tokens) {
   if (Tokens.size() == 0)
     return {};
 
   // FIXME: Tokens might be multiple lines long (block comments) in this case
   // this needs to add multiple lines for those tokens.
-  std::vector<SemanticHighlightingInformation> Lines;
+  std::vector<TheiaSemanticHighlightingInformation> Lines;
   Lines.reserve(Tokens.size());
   for (const auto &Line : Tokens) {
     llvm::SmallVector<char, 128> LineByteTokens;

clang-tools-extra/clangd/SemanticHighlighting.h

@@ -80,8 +80,9 @@ std::vector<HighlightingToken> getSemanticHighlightings(ParsedAST &AST);
 llvm::StringRef toTextMateScope(HighlightingKind Kind);
 
 /// Convert to LSP's semantic highlighting information.
-std::vector<SemanticHighlightingInformation>
-toSemanticHighlightingInformation(llvm::ArrayRef<LineHighlightings> Tokens);
+std::vector<TheiaSemanticHighlightingInformation>
+toTheiaSemanticHighlightingInformation(
+    llvm::ArrayRef<LineHighlightings> Tokens);
 
 /// Return a line-by-line diff between two highlightings.
 ///  - if the tokens on a line are the same in both highlightings, this line is