Semantic highlighting (v2)

Default.sublime-keymap

@@ -334,6 +334,20 @@
     //         }
     //     ]
     // },
+    // Show Scope Name (a replacement for ST's "Show Scope Name", but including semantic tokens if applicable)
+    // {
+    //     "command": "lsp_show_scope_name",
+    //     "keys": [
+    //         "ctrl+alt+shift+p" // Default keybinding for Windows/Linux
+    //     ],
+    //     "context": [
+    //         {
+    //             "key": "lsp.session_with_capability",
+    //             "operator": "equal",
+    //             "operand": "semanticTokensProvider"
+    //         }
+    //     ]
+    // },
     // Internal key-binding
     {
         "keys": [

LSP.sublime-settings

@@ -89,6 +89,13 @@
   // Valid values are "dot", "circle", "bookmark", "sign" or ""
   "diagnostics_gutter_marker": "dot",
 
+  // Enable semantic highlighting in addition to standard syntax highlighting (experimental!).
+  // Note: Must be supported by the language server and also requires a special rule
+  // in the color scheme to work. Please see the documentation under
+  //   https://lsp.sublimetext.io/customization/#semantic-highlighting
+  // for a description about how to configure your color scheme for semantic highlighting.
+  "semantic_highlighting": false,
+
   // Show code actions:
   // "annotation" - show an annotation on the right when code actions are available.
   // "bulb" - show a bulb in the gutter when code actions are available.

boot.py

@@ -57,6 +57,7 @@
 from .plugin.rename import LspSymbolRenameCommand
 from .plugin.save_command import LspSaveCommand
 from .plugin.selection_range import LspExpandSelectionCommand
+from .plugin.semantic_highlighting import LspShowScopeNameCommand
 from .plugin.symbols import LspDocumentSymbolsCommand
 from .plugin.symbols import LspSelectionAddCommand
 from .plugin.symbols import LspSelectionClearCommand

docs/src/customization.md

@@ -74,6 +74,57 @@ There is an example in the [official ST documentation](https://www.sublimetext.c
 
 The following tables give an overview about the scope names used by LSP.
 
+### Semantic Highlighting
+
+!!! info "This feature is only available if the server has the *semanticTokensProvider* capability."
+    Language servers which support semantic highlighting are for example *clangd*, *rust-analyzer* and *lua-language-server*.
+
+In order to support semantic highlighting, the color scheme requires a special rule with a background color set for semantic tokens, which is (marginally) different from the original background.
+To add such a rule to your color scheme, select `UI: Customize Color Scheme` from the Command Palette and add for example the following code:
+
+```json
+{
+    "rules": [
+        {
+            "scope": "meta.semantic-token",
+            "background": "#00000101"
+        },
+    ]
+}
+```
+
+Furthermore it is possible to adjust the colors for semantic tokens by applying a foreground color to the individual token types:
+
+| scope | [SemanticTokenTypes](https://microsoft.github.io/language-server-protocol/specifications/specification-3-17/#semanticTokenTypes) |
+| ----- | ------------------ |
+| `meta.semantic-token.namespace` | namespace |
+| `meta.semantic-token.type` | type |
+| `meta.semantic-token.class` | class |
+| `meta.semantic-token.enum` | enum |
+| `meta.semantic-token.interface` | interface |
+| `meta.semantic-token.struct` | struct |
+| `meta.semantic-token.typeparameter` | typeParameter |
+| `meta.semantic-token.parameter` | parameter |
+| `meta.semantic-token.variable` | variable |
+| `meta.semantic-token.property` | property |
+| `meta.semantic-token.enummember` | enumMember |
+| `meta.semantic-token.event` | event |
+| `meta.semantic-token.function` | function |
+| `meta.semantic-token.method` | method |
+| `meta.semantic-token.macro` | macro |
+| `meta.semantic-token.keyword` | keyword |
+| `meta.semantic-token.modifier` | modifier |
+| `meta.semantic-token.comment` | comment |
+| `meta.semantic-token.string` | string |
+| `meta.semantic-token.number` | number |
+| `meta.semantic-token.regexp` | regexp |
+| `meta.semantic-token.operator` | operator |
+
+By default, LSP will assign scopes based on the [scope naming guideline](https://www.sublimetext.com/docs/scope_naming.html) to each of these token types, but if you define rules (with foreground colors) for the scopes specified above, the latter will take precedence.
+
+Language servers can also add their own non-standard token types.
+Those will only be highlighted if they are supported by a "LSP-*" helper package.
+
 ### Document Highlights
 
 !!! info "This feature is only available if the server has the *documentHighlightProvider* capability."

plugin/core/protocol.py

@@ -292,6 +292,18 @@ def documentSymbols(cls, params: Mapping[str, Any], view: sublime.View) -> 'Requ
     def documentHighlight(cls, params: Mapping[str, Any], view: sublime.View) -> 'Request':
         return Request("textDocument/documentHighlight", params, view)
 
+    @classmethod
+    def semanticTokensFull(cls, params: Mapping[str, Any], view: sublime.View) -> 'Request':
+        return Request("textDocument/semanticTokens/full", params, view)
+
+    @classmethod
+    def semanticTokensFullDelta(cls, params: Mapping[str, Any], view: sublime.View) -> 'Request':
+        return Request("textDocument/semanticTokens/full/delta", params, view)
+
+    @classmethod
+    def semanticTokensRange(cls, params: Mapping[str, Any], view: sublime.View) -> 'Request':
+        return Request("textDocument/semanticTokens/range", params, view)
+
     @classmethod
     def resolveCompletionItem(cls, params: CompletionItem, view: sublime.View) -> 'Request':
         return Request("completionItem/resolve", params, view)

plugin/core/sessions.py

@@ -279,6 +279,56 @@ def get_initialize_params(variables: Dict[str, str], workspace_folders: List[Wor
         },
         "codeLens": {
             "dynamicRegistration": True
+        },
+        "semanticTokens": {
+            "dynamicRegistration": True,
+            "requests": {
+                "range": True,
+                "full": {
+                    "delta": True
+                }
+            },
+            "tokenTypes": [  # TODO append additional token types supported by a helper plugin
+                "namespace",
+                "type",
+                "class",
+                "enum",
+                "interface",
+                "struct",
+                "typeParameter",
+                "parameter",
+                "variable",
+                "property",
+                "enumMember",
+                "event",
+                "function",
+                "method",
+                "macro",
+                "keyword",
+                "modifier",
+                "comment",
+                "string",
+                "number",
+                "regexp",
+                "operator"
+            ],
+            "tokenModifiers": [
+                "declaration",
+                "definition",
+                "readonly",
+                "static",
+                "deprecated",
+                "abstract",
+                "async",
+                "modification",
+                "documentation",
+                "defaultLibrary"
+            ],
+            "formats": [
+                "relative"
+            ],
+            "overlappingTokenSupport": False,
+            "multilineTokenSupport": True
         }
     }
     workspace_capabilites = {
@@ -301,7 +351,10 @@ def get_initialize_params(variables: Dict[str, str], workspace_folders: List[Wor
                 "valueSet": symbol_tag_value_set
             }
         },
-        "configuration": True
+        "configuration": True,
+        "semanticTokens": {
+            "refreshSupport": True
+        }
     }
     window_capabilities = {
         "showDocument": {
@@ -500,6 +553,15 @@ def additional_variables(cls) -> Optional[Dict[str, str]]:
         """
         return None
 
+    @classmethod
+    def semantic_tokens(cls) -> Dict[str, str]:
+        """
+        If the server is a semanticTokensProvider and uses custom token types that are not defined in the protocol,
+        you can provide a mapping which assigns scopes to token types and is used by the color scheme for semantic
+        highlighting. Keys of the returned dictionary should be token types and values are the corresponding scopes.
+        """
+        return {}
+
     @classmethod
     def storage_path(cls) -> str:
         """
@@ -1104,6 +1166,12 @@ def _handle_initialize_success(self, result: Any) -> None:
         code_action_kinds = self.get_capability('codeActionProvider.codeActionKinds')
         if code_action_kinds:
             debug('{}: supported code action kinds: {}'.format(self.config.name, code_action_kinds))
+        semantic_token_types = self.get_capability('semanticTokensProvider.legend.tokenTypes')
+        if semantic_token_types is not None:
+            debug('{}: Supported semantic token types: {}'.format(self.config.name, semantic_token_types))
+        semantic_token_modifiers = self.get_capability('semanticTokensProvider.legend.tokenModifiers')
+        if semantic_token_modifiers is not None:
+            debug('{}: Supported semantic token modifiers: {}'.format(self.config.name, semantic_token_modifiers))
         if self._watcher_impl:
             config = self.config.file_watcher
             pattern = config.get('pattern')
@@ -1323,6 +1391,19 @@ def m_workspace_applyEdit(self, params: Any, request_id: Any) -> None:
         self._apply_workspace_edit_async(params.get('edit', {})).then(
             lambda _: self.send_response(Response(request_id, {"applied": True})))
 
+    def m_workspace_semanticTokens_refresh(self, params: Any, request_id: Any) -> None:
+        """handles the workspace/semanticTokens/refresh request"""
+        self.send_response(Response(request_id, None))
+        debug("server requested a refresh of semantic tokens")
+        # TODO should we send a request only for the active view, or also for all other SessionBuffers of the Session?
+        active_view = sublime.active_window().active_view()
+        if not active_view:
+            return
+        session_view = self.session_view_for_view_async(active_view)
+        if not session_view:
+            return
+        session_view.session_buffer.do_semantic_tokens_async()
+
     def m_textDocument_publishDiagnostics(self, params: Any) -> None:
         """handles the textDocument/publishDiagnostics notification"""
         uri = params["uri"]

plugin/core/types.py

@@ -201,6 +201,7 @@ class Settings:
     only_show_lsp_completions = None  # type: bool
     popup_max_characters_height = None  # type: int
     popup_max_characters_width = None  # type: int
+    semantic_highlighting = None  # type: bool
     show_code_actions = None  # type: str
     show_code_lens = None  # type: str
     show_code_actions_in_hover = None  # type: bool
@@ -236,6 +237,7 @@ def r(name: str, default: Union[bool, int, str, list, dict]) -> None:
         r("only_show_lsp_completions", False)
         r("popup_max_characters_height", 1000)
         r("popup_max_characters_width", 120)
+        r("semantic_highlighting", False)
         r("show_code_actions", "annotation")
         r("show_code_lens", "annotation")
         r("show_code_actions_in_hover", True)

plugin/documents.py

@@ -138,6 +138,7 @@ class DocumentSyncListener(sublime_plugin.ViewEventListener, AbstractViewListene
     color_boxes_debounce_time = FEATURES_TIMEOUT
     highlights_debounce_time = FEATURES_TIMEOUT
     code_lenses_debounce_time = FEATURES_TIMEOUT
+    semantic_tokens_debounce_time = FEATURES_TIMEOUT
 
     _uri = None  # type: str
 
@@ -227,6 +228,7 @@ def on_session_initialized_async(self, session: Session) -> None:
         if added:
             self._do_color_boxes_async()
             self._do_code_lenses_async()
+            self._do_semantic_tokens_async()
 
     def on_session_shutdown_async(self, session: Session) -> None:
         removed_session = self._session_views.pop(session.config.name, None)
@@ -340,6 +342,8 @@ def on_text_changed_async(self, change_count: int, changes: Iterable[sublime.Tex
         self.do_signature_help_async(manual=False)
         self._when_selection_remains_stable_async(self._do_code_lenses_async, current_region,
                                                   after_ms=self.code_lenses_debounce_time)
+        self._when_selection_remains_stable_async(self._do_semantic_tokens_async, current_region,
+                                                  after_ms=self.semantic_tokens_debounce_time)
 
     def get_uri(self) -> str:
         return self._uri
@@ -635,6 +639,17 @@ def render_highlights_on_main_thread() -> None:
 
         sublime.set_timeout(render_highlights_on_main_thread)
 
+    # --- textDocument/semanticTokens ----------------------------------------------------------------------------------
+
+    def _do_semantic_tokens_async(self) -> None:
+        session = self.session_async("semanticTokensProvider")
+        if not session:
+            return
+        session_view = session.session_view_for_view_async(self.view)
+        if not session_view:
+            return
+        session_view.session_buffer.do_semantic_tokens_async()
+
     # --- textDocument/complete ----------------------------------------------------------------------------------------
 
     def _on_query_completions_async(self, resolve_completion_list: ResolveCompletionsFn, location: int) -> None: