gopls/internal/regtest/marker: port the semantic tokens tests

These tests weren't actually asserting anything other than the lack of
an error in the semantic tokens request (oops). Add some basic test
coverage -- we also have regression tests for this feature. Notably,
though, we had zero coverage of SemanticTokensRange.

For golang/go#54845

Change-Id: Ib7b0df5406fbbbbb34c4a57a8d31395dd9e399d4
Reviewed-on: https://go-review.googlesource.com/c/tools/+/541196
LUCI-TryBot-Result: Go LUCI <golang-scoped@luci-project-accounts.iam.gserviceaccount.com>
Reviewed-by: Alan Donovan <adonovan@google.com>

gopls/internal/lsp/fake/editor.go

@@ -39,7 +39,7 @@ type Editor struct {
 	client     *Client
 	sandbox    *Sandbox
 
-	// TODO(adonovan): buffers should be keyed by protocol.DocumentURI.
+	// TODO(rfindley): buffers should be keyed by protocol.DocumentURI.
 	mu                 sync.Mutex
 	config             EditorConfig                // editor configuration
 	buffers            map[string]buffer           // open buffers (relative path -> buffer content)
@@ -1512,9 +1512,9 @@ func (e *Editor) DocumentHighlight(ctx context.Context, loc protocol.Location) (
 	return e.Server.DocumentHighlight(ctx, params)
 }
 
-// SemanticTokens invokes textDocument/semanticTokens/full, and interprets its
-// result.
-func (e *Editor) SemanticTokens(ctx context.Context, path string) ([]SemanticToken, error) {
+// SemanticTokensFull invokes textDocument/semanticTokens/full, and interprets
+// its result.
+func (e *Editor) SemanticTokensFull(ctx context.Context, path string) ([]SemanticToken, error) {
 	p := &protocol.SemanticTokensParams{
 		TextDocument: protocol.TextDocumentIdentifier{
 			URI: e.sandbox.Workdir.URI(path),
@@ -1531,6 +1531,26 @@ func (e *Editor) SemanticTokens(ctx context.Context, path string) ([]SemanticTok
 	return e.interpretTokens(resp.Data, content), nil
 }
 
+// SemanticTokensRange invokes textDocument/semanticTokens/range, and
+// interprets its result.
+func (e *Editor) SemanticTokensRange(ctx context.Context, loc protocol.Location) ([]SemanticToken, error) {
+	p := &protocol.SemanticTokensRangeParams{
+		TextDocument: protocol.TextDocumentIdentifier{URI: loc.URI},
+		Range:        loc.Range,
+	}
+	resp, err := e.Server.SemanticTokensRange(ctx, p)
+	if err != nil {
+		return nil, err
+	}
+	path := e.sandbox.Workdir.URIToPath(loc.URI)
+	// As noted above: buffers should be keyed by protocol.DocumentURI.
+	content, ok := e.BufferText(path)
+	if !ok {
+		return nil, fmt.Errorf("buffer %s is not open", path)
+	}
+	return e.interpretTokens(resp.Data, content), nil
+}
+
 // A SemanticToken is an interpreted semantic token value.
 type SemanticToken struct {
 	Token     string

gopls/internal/lsp/lsp_test.go

@@ -212,39 +212,6 @@ func (r *runner) CallHierarchy(t *testing.T, spn span.Span, expectedCalls *tests
 	}
 }
 
-func (r *runner) SemanticTokens(t *testing.T, spn span.Span) {
-	uri := spn.URI()
-	filename := uri.Filename()
-	// this is called solely for coverage in semantic.go
-	_, err := r.server.semanticTokensFull(r.ctx, &protocol.SemanticTokensParams{
-		TextDocument: protocol.TextDocumentIdentifier{
-			URI: protocol.URIFromSpanURI(uri),
-		},
-	})
-	if err != nil {
-		t.Errorf("%v for %s", err, filename)
-	}
-	_, err = r.server.semanticTokensRange(r.ctx, &protocol.SemanticTokensRangeParams{
-		TextDocument: protocol.TextDocumentIdentifier{
-			URI: protocol.URIFromSpanURI(uri),
-		},
-		// any legal range. Just to exercise the call.
-		Range: protocol.Range{
-			Start: protocol.Position{
-				Line:      0,
-				Character: 0,
-			},
-			End: protocol.Position{
-				Line:      2,
-				Character: 0,
-			},
-		},
-	})
-	if err != nil {
-		t.Errorf("%v for Range %s", err, filename)
-	}
-}
-
 func applyTextDocumentEdits(r *runner, edits []protocol.DocumentChanges) (map[span.URI][]byte, error) {
 	res := make(map[span.URI][]byte)
 	for _, docEdits := range edits {

gopls/internal/lsp/regtest/marker.go

@@ -289,6 +289,10 @@ var update = flag.Bool("update", false, "if set, update test data during marker
 //     location information. There is no point to using more than one
 //     @symbol marker in a given file.
 //
+//   - token(location, tokenType, mod): makes a textDocument/semanticTokens/range
+//     request at the given location, and asserts that the result includes
+//     exactly one token with the given token type and modifier string.
+//
 //   - workspacesymbol(query, golden): makes a workspace/symbol request for the
 //     given query, formats the response with one symbol per line, and compares
 //     against the named golden file. As workspace symbols are by definition a
@@ -393,10 +397,9 @@ var update = flag.Bool("update", false, "if set, update test data during marker
 //     suggestedfix be consolidated?
 //
 // Existing marker tests (in ../testdata) to port:
+//   - AddImport
 //   - CallHierarchy
-//   - SemanticTokens
 //   - InlayHints
-//   - Renames
 //   - SelectionRanges
 func RunMarkerTests(t *testing.T, dir string) {
 	// The marker tests must be able to run go/packages.Load.
@@ -740,6 +743,7 @@ var actionMarkerFuncs = map[string]func(marker){
 	"snippet":          actionMarkerFunc(snippetMarker),
 	"suggestedfix":     actionMarkerFunc(suggestedfixMarker),
 	"symbol":           actionMarkerFunc(symbolMarker),
+	"token":            actionMarkerFunc(tokenMarker),
 	"typedef":          actionMarkerFunc(typedefMarker),
 	"workspacesymbol":  actionMarkerFunc(workspaceSymbolMarker),
 }
@@ -1864,6 +1868,21 @@ func renameErrMarker(mark marker, loc protocol.Location, newName string, wantErr
 	wantErr.check(mark, err)
 }
 
+func tokenMarker(mark marker, loc protocol.Location, tokenType, mod string) {
+	tokens := mark.run.env.SemanticTokensRange(loc)
+	if len(tokens) != 1 {
+		mark.errorf("got %d tokens, want 1", len(tokens))
+		return
+	}
+	tok := tokens[0]
+	if tok.TokenType != tokenType {
+		mark.errorf("token type = %q, want %q", tok.TokenType, tokenType)
+	}
+	if tok.Mod != mod {
+		mark.errorf("token mod = %q, want %q", tok.Mod, mod)
+	}
+}
+
 func signatureMarker(mark marker, src protocol.Location, label string, active int64) {
 	got := mark.run.env.SignatureHelp(src)
 	if label == "" {

gopls/internal/lsp/regtest/wrappers.go

@@ -514,7 +514,7 @@ func (e *Env) AcceptCompletion(loc protocol.Location, item protocol.CompletionIt
 	}
 }
 
-// CodeAction calls testDocument/codeAction for the given path, and calls
+// CodeAction calls textDocument/codeAction for the given path, and calls
 // t.Fatal if there are errors.
 func (e *Env) CodeAction(path string, diagnostics []protocol.Diagnostic) []protocol.CodeAction {
 	e.T.Helper()
@@ -543,6 +543,28 @@ func (e *Env) ChangeWorkspaceFolders(newFolders ...string) {
 	}
 }
 
+// SemanticTokensFull invokes textDocument/semanticTokens/full, calling t.Fatal
+// on any error.
+func (e *Env) SemanticTokensFull(path string) []fake.SemanticToken {
+	e.T.Helper()
+	toks, err := e.Editor.SemanticTokensFull(e.Ctx, path)
+	if err != nil {
+		e.T.Fatal(err)
+	}
+	return toks
+}
+
+// SemanticTokensRange invokes textDocument/semanticTokens/range, calling t.Fatal
+// on any error.
+func (e *Env) SemanticTokensRange(loc protocol.Location) []fake.SemanticToken {
+	e.T.Helper()
+	toks, err := e.Editor.SemanticTokensRange(e.Ctx, loc)
+	if err != nil {
+		e.T.Fatal(err)
+	}
+	return toks
+}
+
 // Close shuts down the editor session and cleans up the sandbox directory,
 // calling t.Error on any error.
 func (e *Env) Close() {