Add an LSP server command

.golangci.yml

@@ -100,6 +100,11 @@ issues:
       # trip this off.
       path: private/buf/bufcli/env.go
       text: "G101:"
+    - linters:
+        - gosec
+      # G404 checks for use of the ordinary non-CPRNG.
+      path: private/buf/buflsp/progress.go
+      text: "G404:"
     - linters:
         - containedctx
       # Type must implement an interface whose methods do not accept context. But this

CHANGELOG.md

@@ -2,7 +2,7 @@
 
 ## [Unreleased]
 
-- No changes yet.
+- Add new experimental LSP support under `buf beta lsp`.
 
 ## [v1.41.0] - 2024-09-11
 

go.mod

@@ -34,6 +34,8 @@ require (
 	github.com/spf13/cobra v1.8.1
 	github.com/spf13/pflag v1.0.5
 	github.com/stretchr/testify v1.9.0
+	go.lsp.dev/jsonrpc2 v0.10.0
+	go.lsp.dev/protocol v0.12.0
 	go.opentelemetry.io/otel v1.30.0
 	go.opentelemetry.io/otel/sdk v1.30.0
 	go.opentelemetry.io/otel/trace v1.30.0
@@ -106,9 +108,13 @@ require (
 	github.com/quic-go/qpack v0.5.1 // indirect
 	github.com/rogpeppe/go-internal v1.12.0 // indirect
 	github.com/russross/blackfriday/v2 v2.1.0 // indirect
+	github.com/segmentio/asm v1.1.3 // indirect
+	github.com/segmentio/encoding v0.3.4 // indirect
 	github.com/sirupsen/logrus v1.9.3 // indirect
 	github.com/stoewer/go-strcase v1.3.0 // indirect
 	github.com/vbatts/tar-split v0.11.5 // indirect
+	go.lsp.dev/pkg v0.0.0-20210717090340-384b27a52fb2 // indirect
+	go.lsp.dev/uri v0.3.0 // indirect
 	go.opencensus.io v0.24.0 // indirect
 	go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.54.0 // indirect
 	go.opentelemetry.io/otel/metric v1.30.0 // indirect

go.sum

@@ -230,6 +230,10 @@ github.com/rs/cors v1.11.1 h1:eU3gRzXLRK57F5rKMGMZURNdIG4EoAmX8k94r9wXWHA=
 github.com/rs/cors v1.11.1/go.mod h1:XyqrcTp5zjWr1wsJ8PIRZssZ8b/WMcMf71DJnit4EMU=
 github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk=
 github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
+github.com/segmentio/asm v1.1.3 h1:WM03sfUOENvvKexOLp+pCqgb/WDjsi7EK8gIsICtzhc=
+github.com/segmentio/asm v1.1.3/go.mod h1:Ld3L4ZXGNcSLRg4JBsZ3//1+f/TjYl0Mzen/DQy1EJg=
+github.com/segmentio/encoding v0.3.4 h1:WM4IBnxH8B9TakiM2QD5LyNl9JSndh88QbHqVC+Pauc=
+github.com/segmentio/encoding v0.3.4/go.mod h1:n0JeuIqEQrQoPDGsjo8UNd1iA0U8d8+oHAA4E3G3OxM=
 github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ=
 github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ=
 github.com/spf13/cobra v1.8.1 h1:e5/vxKd/rZsfSJMUX1agtjeTDf+qv1/JdBF8gg5k9ZM=
@@ -251,6 +255,14 @@ github.com/vbatts/tar-split v0.11.5 h1:3bHCTIheBm1qFTcgh9oPu+nNBtX+XJIupG/vacinC
 github.com/vbatts/tar-split v0.11.5/go.mod h1:yZbwRsSeGjusneWgA781EKej9HF8vme8okylkAeNKLk=
 github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
+go.lsp.dev/jsonrpc2 v0.10.0 h1:Pr/YcXJoEOTMc/b6OTmcR1DPJ3mSWl/SWiU1Cct6VmI=
+go.lsp.dev/jsonrpc2 v0.10.0/go.mod h1:fmEzIdXPi/rf6d4uFcayi8HpFP1nBF99ERP1htC72Ac=
+go.lsp.dev/pkg v0.0.0-20210717090340-384b27a52fb2 h1:hCzQgh6UcwbKgNSRurYWSqh8MufqRRPODRBblutn4TE=
+go.lsp.dev/pkg v0.0.0-20210717090340-384b27a52fb2/go.mod h1:gtSHRuYfbCT0qnbLnovpie/WEmqyJ7T4n6VXiFMBtcw=
+go.lsp.dev/protocol v0.12.0 h1:tNprUI9klQW5FAFVM4Sa+AbPFuVQByWhP1ttNUAjIWg=
+go.lsp.dev/protocol v0.12.0/go.mod h1:Qb11/HgZQ72qQbeyPfJbu3hZBH23s1sr4st8czGeDMQ=
+go.lsp.dev/uri v0.3.0 h1:KcZJmh6nFIBeJzTugn5JTU6OOyG0lDOo3R9KwTxTYbo=
+go.lsp.dev/uri v0.3.0/go.mod h1:P5sbO1IQR+qySTWOCnhnK7phBx+W3zbLqSMDJNTw88I=
 go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0=
 go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo=
 go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.54.0 h1:TT4fX+nBOA/+LUkobKGW1ydGcn+G3vRw9+g5HwCphpk=
@@ -321,6 +333,7 @@ golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20211007075335-d3039528d8ac/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20211110154304-99a53858aa08/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220310020820-b874c991c1a5/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=

private/buf/buflsp/buflsp.go

@@ -0,0 +1,269 @@
+// Copyright 2020-2024 Buf Technologies, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Package buflsp implements a language server for Protobuf.
+//
+// The main entry-point of this package is the Serve() function, which creates a new LSP server.
+package buflsp
+
+import (
+	"context"
+	"fmt"
+	"sync/atomic"
+
+	"github.com/bufbuild/buf/private/buf/bufctl"
+	"github.com/bufbuild/buf/private/bufpkg/bufcheck"
+	"github.com/bufbuild/buf/private/bufpkg/bufimage"
+	"github.com/bufbuild/buf/private/pkg/app/appext"
+	"github.com/bufbuild/buf/private/pkg/command"
+	"github.com/bufbuild/buf/private/pkg/pluginrpcutil"
+	"github.com/bufbuild/buf/private/pkg/storage"
+	"github.com/bufbuild/buf/private/pkg/storage/storageos"
+	"github.com/bufbuild/buf/private/pkg/tracing"
+	"go.lsp.dev/jsonrpc2"
+	"go.lsp.dev/protocol"
+	"go.opentelemetry.io/otel/attribute"
+	"go.uber.org/zap"
+)
+
+// Serve spawns a new LSP server, listening on the given stream.
+//
+// Returns a context for managing the server.
+func Serve(
+	ctx context.Context,
+	container appext.Container,
+	controller bufctl.Controller,
+	stream jsonrpc2.Stream,
+) (jsonrpc2.Conn, error) {
+	// The LSP protocol deals with absolute filesystem paths. This requires us to
+	// bypass the bucket API completely, so we create a bucket pointing at the filesystem
+	// root.
+	bucketProvider := storageos.NewProvider(storageos.ProviderWithSymlinks())
+	bucket, err := bucketProvider.NewReadWriteBucket(
+		"/", // This is not correct for Windows.
+		storageos.ReadWriteBucketWithSymlinksIfSupported(),
+	)
+	if err != nil {
+		return nil, err
+	}
+
+	container.CacheDirPath()
+
+	tracer := tracing.NewTracer(container.Tracer())
+	checkClient, err := bufcheck.NewClient(container.Logger(), tracer, pluginrpcutil.NewRunnerProvider(command.NewRunner()), bufcheck.ClientWithStderr(container.Stderr()))
+	if err != nil {
+		return nil, err
+	}
+
+	conn := jsonrpc2.NewConn(stream)
+	lsp := &lsp{
+		conn: conn,
+		client: protocol.ClientDispatcher(
+			&connAdapter{Conn: conn, logger: container.Logger()},
+			zap.NewNop(), // The logging from protocol itself isn't very good, we've replaced it with connAdapter here.
+		),
+		logger:      container.Logger(),
+		tracer:      tracer,
+		controller:  controller,
+		checkClient: checkClient,
+		rootBucket:  bucket,
+	}
+	lsp.fileManager = newFileManager(lsp)
+	off := protocol.TraceOff
+	lsp.traceValue.Store(&off)
+
+	conn.Go(ctx, lsp.newHandler())
+	return conn, nil
+}
+
+// lsp contains all of the LSP server's state. (I.e., it is the "god class" the protocol requires
+// that we implement).
+//
+// This type does not implement protocol.Server; see server.go for that.
+// This type contains all the necessary book-keeping for keeping the server running.
+// Its handler methods are not defined in buflsp.go; they are defined in other files, grouped
+// according to the groupings in
+type lsp struct {
+	conn   jsonrpc2.Conn
+	client protocol.Client
+
+	logger      *zap.Logger
+	tracer      tracing.Tracer
+	controller  bufctl.Controller
+	checkClient bufcheck.Client
+	rootBucket  storage.ReadBucket
+	fileManager *fileManager
+
+	// These are atomics, because they are read often and written to
+	// almost never, but potentially concurrently. Having them side-by-side
+	// is fine; they are almost never written to so false sharing is not a
+	// concern.
+	initParams atomic.Pointer[protocol.InitializeParams]
+	traceValue atomic.Pointer[protocol.TraceValue]
+}
+
+// init performs *actual* initialization of the server. This is called by Initialize().
+//
+// It may only be called once for a given server.
+func (l *lsp) init(params *protocol.InitializeParams) error {
+	if l.initParams.Load() != nil {
+		return fmt.Errorf("called the %q method more than once", protocol.MethodInitialize)
+	}
+	l.initParams.Store(params)
+
+	// TODO: set up logging. We need to forward everything from server.logger through to
+	// the client, if tracing is turned on. The right way to do this is with an extra
+	// goroutine and some channels.
+
+	return nil
+}
+
+// findImportable finds all files that can potentially be imported by the proto file at
+// uri. This returns a map from potential Protobuf import path to the URI of the file it would import.
+//
+// Note that this performs no validation on these files, because those files might be open in the
+// editor and might contain invalid syntax at the moment. We only want to get their paths and nothing
+// more.
+func (l *lsp) findImportable(
+	ctx context.Context,
+	uri protocol.URI,
+) (map[string]bufimage.ImageFileInfo, error) {
+	fileInfos, err := l.controller.GetImportableImageFileInfos(ctx, uri.Filename())
+	if err != nil {
+		return nil, err
+	}
+
+	imports := make(map[string]bufimage.ImageFileInfo)
+	for _, fileInfo := range fileInfos {
+		imports[fileInfo.Path()] = fileInfo
+	}
+
+	l.logger.Sugar().Debugf("found imports for %q: %#v", uri, imports)
+
+	return imports, nil
+}
+
+// newHandler constructs an RPC handler that wraps the default one from jsonrpc2. This allows us
+// to inject debug logging, tracing, and timeouts to requests.
+func (l *lsp) newHandler() jsonrpc2.Handler {
+	actual := protocol.ServerHandler(newServer(l), nil)
+	return func(ctx context.Context, reply jsonrpc2.Replier, req jsonrpc2.Request) (retErr error) {
+		ctx, span := l.tracer.Start(
+			ctx,
+			tracing.WithErr(&retErr),
+			tracing.WithAttributes(attribute.String("method", req.Method())),
+		)
+		defer span.End()
+
+		l.logger.Debug(
+			"processing request",
+			zap.String("method", req.Method()),
+			zap.ByteString("params", req.Params()),
+		)
+
+		ctx = withRequestID(ctx)
+
+		replier := l.adaptReplier(reply, req)
+
+		// Verify that the server has been initialized if this isn't the initialization
+		// request.
+		if req.Method() != protocol.MethodInitialize && l.initParams.Load() == nil {
+			return replier(ctx, nil, fmt.Errorf("the first call to the server must be the %q method", protocol.MethodInitialize))
+		}
+
+		return actual(ctx, replier, req)
+	}
+}
+
+// adaptReplier wraps a jsonrpc2.Replier, allowing us to inject logging and tracing and so on.
+func (l *lsp) adaptReplier(reply jsonrpc2.Replier, req jsonrpc2.Request) jsonrpc2.Replier {
+	return func(ctx context.Context, result any, err error) error {
+		if err != nil {
+			l.logger.Warn(
+				"responding with error",
+				zap.String("method", req.Method()),
+				zap.Error(err),
+			)
+		} else {
+			l.logger.Debug(
+				"responding",
+				zap.String("method", req.Method()),
+				zap.Reflect("params", result),
+			)
+		}
+
+		return reply(ctx, result, err)
+	}
+}
+
+// connAdapter wraps a connection and logs calls and notifications.
+//
+// By default, the ClientDispatcher does not log the bodies of requests and responses, making
+// for much lower-quality debugging.
+type connAdapter struct {
+	jsonrpc2.Conn
+
+	logger *zap.Logger
+}
+
+func (c *connAdapter) Call(
+	ctx context.Context, method string, params, result any) (id jsonrpc2.ID, err error) {
+	c.logger.Debug(
+		"call",
+		zap.String("method", method),
+		zap.Reflect("params", params),
+	)
+
+	id, err = c.Conn.Call(ctx, method, params, result)
+	if err != nil {
+		c.logger.Warn(
+			"call returned error",
+			zap.String("method", method),
+			zap.Error(err),
+		)
+	} else {
+		c.logger.Warn(
+			"call returned",
+			zap.String("method", method),
+			zap.Reflect("result", result),
+		)
+	}
+
+	return
+}
+
+func (c *connAdapter) Notify(
+	ctx context.Context, method string, params any) error {
+	c.logger.Debug(
+		"notify",
+		zap.String("method", method),
+		zap.Reflect("params", params),
+	)
+
+	err := c.Conn.Notify(ctx, method, params)
+	if err != nil {
+		c.logger.Warn(
+			"notify returned error",
+			zap.String("method", method),
+			zap.Error(err),
+		)
+	} else {
+		c.logger.Warn(
+			"notify returned",
+			zap.String("method", method),
+		)
+	}
+
+	return err
+}