Update cache to use tamper proofing (#1847)

Update the buf CLI's cache to take advantage of tamper proofing features
(manifest, blobs, and digests) to store files as content addressable
storage. When enabled, the new cache directory is stored as:

```
  ~/.cache/buf/v2/module/{remote}/{owner}/{repo}
      blobs/
        {digest[:2]}/
          {digest[2:]} => The contents of the module's blob/manifest.
      commits/
        {commit} => The manifest digest
```

.golangci.yml

@@ -82,6 +82,12 @@ issues:
         - forbidigo
       # this is the one file we want to allow exec.Cmd functions in
       path: private/pkg/command/runner.go
+    - linters:
+        - forbidigo
+      # we use os.Rename here to rename files in the same directory
+      # This is safe (we aren't traversing filesystem boundaries).
+      path: private/pkg/storage/storageos/bucket.go
+      text: "os.Rename"
     - linters:
         - stylecheck
       text: "ST1005:"

private/buf/bufcli/bufcli.go

@@ -121,6 +121,7 @@ var (
 		v1CacheModuleDataRelDirPath,
 		v1CacheModuleLockRelDirPath,
 		v1CacheModuleSumRelDirPath,
+		v2CacheModuleRelDirPath,
 	}
 
 	// ErrNotATTY is returned when an input io.Reader is not a TTY where it is expected.
@@ -154,6 +155,12 @@ var (
 	// These digests are used to make sure that the data written is actually what we expect, and if it is not,
 	// we clear an entry from the cache, i.e. delete the relevant data directory.
 	v1CacheModuleSumRelDirPath = normalpath.Join("v1", "module", "sum")
+	// v2CacheModuleRelDirPath is the relative path to the cache directory for content addressable storage.
+	//
+	// Normalized.
+	// This directory replaces the use of v1CacheModuleDataRelDirPath, v1CacheModuleLockRelDirPath, and
+	// v1CacheModuleSumRelDirPath for modules which support tamper proofing.
+	v2CacheModuleRelDirPath = normalpath.Join("v2", "module")
 
 	// allVisibiltyStrings are the possible options that a user can set the visibility flag with.
 	allVisibiltyStrings = []string{
@@ -546,62 +553,81 @@ func newModuleReaderAndCreateCacheDirs(
 	clientConfig *connectclient.Config,
 	cacheModuleReaderOpts ...bufmodulecache.ModuleReaderOption,
 ) (bufmodule.ModuleReader, error) {
-	cacheModuleDataDirPath := normalpath.Join(container.CacheDirPath(), v1CacheModuleDataRelDirPath)
-	cacheModuleLockDirPath := normalpath.Join(container.CacheDirPath(), v1CacheModuleLockRelDirPath)
-	cacheModuleSumDirPath := normalpath.Join(container.CacheDirPath(), v1CacheModuleSumRelDirPath)
-	if err := checkExistingCacheDirs(
-		container.CacheDirPath(),
-		container.CacheDirPath(),
-		cacheModuleDataDirPath,
-		cacheModuleLockDirPath,
-		cacheModuleSumDirPath,
-	); err != nil {
-		return nil, err
-	}
-	if err := createCacheDirs(
-		cacheModuleDataDirPath,
-		cacheModuleLockDirPath,
-		cacheModuleSumDirPath,
-	); err != nil {
-		return nil, err
-	}
-	storageosProvider := storageos.NewProvider(storageos.ProviderWithSymlinks())
-	// do NOT want to enable symlinks for our cache
-	dataReadWriteBucket, err := storageosProvider.NewReadWriteBucket(cacheModuleDataDirPath)
+	cacheModuleDataDirPathV1 := normalpath.Join(container.CacheDirPath(), v1CacheModuleDataRelDirPath)
+	cacheModuleLockDirPathV1 := normalpath.Join(container.CacheDirPath(), v1CacheModuleLockRelDirPath)
+	cacheModuleSumDirPathV1 := normalpath.Join(container.CacheDirPath(), v1CacheModuleSumRelDirPath)
+	cacheModuleDirPathV2 := normalpath.Join(container.CacheDirPath(), v2CacheModuleRelDirPath)
+	// Check if tamper proofing env var is enabled
+	tamperProofingEnabled, err := IsBetaTamperProofingEnabled(container)
 	if err != nil {
 		return nil, err
 	}
-	// do NOT want to enable symlinks for our cache
-	sumReadWriteBucket, err := storageosProvider.NewReadWriteBucket(cacheModuleSumDirPath)
-	if err != nil {
-		return nil, err
+	var cacheDirsToCreate []string
+	if tamperProofingEnabled {
+		cacheDirsToCreate = append(cacheDirsToCreate, cacheModuleDirPathV2)
+	} else {
+		cacheDirsToCreate = append(
+			cacheDirsToCreate,
+			cacheModuleDataDirPathV1,
+			cacheModuleLockDirPathV1,
+			cacheModuleSumDirPathV1,
+		)
 	}
-	fileLocker, err := filelock.NewLocker(cacheModuleLockDirPath)
-	if err != nil {
+	if err := checkExistingCacheDirs(container.CacheDirPath(), cacheDirsToCreate...); err != nil {
 		return nil, err
 	}
-	var moduleReaderOpts []bufapimodule.ModuleReaderOption
-	// Check if tamper proofing env var is enabled
-	tamperProofingEnabled, err := IsBetaTamperProofingEnabled(container)
-	if err != nil {
+	if err := createCacheDirs(cacheDirsToCreate...); err != nil {
 		return nil, err
 	}
+	var moduleReaderOpts []bufapimodule.ModuleReaderOption
 	if tamperProofingEnabled {
 		moduleReaderOpts = append(moduleReaderOpts, bufapimodule.WithTamperProofing())
 	}
-	moduleReader := bufmodulecache.NewModuleReader(
-		container.Logger(),
-		container.VerbosePrinter(),
-		fileLocker,
-		dataReadWriteBucket,
-		sumReadWriteBucket,
-		bufapimodule.NewModuleReader(
-			bufapimodule.NewDownloadServiceClientFactory(clientConfig),
-			moduleReaderOpts...,
-		),
-		bufmodulecache.NewRepositoryServiceClientFactory(clientConfig),
-		cacheModuleReaderOpts...,
+	delegateReader := bufapimodule.NewModuleReader(
+		bufapimodule.NewDownloadServiceClientFactory(clientConfig),
+		moduleReaderOpts...,
 	)
+	repositoryClientFactory := bufmodulecache.NewRepositoryServiceClientFactory(clientConfig)
+	storageosProvider := storageos.NewProvider(storageos.ProviderWithSymlinks())
+	var moduleReader bufmodule.ModuleReader
+	if tamperProofingEnabled {
+		casModuleBucket, err := storageosProvider.NewReadWriteBucket(cacheModuleDirPathV2)
+		if err != nil {
+			return nil, err
+		}
+		moduleReader = bufmodulecache.NewCASModuleReader(
+			container.Logger(),
+			container.VerbosePrinter(),
+			casModuleBucket,
+			delegateReader,
+			repositoryClientFactory,
+		)
+	} else {
+		// do NOT want to enable symlinks for our cache
+		dataReadWriteBucket, err := storageosProvider.NewReadWriteBucket(cacheModuleDataDirPathV1)
+		if err != nil {
+			return nil, err
+		}
+		// do NOT want to enable symlinks for our cache
+		sumReadWriteBucket, err := storageosProvider.NewReadWriteBucket(cacheModuleSumDirPathV1)
+		if err != nil {
+			return nil, err
+		}
+		fileLocker, err := filelock.NewLocker(cacheModuleLockDirPathV1)
+		if err != nil {
+			return nil, err
+		}
+		moduleReader = bufmodulecache.NewModuleReader(
+			container.Logger(),
+			container.VerbosePrinter(),
+			fileLocker,
+			dataReadWriteBucket,
+			sumReadWriteBucket,
+			delegateReader,
+			repositoryClientFactory,
+			cacheModuleReaderOpts...,
+		)
+	}
 	return moduleReader, nil
 }
 
@@ -1040,7 +1066,11 @@ func newFetchImageReader(
 }
 
 func checkExistingCacheDirs(baseCacheDirPath string, dirPaths ...string) error {
-	for _, dirPath := range dirPaths {
+	dirPathsToCheck := make([]string, 0, len(dirPaths)+1)
+	// Check base cache directory in addition to subdirectories
+	dirPathsToCheck = append(dirPathsToCheck, baseCacheDirPath)
+	dirPathsToCheck = append(dirPathsToCheck, dirPaths...)
+	for _, dirPath := range dirPathsToCheck {
 		dirPath = normalpath.Unnormalize(dirPath)
 		// OK to use os.Stat instead of os.LStat here as this is CLI-only
 		fileInfo, err := os.Stat(dirPath)

private/bufpkg/bufapimodule/module_reader.go

@@ -67,15 +67,15 @@ func (m *moduleReader) GetModule(ctx context.Context, modulePin bufmoduleref.Mod
 			return nil, errors.New("expected non-nil manifest with tamper proofing enabled")
 		}
 		// use manifest and blobs
-		bucket, err := bufmanifest.NewBucketFromManifestBlobs(
-			ctx,
-			resp.Manifest,
-			resp.Blobs,
-		)
+		moduleManifest, err := bufmanifest.NewManifestFromProto(ctx, resp.Manifest)
 		if err != nil {
 			return nil, err
 		}
-		return bufmodule.NewModuleForBucket(ctx, bucket, identityAndCommitOpt)
+		blobSet, err := bufmanifest.NewBlobSetFromProto(ctx, resp.Blobs)
+		if err != nil {
+			return nil, err
+		}
+		return bufmodule.NewModuleForManifestAndBlobSet(ctx, moduleManifest, blobSet)
 	}
 	resp, err := m.download(ctx, modulePin)
 	if err != nil {

private/bufpkg/bufmanifest/bucket.go

@@ -15,7 +15,6 @@
 package bufmanifest
 
 import (
-	"bytes"
 	"context"
 	"fmt"
 
@@ -34,26 +33,13 @@ func NewBucketFromManifestBlobs(
 	manifestBlob *modulev1alpha1.Blob,
 	blobs []*modulev1alpha1.Blob,
 ) (storage.ReadBucket, error) {
-	if _, err := NewBlobFromProto(manifestBlob); err != nil {
-		return nil, fmt.Errorf("invalid manifest: %w", err)
-	}
-	parsedManifest, err := manifest.NewFromReader(
-		bytes.NewReader(manifestBlob.Content),
-	)
+	parsedManifest, err := NewManifestFromProto(ctx, manifestBlob)
 	if err != nil {
-		return nil, fmt.Errorf("parse manifest content: %w", err)
-	}
-	var memBlobs []manifest.Blob
-	for i, modBlob := range blobs {
-		memBlob, err := NewBlobFromProto(modBlob)
-		if err != nil {
-			return nil, fmt.Errorf("invalid blob at index %d: %w", i, err)
-		}
-		memBlobs = append(memBlobs, memBlob)
+		return nil, err
 	}
-	blobSet, err := manifest.NewBlobSet(ctx, memBlobs)
+	blobSet, err := NewBlobSetFromProto(ctx, blobs)
 	if err != nil {
-		return nil, fmt.Errorf("invalid blobs: %w", err)
+		return nil, err
 	}
 	manifestBucket, err := manifest.NewBucket(
 		*parsedManifest,

private/bufpkg/bufmanifest/mapper.go

@@ -71,6 +71,37 @@ func AsProtoBlob(ctx context.Context, b manifest.Blob) (_ *modulev1alpha1.Blob,
 	}, nil
 }
 
+// NewManifestFromProto returns a Manifest from a proto module blob. It makes sure the
+// digest and content matches.
+func NewManifestFromProto(ctx context.Context, b *modulev1alpha1.Blob) (_ *manifest.Manifest, retErr error) {
+	blob, err := NewBlobFromProto(b)
+	if err != nil {
+		return nil, fmt.Errorf("invalid manifest: %w", err)
+	}
+	r, err := blob.Open(ctx)
+	if err != nil {
+		return nil, err
+	}
+	defer func() {
+		retErr = multierr.Append(retErr, r.Close())
+	}()
+	return manifest.NewFromReader(r)
+}
+
+// NewBlobSetFromProto returns a BlobSet from a slice of proto module blobs.
+// It makes sure the digest and content matches for each blob.
+func NewBlobSetFromProto(ctx context.Context, blobs []*modulev1alpha1.Blob) (*manifest.BlobSet, error) {
+	var memBlobs []manifest.Blob
+	for i, modBlob := range blobs {
+		memBlob, err := NewBlobFromProto(modBlob)
+		if err != nil {
+			return nil, fmt.Errorf("invalid blob at index %d: %w", i, err)
+		}
+		memBlobs = append(memBlobs, memBlob)
+	}
+	return manifest.NewBlobSet(ctx, memBlobs)
+}
+
 // NewBlobFromProto returns a Blob from a proto module blob. It makes sure the
 // digest and content matches.
 func NewBlobFromProto(b *modulev1alpha1.Blob) (manifest.Blob, error) {

private/bufpkg/bufmodule/bufmodule.go

@@ -28,6 +28,7 @@ import (
 	breakingv1 "github.com/bufbuild/buf/private/gen/proto/go/buf/alpha/breaking/v1"
 	lintv1 "github.com/bufbuild/buf/private/gen/proto/go/buf/alpha/lint/v1"
 	modulev1alpha1 "github.com/bufbuild/buf/private/gen/proto/go/buf/alpha/module/v1alpha1"
+	"github.com/bufbuild/buf/private/pkg/manifest"
 	"github.com/bufbuild/buf/private/pkg/storage"
 	"go.uber.org/multierr"
 )
@@ -110,6 +111,21 @@ type Module interface {
 	//
 	// This may be nil, since older versions of the module would not have this stored.
 	LintConfig() *buflintconfig.Config
+	// Manifest returns the manifest for the module (possibly nil).
+	// A manifest's contents contain a lexicographically sorted list of path names along
+	// with each path's digest. The manifest also stores a digest of its own contents which
+	// allows verification of the entire Buf module. In addition to the .proto files in
+	// the module, it also lists the buf.yaml, LICENSE, buf.md, and buf.lock files (if
+	// present).
+	Manifest() *manifest.Manifest
+	// BlobSet returns the raw data for the module (possibly nil).
+	// Each blob in the blob set is indexed by the digest of the blob's contents. For
+	// example, the buf.yaml file will be listed in the Manifest with a given digest,
+	// whose contents can be retrieved by looking up the corresponding digest in the
+	// blob set. This allows API consumers to get access to the original file contents
+	// of every file in the module, which is useful for caching or recreating a module's
+	// original files.
+	BlobSet() *manifest.BlobSet
 
 	getSourceReadBucket() storage.ReadBucket
 	// Note this *can* be nil if we did not build from a named module.
@@ -146,7 +162,7 @@ func ModuleWithModuleIdentityAndCommit(moduleIdentity bufmoduleref.ModuleIdentit
 	}
 }
 
-// NewModuleForBucket returns a new Module. It attempts reads dependencies
+// NewModuleForBucket returns a new Module. It attempts to read dependencies
 // from a lock file in the read bucket.
 func NewModuleForBucket(
 	ctx context.Context,
@@ -165,6 +181,16 @@ func NewModuleForProto(
 	return newModuleForProto(ctx, protoModule, options...)
 }
 
+// NewModuleForManifestAndBlobSet returns a new Module given the manifest and blob set.
+func NewModuleForManifestAndBlobSet(
+	ctx context.Context,
+	manifest *manifest.Manifest,
+	blobSet *manifest.BlobSet,
+	options ...ModuleOption,
+) (Module, error) {
+	return newModuleForManifestAndBlobSet(ctx, manifest, blobSet, options...)
+}
+
 // ModuleWithTargetPaths returns a new Module that specifies specific file or directory paths to build.
 //
 // These paths must exist.
@@ -237,7 +263,7 @@ func NewNopModuleResolver() ModuleResolver {
 type ModuleReader interface {
 	// GetModule gets the Module for the ModulePin.
 	//
-	// Returns an error that fufills storage.IsNotExist if the Module does not exist.
+	// Returns an error that fulfills storage.IsNotExist if the Module does not exist.
 	GetModule(ctx context.Context, modulePin bufmoduleref.ModulePin) (Module, error)
 }
 

private/bufpkg/bufmodule/bufmodulecache/bufmodulecache.go

@@ -67,6 +67,24 @@ func ModuleReaderWithExternalPaths() ModuleReaderOption {
 	}
 }
 
+// NewCASModuleReader creates a new module reader using content addressable storage.
+// This doesn't require file locking and enables support for tamper proofing.
+func NewCASModuleReader(
+	logger *zap.Logger,
+	verbosePrinter verbose.Printer,
+	bucket storage.ReadWriteBucket,
+	delegate bufmodule.ModuleReader,
+	repositoryClientFactory RepositoryServiceClientFactory,
+) bufmodule.ModuleReader {
+	return newCASModuleReader(
+		bucket,
+		delegate,
+		repositoryClientFactory,
+		logger,
+		verbosePrinter,
+	)
+}
+
 type moduleReaderOptions struct {
 	allowCacheExternalPaths bool
 }