blob: make blob.Bucket implement io/fs.FS and io/fs.SubFS (#3272)

blob/blob.go

@@ -18,6 +18,9 @@
 //
 // See https://gocloud.dev/howto/blob/ for a detailed how-to guide.
 //
+// *blob.Bucket implements io/fs.FS and io/fs.SubFS, so it can be used with
+// functions in that package.
+//
 // # Errors
 //
 // The errors returned from this package can be inspected in several ways:
@@ -624,6 +627,11 @@ type Bucket struct {
 	b      driver.Bucket
 	tracer *oc.Tracer
 
+	// ioFSCallback is set via SetIOFSCallback, which must be
+	// called before calling various functions implementing interfaces
+	// from the io/fs package.
+	ioFSCallback func() (context.Context, *ReaderOptions)
+
 	// mu protects the closed variable.
 	// Read locks are kept to allow holding a read lock for long-running calls,
 	// and thereby prevent closing until a call finishes.

blob/blob_fs.go

@@ -0,0 +1,223 @@
+// Copyright 2023 The Go Cloud Development Kit Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package blob
+
+import (
+	"context"
+	"io"
+	"io/fs"
+	"path/filepath"
+	"time"
+
+	"gocloud.dev/internal/gcerr"
+)
+
+// Ensure that Bucket implements various io/fs interfaces.
+var _ = fs.FS(&Bucket{})
+var _ = fs.SubFS(&Bucket{})
+
+// iofsFileInfo describes a single file in an io/fs.FS.
+// It implements fs.FileInfo and fs.DirEntry.
+type iofsFileInfo struct {
+	lo   *ListObject
+	name string
+}
+
+func (f *iofsFileInfo) Name() string               { return f.name }
+func (f *iofsFileInfo) Size() int64                { return f.lo.Size }
+func (f *iofsFileInfo) Mode() fs.FileMode          { return fs.ModeIrregular }
+func (f *iofsFileInfo) ModTime() time.Time         { return f.lo.ModTime }
+func (f *iofsFileInfo) IsDir() bool                { return false }
+func (f *iofsFileInfo) Sys() interface{}           { return f.lo }
+func (f *iofsFileInfo) Info() (fs.FileInfo, error) { return f, nil }
+func (f *iofsFileInfo) Type() fs.FileMode          { return fs.ModeIrregular }
+
+// iofsOpenFile describes a single open file in an io/fs.FS.
+// It implements fs.FileInfo and fs.File.
+type iofsOpenFile struct {
+	*Reader
+	name string
+}
+
+func (f *iofsOpenFile) Name() string               { return f.name }
+func (f *iofsOpenFile) Mode() fs.FileMode          { return fs.ModeIrregular }
+func (f *iofsOpenFile) IsDir() bool                { return false }
+func (f *iofsOpenFile) Sys() interface{}           { return f.r }
+func (f *iofsOpenFile) Stat() (fs.FileInfo, error) { return f, nil }
+
+// iofsDir describes a single directory in an io/fs.FS.
+// It implements fs.FileInfo, fs.DirEntry, and fs.File.
+type iofsDir struct {
+	b    *Bucket
+	key  string
+	name string
+	// If opened is true, we've read entries via openOnce().
+	opened  bool
+	entries []fs.DirEntry
+	offset  int
+}
+
+func newDir(b *Bucket, key, name string) *iofsDir {
+	return &iofsDir{b: b, key: key, name: name}
+}
+
+func (d *iofsDir) Name() string               { return d.name }
+func (d *iofsDir) Size() int64                { return 0 }
+func (d *iofsDir) Mode() fs.FileMode          { return fs.ModeDir }
+func (d *iofsDir) Type() fs.FileMode          { return fs.ModeDir }
+func (d *iofsDir) ModTime() time.Time         { return time.Time{} }
+func (d *iofsDir) IsDir() bool                { return true }
+func (d *iofsDir) Sys() interface{}           { return d }
+func (d *iofsDir) Info() (fs.FileInfo, error) { return d, nil }
+func (d *iofsDir) Stat() (fs.FileInfo, error) { return d, nil }
+func (d *iofsDir) Read([]byte) (int, error) {
+	return 0, &fs.PathError{Op: "read", Path: d.key, Err: fs.ErrInvalid}
+}
+func (d *iofsDir) Close() error { return nil }
+func (d *iofsDir) ReadDir(count int) ([]fs.DirEntry, error) {
+	if err := d.openOnce(); err != nil {
+		return nil, err
+	}
+	n := len(d.entries) - d.offset
+	if n == 0 && count > 0 {
+		return nil, io.EOF
+	}
+	if count > 0 && n > count {
+		n = count
+	}
+	list := make([]fs.DirEntry, n)
+	for i := range list {
+		list[i] = d.entries[d.offset+i]
+	}
+	d.offset += n
+	return list, nil
+}
+
+func (d *iofsDir) openOnce() error {
+	if d.opened {
+		return nil
+	}
+	d.opened = true
+
+	// blob expects directories to end in the delimiter, except at the top level.
+	prefix := d.key
+	if prefix != "" {
+		prefix += "/"
+	}
+	listOpts := ListOptions{
+		Prefix:    prefix,
+		Delimiter: "/",
+	}
+	ctx, _ := d.b.ioFSCallback()
+
+	// Fetch all the directory entries.
+	// Conceivably we could only fetch a few here, and fetch the rest lazily
+	// on demand, but that would add significant complexity.
+	iter := d.b.List(&listOpts)
+	for {
+		item, err := iter.Next(ctx)
+		if err == io.EOF {
+			break
+		}
+		if err != nil {
+			return err
+		}
+		name := filepath.Base(item.Key)
+		if item.IsDir {
+			d.entries = append(d.entries, newDir(d.b, item.Key, name))
+		} else {
+			d.entries = append(d.entries, &iofsFileInfo{item, name})
+		}
+	}
+	// There is no such thing as an empty directory in Bucket, so if
+	// we didn't find anything, it doesn't exist.
+	if len(d.entries) == 0 {
+		return fs.ErrNotExist
+	}
+	return nil
+}
+
+// SetIOFSCallback sets a callback that is used during Open and calls on the objects
+// returned from Open.
+//
+// fn should return a context.Context and *ReaderOptions that can be used in
+// calls to List and NewReader on b. It may be called more than once.
+func (b *Bucket) SetIOFSCallback(fn func() (context.Context, *ReaderOptions)) {
+	b.ioFSCallback = fn
+}
+
+// Open implements fs.FS.Open (https://pkg.go.dev/io/fs#FS).
+//
+// SetIOFSCallback must be called prior to calling this function.
+func (b *Bucket) Open(path string) (fs.File, error) {
+	if b.ioFSCallback == nil {
+		return nil, gcerr.Newf(gcerr.InvalidArgument, nil, "blob: Open -- SetIOFSCallback must be called before Open")
+	}
+	if !fs.ValidPath(path) {
+		return nil, &fs.PathError{Op: "open", Path: path, Err: fs.ErrInvalid}
+	}
+
+	// Check if it's a file. If not, assume it's a directory until proven otherwise.
+	ctx, readerOpts := b.ioFSCallback()
+	var isDir bool
+	var key, name string // name is the last part of the path
+	if path == "." {
+		// Root is always a directory, but blob doesn't want the "." in the key.
+		isDir = true
+		key, name = "", "."
+	} else {
+		exists, _ := b.Exists(ctx, path)
+		isDir = !exists
+		key, name = path, filepath.Base(path)
+	}
+
+	// If it's a directory, list the directory contents. We can't do this lazily
+	// because we need to error out here if it doesn't exist.
+	if isDir {
+		dir := newDir(b, key, name)
+		err := dir.openOnce()
+		if err != nil {
+			if err == fs.ErrNotExist && path == "." {
+				// The root directory must exist.
+				return dir, nil
+			}
+			return nil, &fs.PathError{Op: "open", Path: path, Err: err}
+		}
+		return dir, nil
+	}
+
+	// It's a file; open it and return a wrapper.
+	r, err := b.NewReader(ctx, path, readerOpts)
+	if err != nil {
+		return nil, &fs.PathError{Op: "open", Path: path, Err: err}
+	}
+	return &iofsOpenFile{r, filepath.Base(path)}, nil
+}
+
+// Sub implements fs.SubFS.Sub.
+//
+// SetIOFSCallback must be called prior to calling this function.
+func (b *Bucket) Sub(dir string) (fs.FS, error) {
+	if b.ioFSCallback == nil {
+		return nil, gcerr.Newf(gcerr.InvalidArgument, nil, "blob: Sub -- SetIOFSCallback must be called before Sub")
+	}
+	if dir == "." {
+		return b, nil
+	}
+	// blob expects directories to end in the delimiter, except at the top level.
+	pb := PrefixedBucket(b, dir+"/")
+	pb.SetIOFSCallback(b.ioFSCallback)
+	return pb, nil
+}

blob/blob_fs_test.go

@@ -0,0 +1,148 @@
+// Copyright 2023 The Go Cloud Development Kit Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package blob_test
+
+import (
+	"context"
+	"io/fs"
+	"sort"
+	"testing"
+	"testing/fstest"
+
+	"github.com/google/go-cmp/cmp"
+	"gocloud.dev/blob"
+	"gocloud.dev/blob/memblob"
+)
+
+var fsFiles = []string{
+	"a/very/deeply/nested/sub/dir/with/a/file.txt",
+	"baz.txt",
+	"bazfoo.txt",
+	"dir/foo.txt",
+	"dir/subdir/foo.txt",
+	"foo.txt",
+	"foobar.txt",
+}
+
+func initBucket(t *testing.T, files []string) *blob.Bucket {
+	ctx := context.Background()
+
+	b := memblob.OpenBucket(nil)
+	b.SetIOFSCallback(func() (context.Context, *blob.ReaderOptions) { return ctx, nil })
+	for _, f := range files {
+		if err := b.WriteAll(ctx, f, []byte("data"), nil); err != nil {
+			t.Fatal(err)
+		}
+	}
+	return b
+}
+
+// TestIOFS runs the test/fstest test suite for fs.FS.
+func TestIOFS(t *testing.T) {
+	tests := []struct {
+		Description string
+		Files       []string
+	}{
+		{
+			Description: "empty bucket",
+		},
+		{
+			Description: "non-empty bucket",
+			Files:       fsFiles,
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.Description, func(t *testing.T) {
+			b := initBucket(t, test.Files)
+			defer b.Close()
+			if err := fstest.TestFS(b, test.Files...); err != nil {
+				t.Error(err)
+			}
+		})
+	}
+}
+
+// TestGlob does some basic verification that fs.Glob works as expected
+// when given a blob.Bucket.
+func TestGlob(t *testing.T) {
+	b := initBucket(t, fsFiles)
+	defer b.Close()
+
+	tests := []struct {
+		Pattern string
+		Want    []string
+	}{
+		{
+			Pattern: "*",
+			Want:    []string{"a", "baz.txt", "bazfoo.txt", "dir", "foo.txt", "foobar.txt"},
+		},
+		{
+			Pattern: "foo*",
+			Want:    []string{"foo.txt", "foobar.txt"},
+		},
+		{
+			Pattern: "*foo*",
+			Want:    []string{"bazfoo.txt", "foo.txt", "foobar.txt"},
+		},
+	}
+	for _, test := range tests {
+		t.Run(test.Pattern, func(t *testing.T) {
+			if got, err := fs.Glob(b, test.Pattern); err != nil {
+				t.Fatalf("Failed to glob: %v", err)
+			} else if diff := cmp.Diff(got, test.Want); diff != "" {
+				t.Error(diff)
+			}
+		})
+	}
+}
+
+// TestWalkDir does some basic verification that fs.WalkDir works as expected
+// when given a blob.Bucket.
+func TestWalkDir(t *testing.T) {
+	b := initBucket(t, fsFiles)
+	defer b.Close()
+
+	var got []string
+	fn := func(path string, _ fs.DirEntry, err error) error {
+		if err != nil {
+			t.Errorf("WalkFunc with path %s got error: %v", path, err)
+			return err
+		}
+		got = append(got, path)
+		return nil
+	}
+	if err := fs.WalkDir(b, ".", fn); err != nil {
+		t.Fatalf("WalkDir got an unexpected error: %v", err)
+	}
+	// We want all of the files, plus the directories.
+	want := append(fsFiles,
+		".",
+		"a",
+		"a/very",
+		"a/very/deeply",
+		"a/very/deeply/nested",
+		"a/very/deeply/nested/sub",
+		"a/very/deeply/nested/sub/dir",
+		"a/very/deeply/nested/sub/dir/with",
+		"a/very/deeply/nested/sub/dir/with/a",
+		"dir",
+		"dir/subdir",
+	)
+	sort.Strings(want)
+	if diff := cmp.Diff(got, want); diff != "" {
+		t.Error(diff)
+	}
+}