Merge pull request #4 from jarxorg/glob-with-delimiter

Glob with delimiter

fs.go

@@ -5,6 +5,7 @@ import (
 	"io"
 	"io/fs"
 	"path"
+	"sort"
 	"strings"
 	"syscall"
 
@@ -156,8 +157,7 @@ func (fsys *GCSFS) ReadDir(dir string) ([]fs.DirEntry, error) {
 	if !fs.ValidPath(dir) {
 		return nil, toPathError(fs.ErrInvalid, "ReadDir", dir)
 	}
-	entries, err := newGcsDir(fsys, dir).ReadDir(-1)
-	return entries, err
+	return newGcsDir(fsys, dir).ReadDir(-1)
 }
 
 // ReadFile reads the named file and returns its contents.
@@ -192,36 +192,63 @@ func (fsys *GCSFS) Sub(dir string) (fs.FS, error) {
 // Glob returns the names of all files matching pattern, providing an implementation
 // of the top-level Glob function.
 func (fsys *GCSFS) Glob(pattern string) ([]string, error) {
+	if pattern == "" || pattern == "*" {
+		entries, err := fsys.ReadDir("")
+		if err != nil {
+			return nil, err
+		}
+		var names []string
+		for _, entry := range entries {
+			names = append(names, entry.Name())
+		}
+		return names, nil
+	}
+	// NOTE: Validate pattern
 	if _, err := path.Match(pattern, ""); err != nil {
 		return nil, err
 	}
-	c, err := fsys.client()
+	names, err := fsys.glob([]string{""}, strings.Split(pattern, "/"), nil)
 	if err != nil {
 		return nil, err
 	}
-
-	query := newQuery("", normalizePrefixPattern(fsys.dir, pattern), "")
-	it := c.bucket(fsys.bucket).objects(fsys.Context(), query)
-
-	var names []string
-	contains := func(name string) bool {
-		for _, n := range names {
-			if n == name {
-				return true
-			}
-		}
-		return false
+	var matches []string
+	for _, name := range names {
+		matches = appendIfMatch(matches, name, pattern)
 	}
-	appendIfMatch := func(name string) error {
-		ok, err := path.Match(pattern, name)
+	sort.Strings(matches)
+	return matches, nil
+}
+
+func (fsys *GCSFS) glob(dirs, patterns []string, matches []string) ([]string, error) {
+	dirOnly := len(patterns) > 1
+	var subDirs []string
+	for _, dir := range dirs {
+		keys, err := fsys.listForGlob(path.Join(dir, patterns[0]), dirOnly)
 		if err != nil {
-			return toPathError(err, "Glob", pattern)
+			return nil, err
 		}
-		if ok && !contains(name) {
-			names = append(names, name)
+		for _, key := range keys {
+			if dirOnly {
+				subDirs = append(subDirs, key)
+			}
+			matches = append(matches, key)
 		}
-		return nil
 	}
+	if len(subDirs) > 0 && dirOnly {
+		return fsys.glob(subDirs, patterns[1:], matches)
+	}
+	return matches, nil
+}
+
+func (fsys *GCSFS) listForGlob(pattern string, dirOnly bool) ([]string, error) {
+	c, err := fsys.client()
+	if err != nil {
+		return nil, err
+	}
+	query := newQuery("/", normalizePrefixPattern(fsys.dir, pattern), "")
+	it := c.bucket(fsys.bucket).objects(fsys.Context(), query)
+
+	var names []string
 	for {
 		attrs, err := it.nextAttrs()
 		if err == iterator.Done {
@@ -230,19 +257,16 @@ func (fsys *GCSFS) Glob(pattern string) ([]string, error) {
 		if err != nil {
 			return nil, toPathError(err, "Glob", pattern)
 		}
-		name := attrs.Name
-		if name == "" {
-			name = strings.TrimSuffix(attrs.Prefix, "/")
+		if attrs.Name == "" {
+			name := fsys.rel(strings.TrimSuffix(attrs.Prefix, "/"))
+			names = appendIfMatch(names, name, pattern)
+			continue
 		}
-		name = fsys.rel(name)
-		if dir := path.Dir(name); dir != "." {
-			if err := appendIfMatch(dir); err != nil {
-				return nil, err
-			}
-		}
-		if err := appendIfMatch(name); err != nil {
-			return nil, err
+		if dirOnly {
+			continue
 		}
+		name := fsys.rel(attrs.Name)
+		names = appendIfMatch(names, name, pattern)
 	}
 	return names, nil
 }

fs_test.go

@@ -158,6 +158,9 @@ func (o *fsObjects) readDir() error {
 			continue
 		}
 		if d.IsDir() {
+			if !o.query.IncludeTrailingDelimiter {
+				continue
+			}
 			name = name + "/"
 		}
 		if o.query.StartOffset != "" && o.query.StartOffset > name {
@@ -206,6 +209,9 @@ func (o *fsObjects) walkDir() error {
 			return err
 		}
 		if d.IsDir() {
+			if !o.query.IncludeTrailingDelimiter {
+				return nil
+			}
 			name = name + "/"
 		}
 		if o.query.StartOffset != "" && o.query.StartOffset > name {

util.go

@@ -63,18 +63,35 @@ LOOP:
 		}
 	}
 	joined := path.Join(prefix, pattern)
-	if strings.HasSuffix(pattern, "/") {
+	if strings.HasSuffix(pattern, "/") || (joined != "" && pattern == "") {
 		return joined + "/"
 	}
 	return joined
 }
 
 func newQuery(delim, prefix, offset string) *storage.Query {
 	query := &storage.Query{
-		Delimiter:   delim,
-		Prefix:      prefix,
-		StartOffset: offset,
+		Delimiter:                delim,
+		Prefix:                   prefix,
+		StartOffset:              offset,
+		IncludeTrailingDelimiter: delim == "/",
 	}
 	query.SetAttrSelection([]string{"Prefix", "Name", "Size", "Updated"})
 	return query
 }
+
+func contains(keys []string, key string) bool {
+	for _, k := range keys {
+		if k == key {
+			return true
+		}
+	}
+	return false
+}
+
+func appendIfMatch(keys []string, key, pattern string) []string {
+	if ok, _ := path.Match(pattern, key); ok && !contains(keys, key) {
+		keys = append(keys, key)
+	}
+	return keys
+}

util_test.go

@@ -155,11 +155,11 @@ func TestNormalizePrefixPattemr(t *testing.T) {
 		}, {
 			prefix:  "dir",
 			pattern: "",
-			want:    "dir",
+			want:    "dir/",
 		}, {
 			prefix:  "dir",
 			pattern: "*.txt",
-			want:    "dir",
+			want:    "dir/",
 		}, {
 			prefix:  "",
 			pattern: "d*",
@@ -196,9 +196,10 @@ func TestNormalizePrefixPattemr(t *testing.T) {
 
 func TestNewQuery(t *testing.T) {
 	want := &storage.Query{
-		Delimiter:   "/",
-		Prefix:      "prefix",
-		StartOffset: "offset",
+		Delimiter:                "/",
+		Prefix:                   "prefix",
+		StartOffset:              "offset",
+		IncludeTrailingDelimiter: true,
 	}
 	want.SetAttrSelection([]string{"Prefix", "Name", "Size", "Updated"})
 
@@ -207,3 +208,27 @@ func TestNewQuery(t *testing.T) {
 		t.Errorf(`Error newQuery returns %v; want %v`, want, got)
 	}
 }
+
+func TestContains(t *testing.T) {
+	tests := []struct {
+		keys []string
+		key  string
+		want bool
+	}{
+		{
+			keys: []string{"abc", "def", "ghi"},
+			key:  "def",
+			want: true,
+		}, {
+			keys: []string{"abc", "def", "ghi"},
+			key:  "xyz",
+			want: false,
+		},
+	}
+	for _, test := range tests {
+		got := contains(test.keys, test.key)
+		if got != test.want {
+			t.Errorf(`Error contains(%v, %v) returns %v; want %v`, test.keys, test.key, got, test.want)
+		}
+	}
+}