hugofs: Add includeFiles and excludeFiles to mount configuration

Fixes gohugoio#9042

docs/content/en/hugo-modules/configuration.md

@@ -155,3 +155,15 @@ target
 lang
 : The language code, e.g. "en". Only relevant for `content` mounts, and `static` mounts when in multihost mode.
 
+includeFiles (string or slice)
+: One or more [glob](https://github.com/gobwas/glob) patterns matching files or directories to include. If `excludeFiles` is not set, the files matching `includeFiles` will be the files mounted. 
+
+The glob patterns are matched to the filenames starting from the `source` root, they should have Unix styled slashes even on Windows, `/` matches the mount root and `**` can be used as a  super-asterisk to match recursively down all directories, e.g `/posts/**.jpg`.
+
+The search is case-insensitive.
+
+{{< new-in "0.89.0" >}}
+
+excludeFiles (string or slice)
+: One or more glob patterns matching files to exclude.
+

hugofs/fileinfo.go

@@ -23,6 +23,8 @@ import (
 	"strings"
 	"time"
 
+	"github.com/gohugoio/hugo/hugofs/glob"
+
 	"github.com/gohugoio/hugo/hugofs/files"
 	"golang.org/x/text/unicode/norm"
 
@@ -76,6 +78,9 @@ type FileMeta struct {
 	Fs           afero.Fs
 	OpenFunc     func() (afero.File, error)
 	JoinStatFunc func(name string) (FileMetaInfo, error)
+
+	// Include only files or directories that match.
+	InclusionFilter *glob.FilenameFilter
 }
 
 func (m *FileMeta) Copy() *FileMeta {
@@ -95,10 +100,17 @@ func (m *FileMeta) Merge(from *FileMeta) {
 
 	for i := 0; i < dstv.NumField(); i++ {
 		v := dstv.Field(i)
+		if !v.CanSet() {
+			continue
+		}
 		if !hreflect.IsTruthfulValue(v) {
 			v.Set(srcv.Field(i))
 		}
 	}
+
+	if m.InclusionFilter == nil {
+		m.InclusionFilter = from.InclusionFilter
+	}
 }
 
 func (f *FileMeta) Open() (afero.File, error) {

hugofs/filename_filter_fs.go

@@ -0,0 +1,170 @@
+// Copyright 2021 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package hugofs
+
+import (
+	"os"
+	"strings"
+	"syscall"
+	"time"
+
+	"github.com/gohugoio/hugo/hugofs/glob"
+	"github.com/spf13/afero"
+)
+
+func newFilenameFilterFs(fs afero.Fs, base string, filter *glob.FilenameFilter) afero.Fs {
+	return &filenameFilterFs{
+		fs:     fs,
+		base:   base,
+		filter: filter,
+	}
+}
+
+// filenameFilterFs is a filesystem that filters by filename.
+type filenameFilterFs struct {
+	base string
+	fs   afero.Fs
+
+	filter *glob.FilenameFilter
+}
+
+func (fs *filenameFilterFs) LstatIfPossible(name string) (os.FileInfo, bool, error) {
+	fi, b, err := fs.fs.(afero.Lstater).LstatIfPossible(name)
+	if err != nil {
+		return nil, false, err
+	}
+	if !fs.filter.Match(name, fi.IsDir()) {
+		return nil, false, os.ErrNotExist
+	}
+	return fi, b, nil
+}
+
+func (fs *filenameFilterFs) Open(name string) (afero.File, error) {
+	fi, err := fs.fs.Stat(name)
+	if err != nil {
+		return nil, err
+	}
+
+	if !fs.filter.Match(name, fi.IsDir()) {
+		return nil, os.ErrNotExist
+	}
+
+	f, err := fs.fs.Open(name)
+	if err != nil {
+		return nil, err
+	}
+
+	if !fi.IsDir() {
+		return f, nil
+	}
+
+	return &filenameFilterDir{
+		File:   f,
+		base:   fs.base,
+		filter: fs.filter,
+	}, nil
+}
+
+func (fs *filenameFilterFs) OpenFile(name string, flag int, perm os.FileMode) (afero.File, error) {
+	return fs.Open(name)
+}
+
+func (fs *filenameFilterFs) Stat(name string) (os.FileInfo, error) {
+	fi, _, err := fs.LstatIfPossible(name)
+	return fi, err
+}
+
+func (fs *filenameFilterFs) getOpener(name string) func() (afero.File, error) {
+	return func() (afero.File, error) {
+		return fs.Open(name)
+	}
+}
+
+type filenameFilterDir struct {
+	afero.File
+	base   string
+	filter *glob.FilenameFilter
+}
+
+func (f *filenameFilterDir) Readdir(count int) ([]os.FileInfo, error) {
+	fis, err := f.File.Readdir(-1)
+	if err != nil {
+		return nil, err
+	}
+
+	var result []os.FileInfo
+	for _, fi := range fis {
+		fim := fi.(FileMetaInfo)
+		if f.filter.Match(strings.TrimPrefix(fim.Meta().Filename, f.base), fim.IsDir()) {
+			result = append(result, fi)
+		}
+	}
+
+	return result, nil
+}
+
+func (f *filenameFilterDir) Readdirnames(count int) ([]string, error) {
+	dirsi, err := f.Readdir(count)
+	if err != nil {
+		return nil, err
+	}
+
+	dirs := make([]string, len(dirsi))
+	for i, d := range dirsi {
+		dirs[i] = d.Name()
+	}
+	return dirs, nil
+}
+
+func (fs *filenameFilterFs) Chmod(n string, m os.FileMode) error {
+	return syscall.EPERM
+}
+
+func (fs *filenameFilterFs) Chtimes(n string, a, m time.Time) error {
+	return syscall.EPERM
+}
+
+func (fs *filenameFilterFs) Chown(n string, uid, gid int) error {
+	return syscall.EPERM
+}
+
+func (fs *filenameFilterFs) ReadDir(name string) ([]os.FileInfo, error) {
+	panic("not implemented")
+}
+
+func (fs *filenameFilterFs) Remove(n string) error {
+	return syscall.EPERM
+}
+
+func (fs *filenameFilterFs) RemoveAll(p string) error {
+	return syscall.EPERM
+}
+
+func (fs *filenameFilterFs) Rename(o, n string) error {
+	return syscall.EPERM
+}
+func (fs *filenameFilterFs) Create(n string) (afero.File, error) {
+	return nil, syscall.EPERM
+}
+func (fs *filenameFilterFs) Name() string {
+	return "FinameFilterFS"
+}
+
+func (fs *filenameFilterFs) Mkdir(n string, p os.FileMode) error {
+	return syscall.EPERM
+}
+
+func (fs *filenameFilterFs) MkdirAll(n string, p os.FileMode) error {
+	return syscall.EPERM
+}

hugofs/filename_filter_fs_test.go

@@ -0,0 +1,83 @@
+// Copyright 2019 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package hugofs
+
+import (
+	"errors"
+	"fmt"
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/gohugoio/hugo/hugofs/glob"
+
+	"github.com/spf13/afero"
+
+	qt "github.com/frankban/quicktest"
+)
+
+func TestFilenameFilterFs(t *testing.T) {
+	c := qt.New(t)
+
+	const base = "/mybase"
+
+	fs := NewBaseFileDecorator(afero.NewMemMapFs())
+
+	for _, letter := range []string{"a", "b", "c"} {
+		for i := 1; i <= 3; i++ {
+			c.Assert(afero.WriteFile(fs, filepath.Join(base, letter, fmt.Sprintf("my%d.txt", i)), []byte("some text file for"+letter), 0755), qt.IsNil)
+			c.Assert(afero.WriteFile(fs, filepath.Join(base, letter, fmt.Sprintf("my%d.json", i)), []byte("some json file for"+letter), 0755), qt.IsNil)
+		}
+	}
+
+	fs = afero.NewBasePathFs(fs, base)
+
+	filter, err := glob.NewFilenameFilter(nil, []string{"/b/**.txt"})
+	c.Assert(err, qt.IsNil)
+
+	fs = newFilenameFilterFs(fs, base, filter)
+
+	assertExists := func(filename string, shouldExist bool) {
+		filename = filepath.Clean(filename)
+		_, err1 := fs.Stat(filename)
+		f, err2 := fs.Open(filename)
+		if shouldExist {
+			c.Assert(err1, qt.IsNil)
+			c.Assert(err2, qt.IsNil)
+			defer f.Close()
+
+		} else {
+			for _, err := range []error{err1, err2} {
+				c.Assert(err, qt.Not(qt.IsNil))
+				c.Assert(errors.Is(err, os.ErrNotExist), qt.IsTrue)
+			}
+		}
+	}
+
+	assertExists("/a/my1.txt", true)
+	assertExists("/b/my1.txt", false)
+
+	dirB, err := fs.Open("/b")
+	defer dirB.Close()
+	c.Assert(err, qt.IsNil)
+	dirBEntries, err := dirB.Readdirnames(-1)
+	c.Assert(dirBEntries, qt.DeepEquals, []string{"my1.json", "my2.json", "my3.json"})
+
+	dirC, err := fs.Open("/c")
+	defer dirC.Close()
+	c.Assert(err, qt.IsNil)
+	dirCEntries, err := dirC.Readdirnames(-1)
+	c.Assert(dirCEntries, qt.DeepEquals, []string{"my1.json", "my1.txt", "my2.json", "my2.txt", "my3.json", "my3.txt"})
+
+}