hugofs: Add includeFiles and excludeFiles to mount configuration

Fixes gohugoio#9042

hugofs/fileinfo.go

@@ -23,6 +23,8 @@ import (
 	"strings"
 	"time"
 
+	"github.com/gohugoio/hugo/hugofs/glob"
+
 	"github.com/gohugoio/hugo/hugofs/files"
 	"golang.org/x/text/unicode/norm"
 
@@ -76,6 +78,9 @@ type FileMeta struct {
 	Fs           afero.Fs
 	OpenFunc     func() (afero.File, error)
 	JoinStatFunc func(name string) (FileMetaInfo, error)
+
+	// Include only files or directories that match.
+	InclusionFilter *glob.FilenameFilter
 }
 
 func (m *FileMeta) Copy() *FileMeta {
@@ -95,10 +100,17 @@ func (m *FileMeta) Merge(from *FileMeta) {
 
 	for i := 0; i < dstv.NumField(); i++ {
 		v := dstv.Field(i)
+		if !v.CanSet() {
+			continue
+		}
 		if !hreflect.IsTruthfulValue(v) {
 			v.Set(srcv.Field(i))
 		}
 	}
+
+	if m.InclusionFilter == nil {
+		m.InclusionFilter = from.InclusionFilter
+	}
 }
 
 func (f *FileMeta) Open() (afero.File, error) {

hugofs/glob/glob.go

@@ -170,7 +170,7 @@ type FilenameFilter struct {
 }
 
 // NewFilenameFilter creates a new Glob where the Match method will
-// return true if the file should be exluded.
+// return true if the file should be included.
 // Note that the inclusions will be checked first.
 func NewFilenameFilter(inclusions, exclusions []string) (*FilenameFilter, error) {
 	filter := &FilenameFilter{isWindows: isWindows}
@@ -193,6 +193,15 @@ func NewFilenameFilter(inclusions, exclusions []string) (*FilenameFilter, error)
 	return filter, nil
 }
 
+// MustNewFilenameFilter invokes NewFilenameFilter and panics on error.
+func MustNewFilenameFilter(inclusions, exclusions []string) *FilenameFilter {
+	filter, err := NewFilenameFilter(inclusions, exclusions)
+	if err != nil {
+		panic(err)
+	}
+	return filter
+}
+
 // NewFilenameFilterForInclusionFunc create a new filter using the provided inclusion func.
 func NewFilenameFilterForInclusionFunc(shouldInclude func(filename string) bool) *FilenameFilter {
 	return &FilenameFilter{shouldInclude: shouldInclude, isWindows: isWindows}

hugofs/rootmapping_fs.go

@@ -367,6 +367,9 @@ func (fs *RootMappingFs) collectDirEntries(prefix string) ([]os.FileInfo, error)
 
 		for _, fi := range direntries {
 			meta := fi.(FileMetaInfo).Meta()
+			if !rm.Meta.InclusionFilter.Match(meta.Filename) {
+				continue
+			}
 			meta.Merge(rm.Meta)
 			if fi.IsDir() {
 				name := fi.Name()
@@ -508,7 +511,13 @@ func (fs *RootMappingFs) doLstat(name string) ([]FileMetaInfo, error) {
 	}
 
 	fileCount := 0
+	var wasFiltered bool
 	for _, root := range roots {
+		// TODO1 relevant?
+		if !root.Meta.InclusionFilter.Match(name) {
+			wasFiltered = true
+			continue
+		}
 		if !root.fi.IsDir() {
 			fileCount++
 		}
@@ -518,6 +527,9 @@ func (fs *RootMappingFs) doLstat(name string) ([]FileMetaInfo, error) {
 	}
 
 	if fileCount == 0 {
+		if wasFiltered {
+			return nil, os.ErrNotExist
+		}
 		// Dir only.
 		return []FileMetaInfo{newDirNameOnlyFileInfo(name, roots[0].Meta, fs.virtualDirOpener(name))}, nil
 	}
@@ -531,6 +543,9 @@ func (fs *RootMappingFs) doLstat(name string) ([]FileMetaInfo, error) {
 }
 
 func (fs *RootMappingFs) statRoot(root RootMapping, name string) (FileMetaInfo, bool, error) {
+	if !root.Meta.InclusionFilter.Match(name) {
+		return nil, false, os.ErrNotExist
+	}
 	filename := root.filename(name)
 
 	fi, b, err := lstatIfPossible(fs.Fs, filename)
@@ -586,16 +601,21 @@ func (f *rootMappingFile) Name() string {
 
 func (f *rootMappingFile) Readdir(count int) ([]os.FileInfo, error) {
 	if f.File != nil {
+
 		fis, err := f.File.Readdir(count)
 		if err != nil {
 			return nil, err
 		}
 
-		for i, fi := range fis {
-			fis[i] = decorateFileInfo(fi, f.fs, nil, "", "", f.meta)
+		var result []os.FileInfo
+		for _, fi := range fis {
+			if f.meta.InclusionFilter.Match(fi.(FileMetaInfo).Meta().Filename) {
+				result = append(result, decorateFileInfo(fi, f.fs, nil, "", "", f.meta))
+			}
 		}
-		return fis, nil
+		return result, nil
 	}
+
 	return f.fs.collectDirEntries(f.name)
 }
 

hugofs/rootmapping_fs_test.go

@@ -20,6 +20,8 @@ import (
 	"sort"
 	"testing"
 
+	"github.com/gohugoio/hugo/hugofs/glob"
+
 	"github.com/gohugoio/hugo/config"
 
 	qt "github.com/frankban/quicktest"
@@ -483,3 +485,70 @@ func TestRootMappingFsOsBase(t *testing.T) {
 
 	c.Assert(getDirnames("static/a/b/c"), qt.DeepEquals, []string{"d4", "f-1.txt", "f-2.txt", "f-3.txt", "ms-1.txt"})
 }
+
+func TestRootMappingFileFilter(t *testing.T) {
+	c := qt.New(t)
+	fs := NewBaseFileDecorator(afero.NewMemMapFs())
+
+	for _, lang := range []string{"no", "en", "fr"} {
+		for i := 1; i <= 3; i++ {
+			c.Assert(afero.WriteFile(fs, filepath.Join(lang, fmt.Sprintf("my%s%d.txt", lang, i)), []byte("some text file for"+lang), 0755), qt.IsNil)
+		}
+	}
+
+	for _, lang := range []string{"no", "en", "fr"} {
+		for i := 1; i <= 3; i++ {
+			c.Assert(afero.WriteFile(fs, filepath.Join(lang, "sub", fmt.Sprintf("mysub%s%d.txt", lang, i)), []byte("some text file for"+lang), 0755), qt.IsNil)
+		}
+	}
+
+	rm := []RootMapping{
+		{
+			From: "content",
+			To:   "no",
+			Meta: &FileMeta{Lang: "no", InclusionFilter: glob.MustNewFilenameFilter(nil, []string{"**.txt"})},
+		},
+		{
+			From: "content",
+			To:   "en",
+			Meta: &FileMeta{Lang: "en"},
+		},
+		{
+			From: "content",
+			To:   "fr",
+			Meta: &FileMeta{Lang: "fr", InclusionFilter: glob.MustNewFilenameFilter(nil, []string{"**.txt"})},
+		},
+	}
+
+	rfs, err := NewRootMappingFs(fs, rm...)
+	c.Assert(err, qt.IsNil)
+
+	assertExists := func(filename string, shouldExist bool) {
+		c.Helper()
+		filename = filepath.Clean(filename)
+		_, err1 := rfs.Stat(filename)
+		f, err2 := rfs.Open(filename)
+		if shouldExist {
+			c.Assert(err1, qt.IsNil)
+			c.Assert(err2, qt.IsNil)
+			c.Assert(f.Close(), qt.IsNil)
+		} else {
+			c.Assert(err1, qt.Not(qt.IsNil))
+			c.Assert(err2, qt.Not(qt.IsNil))
+		}
+	}
+
+	assertExists("content/myno1.txt", false)
+	assertExists("content/myen1.txt", true)
+	assertExists("content/myfr1.txt", false)
+
+	dirEntriesSub, err := afero.ReadDir(rfs, filepath.Join("content", "sub"))
+	c.Assert(err, qt.IsNil)
+	c.Assert(len(dirEntriesSub), qt.Equals, 3)
+
+	dirEntries, err := afero.ReadDir(rfs, "content")
+
+	c.Assert(err, qt.IsNil)
+	c.Assert(len(dirEntries), qt.Equals, 4)
+
+}

hugolib/filesystems/basefs.go

@@ -24,6 +24,10 @@ import (
 	"strings"
 	"sync"
 
+	"github.com/gohugoio/hugo/hugofs/glob"
+
+	"github.com/gohugoio/hugo/common/types"
+
 	"github.com/gohugoio/hugo/common/loggers"
 
 	"github.com/gohugoio/hugo/hugofs/files"
@@ -597,6 +601,14 @@ func (b *sourceFilesystemsBuilder) createModFs(
 			mountWeight++
 		}
 
+		inclusionFilter, err := glob.NewFilenameFilter(
+			types.ToStringSlicePreserveString(mount.IncludeFiles),
+			types.ToStringSlicePreserveString(mount.ExcludeFiles),
+		)
+		if err != nil {
+			return err
+		}
+
 		base, filename := absPathify(mount.Source)
 
 		rm := hugofs.RootMapping{
@@ -605,9 +617,10 @@ func (b *sourceFilesystemsBuilder) createModFs(
 			ToBasedir: base,
 			Module:    md.Module.Path(),
 			Meta: &hugofs.FileMeta{
-				Watch:      md.Watch(),
-				Weight:     mountWeight,
-				Classifier: files.ContentClassContent,
+				Watch:           md.Watch(),
+				Weight:          mountWeight,
+				Classifier:      files.ContentClassContent,
+				InclusionFilter: inclusionFilter,
 			},
 		}
 

modules/config.go

@@ -379,6 +379,11 @@ type Mount struct {
 
 	Lang string // any language code associated with this mount.
 
+	// Include only files matching the given Glob patterns (string or slice).
+	IncludeFiles interface{}
+
+	// Exclude all files matching the given Glob patterns (string or slice).
+	ExcludeFiles interface{}
 }
 
 func (m Mount) Component() string {