fileserver: Support virtual file systems

modules/caddyhttp/fileserver/browse.go

@@ -19,6 +19,7 @@ import (
 	_ "embed"
 	"encoding/json"
 	"fmt"
+	"io/fs"
 	"net/http"
 	"os"
 	"path"
@@ -80,7 +81,7 @@ func (fsrv *FileServer) serveBrowse(root, dirPath string, w http.ResponseWriter,
 	repl := r.Context().Value(caddy.ReplacerCtxKey).(*caddy.Replacer)
 
 	// calling path.Clean here prevents weird breadcrumbs when URL paths are sketchy like /%2e%2e%2f
-	listing, err := fsrv.loadDirectoryContents(dir, root, path.Clean(r.URL.Path), repl)
+	listing, err := fsrv.loadDirectoryContents(dir.(fs.ReadDirFile), root, path.Clean(r.URL.Path), repl)
 	switch {
 	case os.IsPermission(err):
 		return caddyhttp.Error(http.StatusForbidden, err)
@@ -133,8 +134,8 @@ func (fsrv *FileServer) serveBrowse(root, dirPath string, w http.ResponseWriter,
 	return nil
 }
 
-func (fsrv *FileServer) loadDirectoryContents(dir *os.File, root, urlPath string, repl *caddy.Replacer) (browseTemplateContext, error) {
-	files, err := dir.Readdir(-1)
+func (fsrv *FileServer) loadDirectoryContents(dir fs.ReadDirFile, root, urlPath string, repl *caddy.Replacer) (browseTemplateContext, error) {
+	files, err := dir.ReadDir(10000) // TODO: this limit should probably be configurable
 	if err != nil {
 		return browseTemplateContext{}, err
 	}
@@ -201,25 +202,25 @@ func (fsrv *FileServer) makeBrowseTemplate(tplCtx *templateContext) (*template.T
 	return tpl, nil
 }
 
-// isSymlink return true if f is a symbolic link
-func isSymlink(f os.FileInfo) bool {
-	return f.Mode()&os.ModeSymlink != 0
-}
-
 // isSymlinkTargetDir returns true if f's symbolic link target
 // is a directory.
-func isSymlinkTargetDir(f os.FileInfo, root, urlPath string) bool {
+func (fsrv *FileServer) isSymlinkTargetDir(f fs.FileInfo, root, urlPath string) bool {
 	if !isSymlink(f) {
 		return false
 	}
 	target := caddyhttp.SanitizedPathJoin(root, path.Join(urlPath, f.Name()))
-	targetInfo, err := os.Stat(target)
+	targetInfo, err := fsrv.fileSystem.Stat(target)
 	if err != nil {
 		return false
 	}
 	return targetInfo.IsDir()
 }
 
+// isSymlink return true if f is a symbolic link.
+func isSymlink(f fs.FileInfo) bool {
+	return f.Mode()&os.ModeSymlink != 0
+}
+
 // templateContext powers the context used when evaluating the browse template.
 // It combines browse-specific features with the standard templates handler
 // features.

modules/caddyhttp/fileserver/browsetplcontext.go

@@ -15,6 +15,7 @@
 package fileserver
 
 import (
+	"io/fs"
 	"net/url"
 	"os"
 	"path"
@@ -26,22 +27,31 @@ import (
 	"github.com/caddyserver/caddy/v2"
 	"github.com/caddyserver/caddy/v2/modules/caddyhttp"
 	"github.com/dustin/go-humanize"
+	"go.uber.org/zap"
 )
 
-func (fsrv *FileServer) directoryListing(files []os.FileInfo, canGoUp bool, root, urlPath string, repl *caddy.Replacer) browseTemplateContext {
+func (fsrv *FileServer) directoryListing(entries []fs.DirEntry, canGoUp bool, root, urlPath string, repl *caddy.Replacer) browseTemplateContext {
 	filesToHide := fsrv.transformHidePaths(repl)
 
 	var dirCount, fileCount int
 	fileInfos := []fileInfo{}
 
-	for _, f := range files {
-		name := f.Name()
+	for _, entry := range entries {
+		name := entry.Name()
 
 		if fileHidden(name, filesToHide) {
 			continue
 		}
 
-		isDir := f.IsDir() || isSymlinkTargetDir(f, root, urlPath)
+		info, err := entry.Info()
+		if err != nil {
+			fsrv.logger.Error("could not get info about directory entry",
+				zap.String("name", entry.Name()),
+				zap.String("root", root))
+			continue
+		}
+
+		isDir := entry.IsDir() || fsrv.isSymlinkTargetDir(info, root, urlPath)
 
 		// add the slash after the escape of path to avoid escaping the slash as well
 		if isDir {
@@ -51,11 +61,11 @@ func (fsrv *FileServer) directoryListing(files []os.FileInfo, canGoUp bool, root
 			fileCount++
 		}
 
-		size := f.Size()
-		fileIsSymlink := isSymlink(f)
+		size := info.Size()
+		fileIsSymlink := isSymlink(info)
 		if fileIsSymlink {
-			path := caddyhttp.SanitizedPathJoin(root, path.Join(urlPath, f.Name()))
-			fileInfo, err := os.Stat(path)
+			path := caddyhttp.SanitizedPathJoin(root, path.Join(urlPath, info.Name()))
+			fileInfo, err := fsrv.fileSystem.Stat(path)
 			if err == nil {
 				size = fileInfo.Size()
 			}
@@ -73,8 +83,8 @@ func (fsrv *FileServer) directoryListing(files []os.FileInfo, canGoUp bool, root
 			Name:      name,
 			Size:      size,
 			URL:       u.String(),
-			ModTime:   f.ModTime().UTC(),
-			Mode:      f.Mode(),
+			ModTime:   info.ModTime().UTC(),
+			Mode:      info.Mode(),
 		})
 	}
 	name, _ := url.PathUnescape(urlPath)

modules/caddyhttp/fileserver/matcher.go

@@ -15,7 +15,9 @@
 package fileserver
 
 import (
+	"encoding/json"
 	"fmt"
+	"io/fs"
 	"net/http"
 	"os"
 	"path"
@@ -54,6 +56,11 @@ func init() {
 // - `{http.matchers.file.remainder}` Set to the remainder
 // of the path if the path was split by `split_path`.
 type MatchFile struct {
+	// The file system implementation to use. By default, the
+	// local disk file system will be used.
+	FileSystemRaw json.RawMessage `json:"file_system,omitempty" caddy:"namespace=caddy.fs inline_key=backend"`
+	fileSystem    fs.StatFS
+
 	// The root directory, used for creating absolute
 	// file paths, and required when working with
 	// relative paths; if not specified, `{http.vars.root}`
@@ -241,10 +248,23 @@ func celFileMatcherMacroExpander() parser.MacroExpander {
 }
 
 // Provision sets up m's defaults.
-func (m *MatchFile) Provision(_ caddy.Context) error {
+func (m *MatchFile) Provision(ctx caddy.Context) error {
+	// establish the file system to use
+	if len(m.FileSystemRaw) > 0 {
+		mod, err := ctx.LoadModule(m, "FileSystemRaw")
+		if err != nil {
+			return fmt.Errorf("loading file system module: %v", err)
+		}
+		m.fileSystem = mod.(fs.StatFS)
+	}
+	if m.fileSystem == nil {
+		m.fileSystem = osFS{}
+	}
+
 	if m.Root == "" {
 		m.Root = "{http.vars.root}"
 	}
+
 	// if list of files to try was omitted entirely, assume URL path
 	// (use placeholder instead of r.URL.Path; see issue #4146)
 	if m.TryFiles == nil {
@@ -316,7 +336,7 @@ func (m MatchFile) selectFile(r *http.Request) (matched bool) {
 				return
 			}
 			suffix, fullpath, remainder := prepareFilePath(f)
-			if info, exists := strictFileExists(fullpath); exists {
+			if info, exists := m.strictFileExists(fullpath); exists {
 				setPlaceholders(info, suffix, fullpath, remainder)
 				return true
 			}
@@ -330,7 +350,7 @@ func (m MatchFile) selectFile(r *http.Request) (matched bool) {
 		var info os.FileInfo
 		for _, f := range m.TryFiles {
 			suffix, fullpath, splitRemainder := prepareFilePath(f)
-			info, err := os.Stat(fullpath)
+			info, err := m.fileSystem.Stat(fullpath)
 			if err == nil && info.Size() > largestSize {
 				largestSize = info.Size()
 				largestFilename = fullpath
@@ -349,7 +369,7 @@ func (m MatchFile) selectFile(r *http.Request) (matched bool) {
 		var info os.FileInfo
 		for _, f := range m.TryFiles {
 			suffix, fullpath, splitRemainder := prepareFilePath(f)
-			info, err := os.Stat(fullpath)
+			info, err := m.fileSystem.Stat(fullpath)
 			if err == nil && (smallestSize == 0 || info.Size() < smallestSize) {
 				smallestSize = info.Size()
 				smallestFilename = fullpath
@@ -368,7 +388,7 @@ func (m MatchFile) selectFile(r *http.Request) (matched bool) {
 		var info os.FileInfo
 		for _, f := range m.TryFiles {
 			suffix, fullpath, splitRemainder := prepareFilePath(f)
-			info, err := os.Stat(fullpath)
+			info, err := m.fileSystem.Stat(fullpath)
 			if err == nil &&
 				(recentDate.IsZero() || info.ModTime().After(recentDate)) {
 				recentDate = info.ModTime()
@@ -404,8 +424,8 @@ func parseErrorCode(input string) error {
 // the file must also be a directory; if it does
 // NOT end in a forward slash, the file must NOT
 // be a directory.
-func strictFileExists(file string) (os.FileInfo, bool) {
-	stat, err := os.Stat(file)
+func (m MatchFile) strictFileExists(file string) (os.FileInfo, bool) {
+	stat, err := m.fileSystem.Stat(file)
 	if err != nil {
 		// in reality, this can be any error
 		// such as permission or even obscure

modules/caddyhttp/fileserver/matcher_test.go

@@ -111,8 +111,9 @@ func TestFileMatcher(t *testing.T) {
 		},
 	} {
 		m := &MatchFile{
-			Root:     "./testdata",
-			TryFiles: []string{"{http.request.uri.path}", "{http.request.uri.path}/"},
+			fileSystem: osFS{},
+			Root:       "./testdata",
+			TryFiles:   []string{"{http.request.uri.path}", "{http.request.uri.path}/"},
 		}
 
 		u, err := url.Parse(tc.path)
@@ -213,9 +214,10 @@ func TestPHPFileMatcher(t *testing.T) {
 		},
 	} {
 		m := &MatchFile{
-			Root:      "./testdata",
-			TryFiles:  []string{"{http.request.uri.path}", "{http.request.uri.path}/index.php"},
-			SplitPath: []string{".php"},
+			fileSystem: osFS{},
+			Root:       "./testdata",
+			TryFiles:   []string{"{http.request.uri.path}", "{http.request.uri.path}/index.php"},
+			SplitPath:  []string{".php"},
 		}
 
 		u, err := url.Parse(tc.path)