Prometheus metric for repository's last update

README.md

@@ -32,24 +32,25 @@ Usage:
   rest-server [flags]
 
 Flags:
-      --append-only            enable append only mode
-      --cpu-profile string     write CPU profile to file
-      --debug                  output debug messages
-  -h, --help                   help for rest-server
-      --htpasswd-file string   location of .htpasswd file (default: "<data directory>/.htpasswd")
-      --listen string          listen address (default ":8000")
-      --log filename           write HTTP requests in the combined log format to the specified filename
-      --max-size int           the maximum size of the repository in bytes
-      --no-auth                disable .htpasswd authentication
-      --no-verify-upload       do not verify the integrity of uploaded data. DO NOT enable unless the rest-server runs on a very low-power device
-      --path string            data directory (default "/tmp/restic")
-      --private-repos          users can only access their private repo
-      --prometheus             enable Prometheus metrics
-      --prometheus-no-auth     disable auth for Prometheus /metrics endpoint
-      --tls                    turn on TLS support
-      --tls-cert string        TLS certificate path
-      --tls-key string         TLS key path
-  -v, --version                version for rest-server
+      --append-only             enable append only mode
+      --cpu-profile string      write CPU profile to file
+      --debug                   output debug messages
+  -h, --help                    help for rest-server
+      --htpasswd-file string    location of .htpasswd file (default: "<data directory>/.htpasswd)"
+      --listen string           listen address (default ":8000")
+      --log filename            write HTTP requests in the combined log format to the specified filename
+      --max-size int            the maximum size of the repository in bytes
+      --no-auth                 disable .htpasswd authentication
+      --no-verify-upload        do not verify the integrity of uploaded data. DO NOT enable unless the rest-server runs on a very low-power device
+      --path string             data directory (default "/tmp/restic")
+      --private-repos           users can only access their private repo
+      --prometheus              enable Prometheus metrics
+      --prometheus-no-auth      disable auth for Prometheus /metrics endpoint
+      --prometheus-no-preload   disable preloading Prometheus metrics during startup
+      --tls                     turn on TLS support
+      --tls-cert string         TLS certificate path
+      --tls-key string          TLS key path
+  -v, --version                 version for rest-server
 ```
 
 By default the server persists backup data in the OS temporary directory (`/tmp/restic` on Linux/BSD and others, in `%TEMP%\\restic` in Windows, etc). **If `rest-server` is launched using the default path, all backups will be lost**. To start the server with a custom persistence directory and with authentication disabled:

changelog/unreleased/pull-197

@@ -0,0 +1,17 @@
+Enhancement: Prometheus metric for repository's last update
+
+A new gauge metric `rest_server_repo_last_update_timestamp` was added to
+monitor each repository's last write access. This allows a basic
+monitoring for each repository's freshness.
+
+This metric can be configured as an alerting rule. For example, to be
+notified if some repository is older than two days:
+> time() - rest_server_repo_last_update_timestamp >= 172800
+
+In order to have this metric available at startup, a basic preloading for
+Prometheus metrics has been implemented. This operates by scanning the file
+system for restic repositories and using their last modified time.
+Subsequently, each write access updates the last update time.
+
+If scanning each repository takes too long, it can be disabled through the
+`--prometheus-no-preload` flag.

cmd/rest-server/main.go

@@ -54,6 +54,7 @@ func init() {
 	flags.BoolVar(&server.PrivateRepos, "private-repos", server.PrivateRepos, "users can only access their private repo")
 	flags.BoolVar(&server.Prometheus, "prometheus", server.Prometheus, "enable Prometheus metrics")
 	flags.BoolVar(&server.PrometheusNoAuth, "prometheus-no-auth", server.PrometheusNoAuth, "disable auth for Prometheus /metrics endpoint")
+	flags.BoolVar(&server.PrometheusNoPreload, "prometheus-no-preload", server.PrometheusNoPreload, "disable preloading Prometheus metrics during startup")
 }
 
 var version = "0.11.0"
@@ -126,6 +127,12 @@ func runRoot(cmd *cobra.Command, args []string) error {
 		log.Println("Private repositories disabled")
 	}
 
+	if server.Prometheus && !server.PrometheusNoPreload {
+		if err := server.PreloadMetrics(); err != nil {
+			return fmt.Errorf("unable to preload metrics: %w", err)
+		}
+	}
+
 	enabledTLS, privateKey, publicKey, err := tlsSettings()
 	if err != nil {
 		return err

handlers.go

@@ -2,8 +2,11 @@ package restserver
 
 import (
 	"errors"
+	"fmt"
+	"io/fs"
 	"log"
 	"net/http"
+	"os"
 	"path"
 	"path/filepath"
 	"strings"
@@ -14,23 +17,24 @@ import (
 
 // Server encapsulates the rest-server's settings and repo management logic
 type Server struct {
-	Path             string
-	HtpasswdPath     string
-	Listen           string
-	Log              string
-	CPUProfile       string
-	TLSKey           string
-	TLSCert          string
-	TLS              bool
-	NoAuth           bool
-	AppendOnly       bool
-	PrivateRepos     bool
-	Prometheus       bool
-	PrometheusNoAuth bool
-	Debug            bool
-	MaxRepoSize      int64
-	PanicOnError     bool
-	NoVerifyUpload   bool
+	Path                string
+	HtpasswdPath        string
+	Listen              string
+	Log                 string
+	CPUProfile          string
+	TLSKey              string
+	TLSCert             string
+	TLS                 bool
+	NoAuth              bool
+	AppendOnly          bool
+	PrivateRepos        bool
+	Prometheus          bool
+	PrometheusNoAuth    bool
+	PrometheusNoPreload bool
+	Debug               bool
+	MaxRepoSize         int64
+	PanicOnError        bool
+	NoVerifyUpload      bool
 
 	htpasswdFile *HtpasswdFile
 	quotaManager *quota.Manager
@@ -46,6 +50,98 @@ func httpDefaultError(w http.ResponseWriter, code int) {
 	http.Error(w, http.StatusText(code), code)
 }
 
+// PreloadMetrics for Prometheus for each available repository.
+func (s *Server) PreloadMetrics() error {
+	// No need to preload metrics if those are disabled.
+	if !s.Prometheus || s.PrometheusNoPreload {
+		return nil
+	}
+
+	if _, statErr := os.Lstat(s.Path); errors.Is(statErr, os.ErrNotExist) {
+		log.Print("PreloadMetrics: skipping preloading as repo does not exists yet")
+		return nil
+	}
+
+	var repoPaths []string
+
+	walkFunc := func(path string, d fs.DirEntry, err error) error {
+		if err != nil {
+			return err
+		}
+
+		if !d.IsDir() {
+			return nil
+		}
+
+		// Verify that we're in an allowed directory.
+		for _, objectType := range repo.ObjectTypes {
+			if d.Name() == objectType {
+				return filepath.SkipDir
+			}
+		}
+
+		// Verify that we're also a valid repository.
+		for _, objectType := range repo.ObjectTypes {
+			stat, statErr := os.Lstat(filepath.Join(path, objectType))
+			if errors.Is(statErr, os.ErrNotExist) || !stat.IsDir() {
+				if s.Debug {
+					log.Printf("PreloadMetrics: %s misses directory %s; skip", path, objectType)
+				}
+				return nil
+			}
+		}
+		for _, fileType := range repo.FileTypes {
+			stat, statErr := os.Lstat(filepath.Join(path, fileType))
+			if errors.Is(statErr, os.ErrNotExist) || !stat.Mode().IsRegular() {
+				if s.Debug {
+					log.Printf("PreloadMetrics: %s misses file %s; skip", path, fileType)
+				}
+				return nil
+			}
+		}
+
+		if s.Debug {
+			log.Printf("PreloadMetrics: found repository %s", path)
+		}
+		repoPaths = append(repoPaths, path)
+		return nil
+	}
+
+	if err := filepath.WalkDir(s.Path, walkFunc); err != nil {
+		return err
+	}
+
+	for _, repoPath := range repoPaths {
+		// Remove leading path prefix.
+		relPath := repoPath[len(s.Path):]
+		if strings.HasPrefix(relPath, string(os.PathSeparator)) {
+			relPath = relPath[1:]
+		}
+		folderPath := strings.Split(relPath, string(os.PathSeparator))
+
+		if !folderPathValid(folderPath) {
+			return fmt.Errorf("invalid foder path %s for preloading",
+				strings.Join(folderPath, string(os.PathSeparator)))
+		}
+
+		opt := repo.Options{
+			Debug:          s.Debug,
+			PanicOnError:   s.PanicOnError,
+			BlobMetricFunc: makeBlobMetricFunc("", folderPath),
+		}
+
+		handler, err := repo.New(repoPath, opt)
+		if err != nil {
+			return err
+		}
+
+		if err := handler.PreloadMetrics(); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
 // ServeHTTP makes this server an http.Handler. It handlers the administrative
 // part of the request (figuring out the filesystem location, performing
 // authentication, etc) and then passes it on to repo.Handler for actual

metrics.go

@@ -2,6 +2,7 @@ package restserver
 
 import (
 	"strings"
+	"time"
 
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/restic/rest-server/repo"
@@ -57,25 +58,39 @@ var metricBlobDeleteBytesTotal = prometheus.NewCounterVec(
 	metricLabelList,
 )
 
+var metricRepoLastUpdateTimestamp = prometheus.NewGaugeVec(
+	prometheus.GaugeOpts{
+		Name: "rest_server_repo_last_update_timestamp",
+		Help: "Unix timestamp of repository's last write update",
+	},
+	[]string{"repo"},
+)
+
 // makeBlobMetricFunc creates a metrics callback function that increments the
 // Prometheus metrics.
 func makeBlobMetricFunc(username string, folderPath []string) repo.BlobMetricFunc {
-	var f repo.BlobMetricFunc = func(objectType string, operation repo.BlobOperation, nBytes uint64) {
+	var f repo.BlobMetricFunc = func(objectType string, operation repo.BlobOperation, payload uint64) {
+		repoPath := strings.Join(folderPath, "/")
 		labels := prometheus.Labels{
 			"user": username,
-			"repo": strings.Join(folderPath, "/"),
+			"repo": repoPath,
 			"type": objectType,
 		}
+
 		switch operation {
 		case repo.BlobRead:
 			metricBlobReadTotal.With(labels).Inc()
-			metricBlobReadBytesTotal.With(labels).Add(float64(nBytes))
+			metricBlobReadBytesTotal.With(labels).Add(float64(payload))
 		case repo.BlobWrite:
 			metricBlobWriteTotal.With(labels).Inc()
-			metricBlobWriteBytesTotal.With(labels).Add(float64(nBytes))
+			metricBlobWriteBytesTotal.With(labels).Add(float64(payload))
+			metricRepoLastUpdateTimestamp.WithLabelValues(repoPath).Set(
+				float64(time.Now().UnixMilli()) / 1000.0)
 		case repo.BlobDelete:
 			metricBlobDeleteTotal.With(labels).Inc()
-			metricBlobDeleteBytesTotal.With(labels).Add(float64(nBytes))
+			metricBlobDeleteBytesTotal.With(labels).Add(float64(payload))
+		case repo.RepoPreloadLastUpdate:
+			metricRepoLastUpdateTimestamp.WithLabelValues(repoPath).Set(float64(payload) / 1000.0)
 		}
 	}
 	return f
@@ -89,4 +104,5 @@ func init() {
 	prometheus.MustRegister(metricBlobReadBytesTotal)
 	prometheus.MustRegister(metricBlobDeleteTotal)
 	prometheus.MustRegister(metricBlobDeleteBytesTotal)
+	prometheus.MustRegister(metricRepoLastUpdateTimestamp)
 }

repo/repo.go

@@ -113,6 +113,8 @@ const (
 	BlobRead   = 'R' // A blob has been read
 	BlobWrite  = 'W' // A blob has been written
 	BlobDelete = 'D' // A blob has been deleted
+
+	RepoPreloadLastUpdate = 'U' // Set last update timestamp for preloading
 )
 
 // BlobMetricFunc is the callback signature for blob metrics. Such a callback
@@ -123,6 +125,21 @@ const (
 // TODO: Perhaps add http.Request for the username so that this can be cached?
 type BlobMetricFunc func(objectType string, operation BlobOperation, nBytes uint64)
 
+// PreloadMetrics for Prometheus.
+func (h *Handler) PreloadMetrics() error {
+	if h.opt.Debug {
+		log.Printf("%v.PreloadMetrics()", h)
+	}
+
+	stat, err := os.Lstat(h.getSubPath("snapshots"))
+	if err != nil {
+		return err
+	}
+	h.sendMetric("", RepoPreloadLastUpdate, uint64(stat.ModTime().UnixMilli()))
+
+	return nil
+}
+
 // ServeHTTP performs strict matching on the repo part of the URL path and
 // dispatches the request to the appropriate handler.
 func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {