fix: for Windows, copy snapshot files being backed up (#22551)

On Windows, make copies of files for snapshots, because
Go does not support the FILE_SHARE_DELETE flag which
allows files (and links) to be deleted while open. This
causes temporary directories to be left behind after
backups.

closes #16289

(cherry picked from commit 3702fe8)

closes #22557

tsdb/engine/tsm1/copy_or_link_unix.go

@@ -0,0 +1,17 @@
+//go:build !windows
+// +build !windows
+
+package tsm1
+
+import (
+	"fmt"
+	"os"
+)
+
+// copyOrLink - allow substitution of a file copy for a hard link when running on Windows systems.
+func copyOrLink(oldPath, newPath string) error {
+	if err := os.Link(oldPath, newPath); err != nil {
+		return fmt.Errorf("error creating hard link for backup from %s to %s: %q", oldPath, newPath, err)
+	}
+	return nil
+}

tsdb/engine/tsm1/copy_or_link_windows.go

@@ -0,0 +1,46 @@
+//go:build windows
+// +build windows
+
+package tsm1
+
+import (
+	"fmt"
+	"io"
+	"os"
+)
+
+// copyOrLink - Windows does not permit deleting a file with open file handles, so
+// instead of hard links, make temporary copies of files that can then be deleted.
+func copyOrLink(oldPath, newPath string) (returnErr error) {
+	rfd, err := os.Open(oldPath)
+	if err != nil {
+		return fmt.Errorf("error opening file for backup %s: %q", oldPath, err)
+	} else {
+		defer func() {
+			if e := rfd.Close(); returnErr == nil && e != nil {
+				returnErr = fmt.Errorf("error closing source file for backup %s: %q", oldPath, e)
+			}
+		}()
+	}
+	fi, err := rfd.Stat()
+	if err != nil {
+		fmt.Errorf("error collecting statistics from file for backup %s: %q", oldPath, err)
+	}
+	wfd, err := os.OpenFile(newPath, os.O_RDWR|os.O_CREATE, fi.Mode())
+	if err != nil {
+		return fmt.Errorf("error creating temporary file for backup %s:  %q", newPath, err)
+	} else {
+		defer func() {
+			if e := wfd.Close(); returnErr == nil && e != nil {
+				returnErr = fmt.Errorf("error closing temporary file for backup %s: %q", newPath, e)
+			}
+		}()
+	}
+	if _, err := io.Copy(wfd, rfd); err != nil {
+		return fmt.Errorf("unable to copy file for backup from %s to %s: %q", oldPath, newPath, err)
+	}
+	if err := os.Chtimes(newPath, fi.ModTime(), fi.ModTime()); err != nil {
+		return fmt.Errorf("unable to set modification time on temporary backup file %s: %q", newPath, err)
+	}
+	return nil
+}

tsdb/engine/tsm1/engine_test.go

@@ -478,6 +478,22 @@ func TestEngine_Backup(t *testing.T) {
 	if !strings.Contains(mostRecentFile, th.Name) || th.Name == "" {
 		t.Fatalf("file name doesn't match:\n\tgot: %s\n\texp: %s", th.Name, mostRecentFile)
 	}
+	storeDir := filepath.Dir(e.FileStore.Files()[0].Path())
+	dfd, err := os.Open(storeDir)
+	if err != nil {
+		t.Fatalf("cannot open filestore directory %s: %q", storeDir, err)
+	} else {
+		defer dfd.Close()
+	}
+	files, err := dfd.Readdirnames(0)
+	if err != nil {
+		t.Fatalf("cannot read directory %s: %q", storeDir, err)
+	}
+	for _, f := range files {
+		if strings.HasSuffix(f, tsm1.TmpTSMFileExtension) {
+			t.Fatalf("temporary directory for backup not cleaned up: %s", f)
+		}
+	}
 }
 
 func TestEngine_Export(t *testing.T) {

tsdb/engine/tsm1/file_store.go

@@ -1074,6 +1074,24 @@ func (f *FileStore) locations(key []byte, t int64, ascending bool) []*location {
 	return locations
 }
 
+// MakeSnapshotLinks creates hardlinks from the supplied TSMFiles to
+// corresponding files under a supplied directory.
+func (f *FileStore) MakeSnapshotLinks(destPath string, files []TSMFile) (returnErr error) {
+	for _, tsmf := range files {
+		newpath := filepath.Join(destPath, filepath.Base(tsmf.Path()))
+		if err := copyOrLink(tsmf.Path(), newpath); err != nil {
+			return err
+		}
+		if tf := tsmf.TombstoneStats(); tf.TombstoneExists {
+			newpath := filepath.Join(destPath, filepath.Base(tf.Path))
+			if err := copyOrLink(tf.Path, newpath); err != nil {
+				return err
+			}
+		}
+	}
+	return nil
+}
+
 // CreateSnapshot creates hardlinks for all tsm and tombstone files
 // in the path provided.
 func (f *FileStore) CreateSnapshot() (string, error) {
@@ -1103,17 +1121,10 @@ func (f *FileStore) CreateSnapshot() (string, error) {
 	if err != nil {
 		return "", err
 	}
-	for _, tsmf := range files {
-		newpath := filepath.Join(tmpPath, filepath.Base(tsmf.Path()))
-		if err := os.Link(tsmf.Path(), newpath); err != nil {
-			return "", fmt.Errorf("error creating tsm hard link: %q", err)
-		}
-		if ts := tsmf.TombstoneStats(); ts.TombstoneExists {
-			newpath := filepath.Join(tmpPath, filepath.Base(ts.Path))
-			if err := os.Link(ts.Path, newpath); err != nil {
-				return "", fmt.Errorf("error creating tombstone hard link: %q", err)
-			}
-		}
+	if err := f.MakeSnapshotLinks(tmpPath, files); err != nil {
+		// remove temporary directory since we couldn't create our hard links.
+		_ = os.RemoveAll(tmpPath)
+		return "", fmt.Errorf("CreateSnapshot() failed to create links %v: %w", tmpPath, err)
 	}
 
 	return tmpPath, nil