From 92b9bdef144e2755464599234ce7e1e34b833a76 Mon Sep 17 00:00:00 2001
From: Edd Robinson <me@edd.io>
Date: Sun, 17 Nov 2019 14:44:03 +0000
Subject: [PATCH] feat(inspect): add verify-tombstone

---
 cmd/influx_inspect/main.go                    |   8 +-
 cmd/influx_inspect/verify/tombstone/verify.go | 145 ++++++++++++++++++
 2 files changed, 152 insertions(+), 1 deletion(-)
 create mode 100644 cmd/influx_inspect/verify/tombstone/verify.go

diff --git a/cmd/influx_inspect/main.go b/cmd/influx_inspect/main.go
index 38ff9662aed..d1994c601fc 100644
--- a/cmd/influx_inspect/main.go
+++ b/cmd/influx_inspect/main.go
@@ -18,6 +18,7 @@ import (
 	"github.com/influxdata/influxdb/cmd/influx_inspect/report"
 	"github.com/influxdata/influxdb/cmd/influx_inspect/reporttsi"
 	"github.com/influxdata/influxdb/cmd/influx_inspect/verify/seriesfile"
+	"github.com/influxdata/influxdb/cmd/influx_inspect/verify/tombstone"
 	"github.com/influxdata/influxdb/cmd/influx_inspect/verify/tsm"
 	_ "github.com/influxdata/influxdb/tsdb/engine"
 )
@@ -112,8 +113,13 @@ func (m *Main) Run(args ...string) error {
 		if err := name.Run(args...); err != nil {
 			return fmt.Errorf("verify-seriesfile: %s", err)
 		}
+	case "verify-tombstone":
+		name := tombstone.NewCommand()
+		if err := name.Run(args...); err != nil {
+			return fmt.Errorf("verify-seriesfile: %s", err)
+		}
 	default:
-		return fmt.Errorf(`unknown command "%s"`+"\n"+`Run 'influx_inspect help' for usage`+"\n\n", name)
+		return fmt.Errorf(`unknown command "%s"`+"\n"+`Run 'influx_inspect help' for usage`, name)
 	}
 
 	return nil
diff --git a/cmd/influx_inspect/verify/tombstone/verify.go b/cmd/influx_inspect/verify/tombstone/verify.go
new file mode 100644
index 00000000000..7b1bcf3a4b4
--- /dev/null
+++ b/cmd/influx_inspect/verify/tombstone/verify.go
@@ -0,0 +1,145 @@
+// Package tsm verifies integrity of TSM files.
+package tombstone
+
+import (
+	"errors"
+	"flag"
+	"fmt"
+	"io"
+	"os"
+	"path/filepath"
+	"time"
+
+	"github.com/influxdata/influxdb/tsdb/engine/tsm1"
+)
+
+// Command represents the program execution for "influx_inspect verify-tombstone".
+type Command struct {
+	Stderr io.Writer
+	Stdout io.Writer
+}
+
+// NewCommand returns a new instance of Command.
+func NewCommand() *Command {
+	return &Command{
+		Stderr: os.Stderr,
+		Stdout: os.Stdout,
+	}
+}
+
+// Run executes the command.
+func (cmd *Command) Run(args ...string) error {
+	runner := verifier{w: cmd.Stdout}
+	fs := flag.NewFlagSet("verify-tombstone", flag.ExitOnError)
+	fs.StringVar(&runner.path, "path", os.Getenv("HOME")+"/.influxdb", "path to find tombstone files")
+	v := fs.Bool("v", false, "verbose: emit periodic progress")
+	vv := fs.Bool("vv", false, "very verbose: emit every tombstone entry key and time range")
+	vvv := fs.Bool("vvv", false, "very very verbose: emit every tombstone entry key and RFC3339Nano time range")
+
+	fs.SetOutput(cmd.Stdout)
+
+	if err := fs.Parse(args); err != nil {
+		return err
+	}
+
+	if *v {
+		if *vv || *vvv {
+			return errors.New("cannot set multiple verbosity levels")
+		}
+		runner.verbosity = verbose
+	} else if *vv && *vvv {
+		return errors.New("cannot set multiple verbosity levels")
+	} else if *vv {
+		runner.verbosity = veryVerbose
+	} else if *vvv {
+		runner.verbosity = veryVeryVerbose
+	}
+
+	return runner.Run()
+}
+
+const (
+	quiet = iota
+	verbose
+	veryVerbose
+	veryVeryVerbose
+)
+
+type verifier struct {
+	path      string
+	verbosity int
+
+	w     io.Writer
+	files []string
+	f     string
+}
+
+func (v *verifier) loadFiles() error {
+	return filepath.Walk(v.path, func(path string, f os.FileInfo, err error) error {
+		if err != nil {
+			return err
+		}
+		if filepath.Ext(path) == "."+tsm1.TombstoneFileExtension {
+			v.files = append(v.files, path)
+		}
+		return nil
+	})
+}
+
+func (v *verifier) Next() bool {
+	if len(v.files) == 0 {
+		return false
+	}
+
+	v.f, v.files = v.files[0], v.files[1:]
+	return true
+}
+
+func (v *verifier) Run() error {
+	if err := v.loadFiles(); err != nil {
+		return err
+	}
+
+	var failed bool
+	start := time.Now()
+	for v.Next() {
+		if v.verbosity > quiet {
+			fmt.Fprintf(v.w, "Verifying: %q\n", v.f)
+		}
+
+		tombstoner := tsm1.NewTombstoner(v.f, nil)
+		if !tombstoner.HasTombstones() {
+			fmt.Fprintf(v.w, "%s has no tombstone entries", v.f)
+			continue
+		}
+
+		var totalEntries int64
+		err := tombstoner.Walk(func(t tsm1.Tombstone) error {
+			totalEntries++
+			if v.verbosity > quiet && totalEntries%(10*1e6) == 0 {
+				fmt.Fprintf(v.w, "Verified %d tombstone entries\n", totalEntries)
+			} else if v.verbosity > verbose {
+				var min interface{} = t.Min
+				var max interface{} = t.Max
+				if v.verbosity > veryVerbose {
+					min = time.Unix(0, t.Min)
+					max = time.Unix(0, t.Max)
+				}
+				fmt.Printf("key: %q, min: %v, max: %v\n", t.Key, min, max)
+			}
+			return nil
+		})
+		if err != nil {
+			fmt.Fprintf(v.w, "%q failed to walk tombstone entries: %v. Last okay entry: %d\n", v.f, err, totalEntries)
+			failed = true
+			continue
+		}
+
+		fmt.Fprintf(v.w, "Completed verification for %q in %v.\nVerified %d entries\n\n", v.f, time.Since(start), totalEntries)
+	}
+
+	if failed {
+		return errors.New("failed tombstone verification")
+	}
+	return nil
+}