Merge branch '1.8' into seriescache

* 1.8: (41 commits)
  chore: Late to the party fix for influxdata/plutonium#3339 (influxdata#21080)
  fix: fix help test for influx_inspect (influxdata#21052) (influxdata#21053)
  fix: Set go version to 1.13 in go.mod; see influxdata/plutonium#3339 (influxdata#21034)
  refactor: separate coarse and fine permission interfaces (influxdata#20996) (influxdata#21035)
  feat: Log query text for POST requests (influxdata#20993) (influxdata#21021)
  feat: influx_inspect export to standard out (influxdata#20977) (influxdata#20989)
  feat(inspect): Add report-disk for disk usage by measurement (influxdata#20917)
  fix: infinite recursion bug (influxdata#20862) (influxdata#20914)
  fix(tsdb): minimize lock contention when adding new fields or measure (influxdata#20912)
  fix(tsm1): fix data race when accessing tombstone stats (influxdata#20909)
  Update changelog
  feat: Make meta queries respect QueryTimeout values (influxdata#20910)
  chore: run goimports on 1.8 branch to bring it up to new check-in standards (influxdata#20907)
  fix(error): SELECT INTO doesn't return error with unsupported value (influxdata#20429) (influxdata#20432)
  build: switch tested centos base images (influxdata#20417) (influxdata#20418)
  chore: update CHANGELOG.md for typo and community PR (influxdata#20389)
  fix(prometheus): regexp handling should comply with PromQL (influxdata#19832)
  fix: cp.Mux.Serve() closes all net.Listener instances silently on error (influxdata#20295)
  Update changelog
  chore: fix CHANGELOG formating (influxdata#20286)
  ...

# Conflicts:
#	cmd/influxd/run/server.go
#	go.sum
#	services/httpd/service.go
#	storage/reads/datatypes/predicate.pb.go
#	storage/reads/datatypes/storage_common.pb.go

CHANGELOG.md

@@ -1,4 +1,29 @@
-v1.8.4 [unreleased]
+v1.8.5 [unreleased]
+-------------------
+
+### Features
+
+-	[#20917](https://github.com/influxdata/influxdb/pull/20917): feat(inspect): Add report-disk for disk usage by measurement
+-	[#20118](https://github.com/influxdata/influxdb/pull/20118): feat: Optimize shard lookups in groups containing only one shard. Thanks @StoneYunZhao!
+-	[#20910](https://github.com/influxdata/influxdb/pull/20910): feat: Make meta queries respect QueryTimeout values
+-	[#20989](https://github.com/influxdata/influxdb/pull/20989): feat: influx_inspect export to standard out
+-	[#21021](https://github.com/influxdata/influxdb/pull/21021): feat: Log query text for POST requests
+
+### Bugfixes
+
+-	[#21053](https://github.com/influxdata/influxdb/pull/21053): fix: help text for influx_inspect
+-	[#20101](https://github.com/influxdata/influxdb/pull/20101): fix(write): Successful writes increment write error statistics incorrectly.
+-	[#20276](https://github.com/influxdata/influxdb/pull/20276): fix(error): unsupported value: +Inf" error not handled gracefully.
+-	[#20277](https://github.com/influxdata/influxdb/pull/20277): fix(query): Group By queries with offset that crosses a DST boundary can fail.
+-	[#20295](https://github.com/influxdata/influxdb/pull/20295): fix: cp.Mux.Serve() closes all net.Listener instances silently on error.
+-	[#19832](https://github.com/influxdata/influxdb/pull/19832): fix(prometheus): regexp handling should comply with PromQL.
+-	[#20432](https://github.com/influxdata/influxdb/pull/20432): fix(error): SELECT INTO doesn't return error with unsupported value
+-	[#20033](https://github.com/influxdata/influxdb/pull/20033): fix(tsm1): "snapshot in progress" error during backup
+-	[#20909](https://github.com/influxdata/influxdb/pull/20909): fix(tsm1): data race when accessing tombstone stats
+-	[#20912](https://github.com/influxdata/influxdb/pull/20912): fix(tsdb): minimize lock contention when adding new fields or measure
+-	[#20914](https://github.com/influxdata/influxdb/pull/20914): fix: infinite recursion bug (#20862)
+
+v1.8.4 [2021-01-27]
 -------------------
 
 ### Bugfixes

Dockerfile_build_ubuntu64_git

@@ -27,7 +27,7 @@ VOLUME $PROJECT_DIR
 
 
 # Install go
-ENV GO_VERSION 1.12
+ENV GO_VERSION 1.13
 ENV GO_ARCH amd64
 RUN wget --no-verbose https://storage.googleapis.com/golang/go${GO_VERSION}.linux-${GO_ARCH}.tar.gz && \
    tar -C /usr/local/ -xf /go${GO_VERSION}.linux-${GO_ARCH}.tar.gz && \

_tools/tmpl/main.go

@@ -248,4 +248,4 @@ func StripComments(raw []byte) []byte {
 	}
 
 	return buf.Bytes()
-}
+}

_tools/tmpl/main_test.go

@@ -54,4 +54,4 @@ func TestStripComments(t *testing.T) {
 			}
 		})
 	}
-}
+}

cluster/points_writer.go

@@ -212,7 +212,7 @@ func (w *PointsWriter) MapShards(wp *WritePointsRequest) (*ShardMapping, error)
 	mapping := NewShardMapping()
 	for _, p := range wp.Points {
 		sg := timeRanges[p.Time().Truncate(rp.ShardGroupDuration)]
-		sh := sg.ShardFor(p.HashID())
+		sh := sg.ShardFor(p)
 		mapping.MapPoint(&sh, p)
 	}
 	return mapping, nil

cmd/influx_inspect/export/export.go

@@ -44,6 +44,8 @@ type Command struct {
 	walFiles map[string][]string
 }
 
+const stdoutMark = "-"
+
 // NewCommand returns a new instance of Command.
 func NewCommand() *Command {
 	return &Command{
@@ -56,13 +58,18 @@ func NewCommand() *Command {
 	}
 }
 
+// Are we writing to standard out?
+func (cmd *Command) usingStdOut() bool {
+	return cmd.out == stdoutMark
+}
+
 // Run executes the command.
 func (cmd *Command) Run(args ...string) error {
 	var start, end string
 	fs := flag.NewFlagSet("export", flag.ExitOnError)
 	fs.StringVar(&cmd.dataDir, "datadir", os.Getenv("HOME")+"/.influxdb/data", "Data storage path")
 	fs.StringVar(&cmd.walDir, "waldir", os.Getenv("HOME")+"/.influxdb/wal", "WAL storage path")
-	fs.StringVar(&cmd.out, "out", os.Getenv("HOME")+"/.influxdb/export", "Destination file to export to")
+	fs.StringVar(&cmd.out, "out", os.Getenv("HOME")+"/.influxdb/export", "'-' for standard out or the destination file to export to")
 	fs.StringVar(&cmd.database, "database", "", "Optional: the database to export")
 	fs.StringVar(&cmd.retentionPolicy, "retention", "", "Optional: the retention policy to export (requires -database)")
 	fs.StringVar(&start, "start", "", "Optional: the start time to export (RFC3339 format)")
@@ -205,23 +212,29 @@ func (cmd *Command) writeDDL(mw io.Writer, w io.Writer) error {
 
 func (cmd *Command) writeDML(mw io.Writer, w io.Writer) error {
 	fmt.Fprintln(mw, "# DML")
+	var msgOut io.Writer
+	if cmd.usingStdOut() {
+		msgOut = cmd.Stderr
+	} else {
+		msgOut = cmd.Stdout
+	}
 	for key := range cmd.manifest {
 		keys := strings.Split(key, string(os.PathSeparator))
 		fmt.Fprintf(mw, "# CONTEXT-DATABASE:%s\n", keys[0])
 		fmt.Fprintf(mw, "# CONTEXT-RETENTION-POLICY:%s\n", keys[1])
 		if files, ok := cmd.tsmFiles[key]; ok {
-			fmt.Fprintf(cmd.Stdout, "writing out tsm file data for %s...", key)
+			fmt.Fprintf(msgOut, "writing out tsm file data for %s...", key)
 			if err := cmd.writeTsmFiles(mw, w, files); err != nil {
 				return err
 			}
-			fmt.Fprintln(cmd.Stdout, "complete.")
+			fmt.Fprintln(msgOut, "complete.")
 		}
 		if _, ok := cmd.walFiles[key]; ok {
-			fmt.Fprintf(cmd.Stdout, "writing out wal file data for %s...", key)
+			fmt.Fprintf(msgOut, "writing out wal file data for %s...", key)
 			if err := cmd.writeWALFiles(mw, w, cmd.walFiles[key], key); err != nil {
 				return err
 			}
-			fmt.Fprintln(cmd.Stdout, "complete.")
+			fmt.Fprintln(msgOut, "complete.")
 		}
 	}
 
@@ -254,20 +267,24 @@ func (cmd *Command) writeFull(mw io.Writer, w io.Writer) error {
 }
 
 func (cmd *Command) write() error {
-	// open our output file and create an output buffer
-	f, err := os.Create(cmd.out)
-	if err != nil {
-		return err
+	var w io.Writer
+	if cmd.usingStdOut() {
+		w = cmd.Stdout
+	} else {
+		// open our output file and create an output buffer
+		f, err := os.Create(cmd.out)
+		if err != nil {
+			return err
+		}
+		defer f.Close()
+		w = f
 	}
-	defer f.Close()
-
 	// Because calling (*os.File).Write is relatively expensive,
 	// and we don't *need* to sync to disk on every written line of export,
 	// use a sized buffered writer so that we only sync the file every megabyte.
-	bw := bufio.NewWriterSize(f, 1024*1024)
+	bw := bufio.NewWriterSize(w, 1024*1024)
 	defer bw.Flush()
-
-	var w io.Writer = bw
+	w = bw
 
 	if cmd.compress {
 		gzw := gzip.NewWriter(w)

cmd/influx_inspect/help/help.go

@@ -37,7 +37,8 @@ The commands are:
     export               exports raw data from a shard to line protocol
     buildtsi             generates tsi1 indexes from tsm1 data
     help                 display this help message
-    report               displays a shard level report
+    report               displays a shard level cardinality report
+    report-disk          displays a shard level disk usage report
     verify               verifies integrity of TSM files
     verify-seriesfile    verifies integrity of the Series file
 

cmd/influx_inspect/main.go

@@ -16,6 +16,7 @@ import (
 	"github.com/influxdata/influxdb/cmd/influx_inspect/export"
 	"github.com/influxdata/influxdb/cmd/influx_inspect/help"
 	"github.com/influxdata/influxdb/cmd/influx_inspect/report"
+	"github.com/influxdata/influxdb/cmd/influx_inspect/reportdisk"
 	"github.com/influxdata/influxdb/cmd/influx_inspect/reporttsi"
 	"github.com/influxdata/influxdb/cmd/influx_inspect/verify/seriesfile"
 	"github.com/influxdata/influxdb/cmd/influx_inspect/verify/tombstone"
@@ -98,6 +99,11 @@ func (m *Main) Run(args ...string) error {
 		if err := name.Run(args...); err != nil {
 			return fmt.Errorf("report: %s", err)
 		}
+	case "report-disk":
+		name := reportdisk.NewCommand()
+		if err := name.Run(args...); err != nil {
+			return fmt.Errorf("report: %s", err)
+		}
 	case "reporttsi":
 		name := reporttsi.NewCommand()
 		if err := name.Run(args...); err != nil {

cmd/influx_inspect/report/report.go

@@ -15,6 +15,7 @@ import (
 	"time"
 
 	"github.com/influxdata/influxdb/models"
+	"github.com/influxdata/influxdb/pkg/reporthelper"
 	"github.com/influxdata/influxdb/tsdb/engine/tsm1"
 	"github.com/retailnext/hllpp"
 )
@@ -60,7 +61,7 @@ func (cmd *Command) Run(args ...string) error {
 
 	cmd.dir = fs.Arg(0)
 
-	err := cmd.isShardDir(cmd.dir)
+	err := reporthelper.IsShardDir(cmd.dir)
 	if cmd.detailed && err != nil {
 		return fmt.Errorf("-detailed only supported for shard dirs")
 	}
@@ -79,8 +80,8 @@ func (cmd *Command) Run(args ...string) error {
 
 	minTime, maxTime := int64(math.MaxInt64), int64(math.MinInt64)
 	var fileCount int
-	if err := cmd.walkShardDirs(cmd.dir, func(db, rp, id, path string) error {
-		if cmd.pattern != "" && strings.Contains(path, cmd.pattern) {
+	if err := reporthelper.WalkShardDirs(cmd.dir, func(db, rp, id, path string) error {
+		if cmd.pattern != "" && !strings.Contains(path, cmd.pattern) {
 			return nil
 		}
 
@@ -218,64 +219,6 @@ func sortKeys(vals map[string]counter) (keys []string) {
 	return keys
 }
 
-func (cmd *Command) isShardDir(dir string) error {
-	name := filepath.Base(dir)
-	if id, err := strconv.Atoi(name); err != nil || id < 1 {
-		return fmt.Errorf("not a valid shard dir: %v", dir)
-	}
-
-	return nil
-}
-
-func (cmd *Command) walkShardDirs(root string, fn func(db, rp, id, path string) error) error {
-	type location struct {
-		db, rp, id, path string
-	}
-
-	var dirs []location
-	if err := filepath.Walk(root, func(path string, info os.FileInfo, err error) error {
-		if err != nil {
-			return err
-		}
-
-		if info.IsDir() {
-			return nil
-		}
-
-		if filepath.Ext(info.Name()) == "."+tsm1.TSMFileExtension {
-			shardDir := filepath.Dir(path)
-
-			if err := cmd.isShardDir(shardDir); err != nil {
-				return err
-			}
-			absPath, err := filepath.Abs(path)
-			if err != nil {
-				return err
-			}
-			parts := strings.Split(absPath, string(filepath.Separator))
-			db, rp, id := parts[len(parts)-4], parts[len(parts)-3], parts[len(parts)-2]
-			dirs = append(dirs, location{db: db, rp: rp, id: id, path: path})
-			return nil
-		}
-		return nil
-	}); err != nil {
-		return err
-	}
-
-	sort.Slice(dirs, func(i, j int) bool {
-		a, _ := strconv.Atoi(dirs[i].id)
-		b, _ := strconv.Atoi(dirs[j].id)
-		return a < b
-	})
-
-	for _, shard := range dirs {
-		if err := fn(shard.db, shard.rp, shard.id, shard.path); err != nil {
-			return err
-		}
-	}
-	return nil
-}
-
 // printUsage prints the usage message to STDERR.
 func (cmd *Command) printUsage() {
 	usage := `Displays shard level report.