Merge pull request #7862 from influxdata/er-debug-all

Adds handler for returning a profile archive

.github/ISSUE_TEMPLATE.md

@@ -25,17 +25,19 @@ __Additional info:__ [Include gist of relevant config, logs, etc.]
 Also, if this is an issue of for performance, locking, etc the following commands are useful to create debug information for the team.
 
 ```
-curl -o block.txt "http://localhost:8086/debug/pprof/block?debug=1" 
-curl -o goroutine.txt "http://localhost:8086/debug/pprof/goroutine?debug=1" 
-curl -o heap.txt "http://localhost:8086/debug/pprof/heap?debug=1" 
-curl -o vars.txt "http://localhost:8086/debug/vars" 
+curl -o profiles.tar.gz "http://localhost:8086/debug/pprof/all?cpu=true"
+
+curl -o vars.txt "http://localhost:8086/debug/vars"
 iostat -xd 1 30 > iostat.txt
-influx -execute "show shards" > shards.txt
-influx -execute "show stats" > stats.txt
-influx -execute "show diagnostics" > diagnostics.txt
 ```
 
-Please run those if possible and link them from a [gist](http://gist.github.com).
+**Please note** It will take at least 30 seconds for the first cURL command above to return a response.
+This is because it will run a CPU profile as part of its information gathering, which takes 30 seconds to collect.
+Ideally you should run these commands when you're experiencing problems, so we can capture the state of the system at that time.
+
+If you're concerned about running a CPU profile (which only has a small, temporary impact on performance), then you can set `?cpu=false` or omit `?cpu=true` altogether.
+
+Please run those if possible and link them from a [gist](http://gist.github.com) or simply attach them as a comment to the issue.
 
 *Please note, the quickest way to fix a bug is to open a Pull Request.*
 

CHANGELOG.md

@@ -30,7 +30,9 @@ The admin UI is removed and unusable in this release. The `[admin]` configuratio
 - [#8348](https://github.com/influxdata/influxdb/pull/8348): Add max concurrent compaction limits
 - [#8366](https://github.com/influxdata/influxdb/pull/8366): Add TSI support tooling.
 - [#8350](https://github.com/influxdata/influxdb/pull/8350): Track HTTP client requests for /write and /query with /debug/requests.
+<<<<<<< 8f8ff0ec612e6e77ff618f572b302e069de8e5c3
 - [#8384](https://github.com/influxdata/influxdb/pull/8384): Write and compaction stability
+- [#7862](https://github.com/influxdata/influxdb/pull/7861): Add new profile endpoint for gathering all debug profiles and querues in single archive.
 
 ### Bugfixes
 

internal/meta_client.go

@@ -34,6 +34,8 @@ type MetaClientMock struct {
 
 	RetentionPolicyFn func(database, name string) (rpi *meta.RetentionPolicyInfo, err error)
 
+	AuthenticateFn           func(username, password string) (ui *meta.UserInfo, err error)
+	AdminUserExistsFn        func() bool
 	SetAdminPrivilegeFn      func(username string, admin bool) error
 	SetDataFn                func(*meta.Data) error
 	SetPrivilegeFn           func(username, database string, p influxql.Privilege) error
@@ -43,6 +45,7 @@ type MetaClientMock struct {
 	UpdateUserFn             func(name, password string) error
 	UserPrivilegeFn          func(username, database string) (*influxql.Privilege, error)
 	UserPrivilegesFn         func(username string) (map[string]influxql.Privilege, error)
+	UserFn                   func(username string) (*meta.UserInfo, error)
 	UsersFn                  func() []meta.UserInfo
 }
 
@@ -150,7 +153,13 @@ func (c *MetaClientMock) UserPrivileges(username string) (map[string]influxql.Pr
 	return c.UserPrivilegesFn(username)
 }
 
-func (c *MetaClientMock) Users() []meta.UserInfo { return c.UsersFn() }
+func (c *MetaClientMock) Authenticate(username, password string) (*meta.UserInfo, error) {
+	return c.AuthenticateFn(username, password)
+}
+func (c *MetaClientMock) AdminUserExists() bool { return c.AdminUserExistsFn() }
+
+func (c *MetaClientMock) User(username string) (*meta.UserInfo, error) { return c.UserFn(username) }
+func (c *MetaClientMock) Users() []meta.UserInfo                       { return c.UsersFn() }
 
 func (c *MetaClientMock) Open() error                { return c.OpenFn() }
 func (c *MetaClientMock) Data() meta.Data            { return c.DataFn() }

services/httpd/handler.go

@@ -10,7 +10,6 @@ import (
 	"io"
 	"log"
 	"net/http"
-	"net/http/pprof"
 	"os"
 	"runtime/debug"
 	"strconv"
@@ -75,6 +74,7 @@ type Handler struct {
 
 	MetaClient interface {
 		Database(name string) *meta.DatabaseInfo
+		Databases() []meta.DatabaseInfo
 		Authenticate(username, password string) (ui *meta.UserInfo, err error)
 		User(username string) (*meta.UserInfo, error)
 		AdminUserExists() bool
@@ -252,16 +252,7 @@ func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 	w.Header().Add("X-Influxdb-Version", h.Version)
 
 	if strings.HasPrefix(r.URL.Path, "/debug/pprof") && h.Config.PprofEnabled {
-		switch r.URL.Path {
-		case "/debug/pprof/cmdline":
-			pprof.Cmdline(w, r)
-		case "/debug/pprof/profile":
-			pprof.Profile(w, r)
-		case "/debug/pprof/symbol":
-			pprof.Symbol(w, r)
-		default:
-			pprof.Index(w, r)
-		}
+		h.handleProfiles(w, r)
 	} else if strings.HasPrefix(r.URL.Path, "/debug/vars") {
 		h.serveExpvar(w, r)
 	} else if strings.HasPrefix(r.URL.Path, "/debug/requests") {

services/httpd/handler_test.go

@@ -14,6 +14,8 @@ import (
 	"testing"
 	"time"
 
+	"github.com/influxdata/influxdb/internal"
+
 	"github.com/dgrijalva/jwt-go"
 	"github.com/influxdata/influxdb/influxql"
 	"github.com/influxdata/influxdb/models"
@@ -605,7 +607,7 @@ func TestHandler_XForwardedFor(t *testing.T) {
 // NewHandler represents a test wrapper for httpd.Handler.
 type Handler struct {
 	*httpd.Handler
-	MetaClient        HandlerMetaStore
+	MetaClient        *internal.MetaClientMock
 	StatementExecutor HandlerStatementExecutor
 	QueryAuthorizer   HandlerQueryAuthorizer
 }
@@ -619,47 +621,17 @@ func NewHandler(requireAuthentication bool) *Handler {
 	h := &Handler{
 		Handler: httpd.NewHandler(config),
 	}
-	h.Handler.MetaClient = &h.MetaClient
+
+	h.MetaClient = &internal.MetaClientMock{}
+
+	h.Handler.MetaClient = h.MetaClient
 	h.Handler.QueryExecutor = influxql.NewQueryExecutor()
 	h.Handler.QueryExecutor.StatementExecutor = &h.StatementExecutor
 	h.Handler.QueryAuthorizer = &h.QueryAuthorizer
 	h.Handler.Version = "0.0.0"
 	return h
 }
 
-// HandlerMetaStore is a mock implementation of Handler.MetaClient.
-type HandlerMetaStore struct {
-	PingFn            func(d time.Duration) error
-	DatabaseFn        func(name string) *meta.DatabaseInfo
-	AuthenticateFn    func(username, password string) (ui *meta.UserInfo, err error)
-	UserFn            func(username string) (*meta.UserInfo, error)
-	AdminUserExistsFn func() bool
-}
-
-func (s *HandlerMetaStore) Ping(b bool) error {
-	if s.PingFn == nil {
-		// Default behaviour is to assume there is a leader.
-		return nil
-	}
-	return s.Ping(b)
-}
-
-func (s *HandlerMetaStore) Database(name string) *meta.DatabaseInfo {
-	return s.DatabaseFn(name)
-}
-
-func (s *HandlerMetaStore) Authenticate(username, password string) (ui *meta.UserInfo, err error) {
-	return s.AuthenticateFn(username, password)
-}
-
-func (s *HandlerMetaStore) AdminUserExists() bool {
-	return s.AdminUserExistsFn()
-}
-
-func (s *HandlerMetaStore) User(username string) (*meta.UserInfo, error) {
-	return s.UserFn(username)
-}
-
 // HandlerStatementExecutor is a mock implementation of Handler.StatementExecutor.
 type HandlerStatementExecutor struct {
 	ExecuteStatementFn func(stmt influxql.Statement, ctx influxql.ExecutionContext) error