Implement server stats and self-monitoring

CHANGELOG.md

@@ -9,6 +9,9 @@
 - [#1955](https://github.com/influxdb/influxdb/pull/1955): Prohibit creation of databases with no name. Thanks @dullgiulio
 - [#1952](https://github.com/influxdb/influxdb/pull/1952): Handle delete statement with an error. Thanks again to @dullgiulio
 
+### Features
+- [#1936](https://github.com/influxdb/influxdb/pull/1936): Implement "SHOW STATS" and self-monitoring
+
 ## v0.9.0-rc11 [2015-03-13]
 
 ### Bugfixes

cmd/influxd/config.go

@@ -110,6 +110,13 @@ type Config struct {
 		RaftTracing  bool   `toml:"raft-tracing"`
 	} `toml:"logging"`
 
+	Statistics struct {
+		Enabled         bool     `toml:"enabled"`
+		Database        string   `toml:"database"`
+		RetentionPolicy string   `toml:"retention-policy"`
+		WriteInterval   Duration `toml:"write-interval"`
+	}
+
 	ContinuousQuery struct {
 		// when continuous queries are run we'll automatically recompute previous intervals
 		// in case lagged data came in. Set to zero if you never have lagged data. We do
@@ -166,6 +173,11 @@ func NewConfig() *Config {
 	c.ContinuousQuery.Disable = false
 	c.ReportingDisabled = false
 
+	c.Statistics.Enabled = false
+	c.Statistics.Database = "_internal"
+	c.Statistics.RetentionPolicy = "default"
+	c.Statistics.WriteInterval = Duration(1 * time.Minute)
+
 	// Detect hostname (or set to localhost).
 	if c.Hostname, _ = os.Hostname(); c.Hostname == "" {
 		c.Hostname = "localhost"

cmd/influxd/config_test.go

@@ -177,6 +177,12 @@ file   = "influxdb.log"
 write-tracing = true
 raft-tracing = true
 
+[statistics]
+enabled = true
+database = "_internal"
+retention-policy = "default"
+write-interval = "1m"
+
 # Configure the admin server
 [admin]
 enabled = true

cmd/influxd/run.go

@@ -180,6 +180,27 @@ func Run(config *Config, join, version string, logWriter *os.File) (*messaging.B
 				log.Fatalf("failed to start %s Graphite server: %s", c.Protocol, err.Error())
 			}
 		}
+
+		// Start up self-monitoring if enabled.
+		if config.Statistics.Enabled {
+			database := config.Statistics.Database
+			policy := config.Statistics.RetentionPolicy
+			interval := time.Duration(config.Statistics.WriteInterval)
+
+			// Ensure database exists.
+			if err := s.CreateDatabaseIfNotExists(database); err != nil {
+				log.Fatalf("failed to create database %s for internal statistics: %s", database, err.Error())
+			}
+
+			// Ensure retention policy exists.
+			rp := influxdb.NewRetentionPolicy(policy)
+			if err := s.CreateRetentionPolicyIfNotExists(database, rp); err != nil {
+				log.Fatalf("failed to create retention policy for internal statistics: %s", err.Error())
+			}
+
+			s.StartSelfMonitoring(database, policy, interval)
+			log.Printf("started self-monitoring at interval of %s", interval)
+		}
 	}
 
 	// unless disabled, start the loop to report anonymous usage stats every 24h

etc/config.sample.toml

@@ -88,3 +88,12 @@ dir = "/tmp/influxdb/development/state"
 file   = "/var/log/influxdb/influxd.log" # Leave blank to redirect logs to stderr.
 write-tracing = false # If true, enables detailed logging of the write system.
 raft-tracing = false # If true, enables detailed logging of Raft consensus.
+
+# InfluxDB can store statistics about itself. This is useful for monitoring purposes.
+# This feature is disabled by default, but if enabled, these statistics can be queried
+# as any other data.
+[statistics]
+enabled = false
+database = "_internal"         # The database to which the data is written.
+retention-policy = "default"   # The retention policy within the database.
+write-interval = "1m"          # Period between writing the data.

influxql/ast.go

@@ -78,6 +78,7 @@ func (*ShowFieldKeysStatement) node()         {}
 func (*ShowRetentionPoliciesStatement) node() {}
 func (*ShowMeasurementsStatement) node()      {}
 func (*ShowSeriesStatement) node()            {}
+func (*ShowStatsStatement) node()             {}
 func (*ShowTagKeysStatement) node()           {}
 func (*ShowTagValuesStatement) node()         {}
 func (*ShowUsersStatement) node()             {}
@@ -169,6 +170,7 @@ func (*ShowFieldKeysStatement) stmt()         {}
 func (*ShowMeasurementsStatement) stmt()      {}
 func (*ShowRetentionPoliciesStatement) stmt() {}
 func (*ShowSeriesStatement) stmt()            {}
+func (*ShowStatsStatement) stmt()             {}
 func (*ShowTagKeysStatement) stmt()           {}
 func (*ShowTagValuesStatement) stmt()         {}
 func (*ShowUsersStatement) stmt()             {}
@@ -1365,6 +1367,27 @@ func (s *ShowRetentionPoliciesStatement) RequiredPrivileges() ExecutionPrivilege
 	return ExecutionPrivileges{{Name: "", Privilege: ReadPrivilege}}
 }
 
+// ShowRetentionPoliciesStatement represents a command for displaying stats for a given server.
+type ShowStatsStatement struct {
+	// Hostname or IP of the server for stats.
+	Host string
+}
+
+// String returns a string representation of a ShowStatsStatement.
+func (s *ShowStatsStatement) String() string {
+	var buf bytes.Buffer
+	_, _ = buf.WriteString("SHOW STATS ")
+	if s.Host != "" {
+		_, _ = buf.WriteString(s.Host)
+	}
+	return buf.String()
+}
+
+// RequiredPrivileges returns the privilege(s) required to execute a ShowStatsStatement
+func (s *ShowStatsStatement) RequiredPrivileges() ExecutionPrivileges {
+	return ExecutionPrivileges{{Name: "", Privilege: AllPrivileges}}
+}
+
 // ShowTagKeysStatement represents a command for listing tag keys.
 type ShowTagKeysStatement struct {
 	// Data source that fields are extracted from.

influxql/parser.go

@@ -119,6 +119,8 @@ func (p *Parser) parseShowStatement() (Statement, error) {
 		return nil, newParseError(tokstr(tok, lit), []string{"POLICIES"}, pos)
 	case SERIES:
 		return p.parseShowSeriesStatement()
+	case STATS:
+		return p.parseShowStatsStatement()
 	case TAG:
 		tok, pos, lit := p.scanIgnoreWhitespace()
 		if tok == KEYS {
@@ -1172,6 +1174,21 @@ func (p *Parser) parseRetentionPolicy() (name string, dfault bool, err error) {
 	return
 }
 
+// parseShowStatsStatement parses a string and returns a ShowStatsStatement.
+// This function assumes the "SHOW STATS" tokens have already been consumed.
+func (p *Parser) parseShowStatsStatement() (*ShowStatsStatement, error) {
+	stmt := &ShowStatsStatement{}
+	var err error
+
+	if tok, _, _ := p.scanIgnoreWhitespace(); tok == ON {
+		stmt.Host, err = p.parseString()
+	} else {
+		p.unscan()
+	}
+
+	return stmt, err
+}
+
 // parseDropContinuousQueriesStatement parses a string and returns a DropContinuousQueryStatement.
 // This function assumes the "DROP CONTINUOUS" tokens have already been consumed.
 func (p *Parser) parseDropContinuousQueryStatement() (*DropContinuousQueryStatement, error) {

influxql/parser_test.go

@@ -769,6 +769,26 @@ func TestParser_ParseStatement(t *testing.T) {
 			stmt: newAlterRetentionPolicyStatement("policy1", "testdb", -1, 4, false),
 		},
 
+		// SHOW STATS
+		{
+			s: `SHOW STATS`,
+			stmt: &influxql.ShowStatsStatement{
+				Host: "",
+			},
+		},
+		{
+			s: `SHOW STATS ON 'servera'`,
+			stmt: &influxql.ShowStatsStatement{
+				Host: "servera",
+			},
+		},
+		{
+			s: `SHOW STATS ON '192.167.1.44'`,
+			stmt: &influxql.ShowStatsStatement{
+				Host: "192.167.1.44",
+			},
+		},
+
 		// Errors
 		{s: ``, err: `found EOF, expected SELECT at line 1, char 1`},
 		{s: `SELECT`, err: `found EOF, expected identifier, string, number, bool at line 1, char 8`},
@@ -798,6 +818,7 @@ func TestParser_ParseStatement(t *testing.T) {
 		{s: `SHOW RETENTION`, err: `found EOF, expected POLICIES at line 1, char 16`},
 		{s: `SHOW RETENTION POLICIES`, err: `found EOF, expected identifier at line 1, char 25`},
 		{s: `SHOW FOO`, err: `found FOO, expected CONTINUOUS, DATABASES, FIELD, MEASUREMENTS, RETENTION, SERIES, SERVERS, TAG, USERS at line 1, char 6`},
+		{s: `SHOW STATS ON`, err: `found EOF, expected string at line 1, char 15`},
 		{s: `DROP CONTINUOUS`, err: `found EOF, expected QUERY at line 1, char 17`},
 		{s: `DROP CONTINUOUS QUERY`, err: `found EOF, expected identifier at line 1, char 23`},
 		{s: `CREATE CONTINUOUS`, err: `found EOF, expected QUERY at line 1, char 19`},

influxql/token.go

@@ -106,6 +106,7 @@ const (
 	SERVERS
 	SHOW
 	SLIMIT
+	STATS
 	SOFFSET
 	TAG
 	TO
@@ -208,6 +209,7 @@ var tokens = [...]string{
 	SHOW:         "SHOW",
 	SLIMIT:       "SLIMIT",
 	SOFFSET:      "SOFFSET",
+	STATS:        "STATS",
 	TAG:          "TAG",
 	TO:           "TO",
 	USER:         "USER",