Merge pull request #662 from kesutton/master

nsqadmin: conf flags for formatting statsd counters and gauges

apps/nsqadmin/main.go

@@ -28,9 +28,11 @@ var (
 	graphiteURL   = flagSet.String("graphite-url", "", "graphite HTTP address")
 	proxyGraphite = flagSet.Bool("proxy-graphite", false, "proxy HTTP requests to graphite")
 
-	useStatsdPrefixes = flagSet.Bool("use-statsd-prefixes", true, "expect statsd prefixed keys in graphite (ie: 'stats.counters.' and 'stats.gauges.')")
-	statsdPrefix      = flagSet.String("statsd-prefix", "nsq.%s", "prefix used for keys sent to statsd (%s for host replacement, must match nsqd)")
-	statsdInterval    = flagSet.Duration("statsd-interval", 60*time.Second, "time interval nsqd is configured to push to statsd (must match nsqd)")
+	useStatsdPrefixes   = flagSet.Bool("use-statsd-prefixes", true, "(Deprecated - Use --statsd-counter-format and --statsd-gauge-format) Expect statsd prefixed keys in graphite (ie: 'stats.counters.' and 'stats.gauges.')")
+	statsdCounterFormat = flagSet.String("statsd-counter-format", "stats.counters.%s.count", "The counter stats key formatting applied by the implementation of statsd. If no formatting is desired, set this to an empty string.")
+	statsdGaugeFormat   = flagSet.String("statsd-gauge-format", "stats.gauges.%s", "The gauge stats key formatting applied by the implementation of statsd. If no formatting is desired, set this to an empty string.")
+	statsdPrefix        = flagSet.String("statsd-prefix", "nsq.%s", "prefix used for keys sent to statsd (%s for host replacement, must match nsqd)")
+	statsdInterval      = flagSet.Duration("statsd-interval", 60*time.Second, "time interval nsqd is configured to push to statsd (must match nsqd)")
 
 	notificationHTTPEndpoint = flagSet.String("notification-http-endpoint", "", "HTTP endpoint (fully qualified) to which POST notifications of admin actions will be sent")
 

contrib/nsqadmin.cfg.example

@@ -10,6 +10,12 @@ proxy_graphite = false
 ## prefix used for keys sent to statsd (%s for host replacement, must match nsqd)
 statsd_prefix = "nsq.%s"
 
+## format of statsd counter stats
+statsd_counter_format = "stats.counters.%s.count"
+
+## format of statsd gauge stats
+statsd_gauge_format = "stats.gauges.%s"
+
 ## time interval nsqd is configured to push to statsd (must match nsqd)
 statsd_interval = "60s"
 

nsqadmin/README.md

@@ -12,6 +12,6 @@ Read the [docs](http://nsq.io/components/nsqadmin.html)
  3. `$ gulp clean watch` or `$ gulp clean build`
  4. `$ go-bindata --debug --pkg=nsqadmin --prefix=static/build static/build/...`
  5. `$ cd ../nsqadmin && go build && ./nsqadmin`
- 6. make changes
+ 6. make changes (repeat step 5 if you make changes to any Go code)
  7. `$ go-bindata --pkg=nsqadmin --prefix=static/build static/build/...`
  8. commit other changes and `bindata.go`

nsqadmin/http.go

@@ -116,21 +116,27 @@ func (s *httpServer) indexHandler(w http.ResponseWriter, req *http.Request, ps h
 
 	w.Header().Set("Content-Type", "text/html")
 	t.Execute(w, struct {
-		Version        string
-		ProxyGraphite  bool
-		GraphEnabled   bool
-		GraphiteURL    string
-		StatsdInterval int
-		StatsdPrefix   string
-		NSQLookupd     []string
+		Version             string
+		ProxyGraphite       bool
+		GraphEnabled        bool
+		GraphiteURL         string
+		StatsdInterval      int
+		UseStatsdPrefixes   bool
+		StatsdCounterFormat string
+		StatsdGaugeFormat   string
+		StatsdPrefix        string
+		NSQLookupd          []string
 	}{
-		Version:        version.Binary,
-		ProxyGraphite:  s.ctx.nsqadmin.opts.ProxyGraphite,
-		GraphEnabled:   s.ctx.nsqadmin.opts.GraphiteURL != "",
-		GraphiteURL:    s.ctx.nsqadmin.opts.GraphiteURL,
-		StatsdInterval: int(s.ctx.nsqadmin.opts.StatsdInterval / time.Second),
-		StatsdPrefix:   s.ctx.nsqadmin.opts.StatsdPrefix,
-		NSQLookupd:     s.ctx.nsqadmin.opts.NSQLookupdHTTPAddresses,
+		Version:             version.Binary,
+		ProxyGraphite:       s.ctx.nsqadmin.opts.ProxyGraphite,
+		GraphEnabled:        s.ctx.nsqadmin.opts.GraphiteURL != "",
+		GraphiteURL:         s.ctx.nsqadmin.opts.GraphiteURL,
+		StatsdInterval:      int(s.ctx.nsqadmin.opts.StatsdInterval / time.Second),
+		UseStatsdPrefixes:   s.ctx.nsqadmin.opts.UseStatsdPrefixes,
+		StatsdCounterFormat: s.ctx.nsqadmin.opts.StatsdCounterFormat,
+		StatsdGaugeFormat:   s.ctx.nsqadmin.opts.StatsdGaugeFormat,
+		StatsdPrefix:        s.ctx.nsqadmin.opts.StatsdPrefix,
+		NSQLookupd:          s.ctx.nsqadmin.opts.NSQLookupdHTTPAddresses,
 	})
 
 	return nil, nil

nsqadmin/options.go

@@ -12,9 +12,12 @@ type Options struct {
 	GraphiteURL   string `flag:"graphite-url"`
 	ProxyGraphite bool   `flag:"proxy-graphite"`
 
-	UseStatsdPrefixes bool          `flag:"use-statsd-prefixes"`
-	StatsdPrefix      string        `flag:"statsd-prefix"`
-	StatsdInterval    time.Duration `flag:"statsd-interval"`
+	UseStatsdPrefixes   bool   `flag:"use-statsd-prefixes"`
+	StatsdPrefix        string `flag:"statsd-prefix"`
+	StatsdCounterFormat string `flag:"statsd-counter-format"`
+	StatsdGaugeFormat   string `flag:"statsd-gauge-format"`
+
+	StatsdInterval time.Duration `flag:"statsd-interval"`
 
 	NSQLookupdHTTPAddresses []string `flag:"lookupd-http-address" cfg:"nsqlookupd_http_addresses"`
 	NSQDHTTPAddresses       []string `flag:"nsqd-http-address" cfg:"nsqd_http_addresses"`
@@ -31,10 +34,12 @@ type Options struct {
 
 func NewOptions() *Options {
 	return &Options{
-		HTTPAddress:       "0.0.0.0:4171",
-		UseStatsdPrefixes: true,
-		StatsdPrefix:      "nsq.%s",
-		StatsdInterval:    60 * time.Second,
-		Logger:            log.New(os.Stderr, "[nsqadmin] ", log.Ldate|log.Ltime|log.Lmicroseconds),
+		HTTPAddress:         "0.0.0.0:4171",
+		UseStatsdPrefixes:   true,
+		StatsdPrefix:        "nsq.%s",
+		StatsdCounterFormat: "stats.counters.%s.count",
+		StatsdGaugeFormat:   "stats.gauges.%s",
+		StatsdInterval:      60 * time.Second,
+		Logger:              log.New(os.Stderr, "[nsqadmin] ", log.Ldate|log.Ltime|log.Lmicroseconds),
 	}
 }

nsqadmin/static/html/index.html

@@ -18,6 +18,9 @@
         var VERSION = '{{.Version}}';
         var GRAPHITE_URL = '{{if .ProxyGraphite}}{{else}}{{.GraphiteURL}}{{end}}';
         var GRAPH_ENABLED = {{if .GraphEnabled}}true{{else}}false{{end}};
+        var USE_STATSD_PREFIXES = {{if .UseStatsdPrefixes}}true{{else}}false{{end}};
+        var STATSD_COUNTER_FORMAT = '{{.StatsdCounterFormat}}';
+        var STATSD_GAUGE_FORMAT = '{{.StatsdGaugeFormat}}';
         var STATSD_INTERVAL = {{.StatsdInterval}};
         var STATSD_PREFIX = '{{.StatsdPrefix}}';
         var NSQLOOKUPD = [{{range .NSQLookupd}}'{{.}}',{{end}}];

nsqadmin/static/js/app_state.js

@@ -8,6 +8,9 @@ var AppState = Backbone.Model.extend({
             'GRAPHITE_URL': GRAPHITE_URL,
             'GRAPH_ENABLED': GRAPH_ENABLED,
             'STATSD_INTERVAL': STATSD_INTERVAL,
+            'USE_STATSD_PREFIXES': USE_STATSD_PREFIXES,
+            'STATSD_COUNTER_FORMAT': STATSD_COUNTER_FORMAT,
+            'STATSD_GAUGE_FORMAT': STATSD_GAUGE_FORMAT,
             'STATSD_PREFIX': STATSD_PREFIX,
             'NSQLOOKUPD': NSQLOOKUPD,
             'graph_interval': '2h'

nsqadmin/static/js/lib/handlebars_helpers.js

@@ -4,18 +4,37 @@ var Handlebars = require('hbsfy/runtime');
 
 var AppState = require('../app_state');
 
-var statsdPrefix = function(host, metricType) {
+var formatStatsdKey = function(metricType, key) {
+    var fullKey = key;
+
+    // Remove this check when the --use-statsd-prefixes field is removed.
+    if (!AppState.get('USE_STATSD_PREFIXES')) {
+        // If the user has overridden the default boolean, then
+        // we will assume they don't want any formatting at all.
+        // In the future, if the user requests no formatting, then
+        // they must set the counter and gauge format flags to empty strings.
+        return fullKey;
+    }
+
+    if (metricType === 'counter') {
+        var format = AppState.get('STATSD_COUNTER_FORMAT');
+        fullKey = format.replace(/%s/g, key)
+    } else if (metricType === 'gauge') {
+        var format = AppState.get('STATSD_GAUGE_FORMAT');
+        fullKey = format.replace(/%s/g, key)
+    }
+
+    return fullKey;
+}
+
+var statsdPrefix = function(host) {
     var prefix = AppState.get('STATSD_PREFIX');
     var statsdHostKey = host.replace(/[\.:]/g, '_');
     prefix = prefix.replace(/%s/g, statsdHostKey);
     if (prefix.substring(prefix.length, 1) !== '.') {
         prefix += '.';
     }
-    if (AppState.get('USE_STATSD_PREFIXES') && metricType === 'counter') {
-        prefix = 'stats.counters.' + prefix;
-    } else if (AppState.get('USE_STATSD_PREFIXES') && metricType === 'gauge') {
-        prefix = 'stats.gauges.' + prefix;
-    }
+
     return prefix;
 };
 
@@ -53,16 +72,19 @@ var genColorList = function(typ, key) {
 
 var genTargets = function(typ, node, ns1, ns2, key) {
     var targets = [];
-    var prefix = statsdPrefix(node ? node : '*', metricType(key));
+    var prefix = statsdPrefix(node ? node : '*');
     if (typ === 'topic') {
-        targets.push('sumSeries(' + prefix + 'topic.' + ns1 + '.' + key + ')');
+        var fullKey = formatStatsdKey(metricType(key), prefix + 'topic.' + ns1 + '.' + key);
+        targets.push('sumSeries(' + fullKey + ')');
     } else if (typ === 'channel') {
-        targets.push('sumSeries(' + prefix + 'topic.' + ns1 + '.channel.' + ns2 + '.' + key + ')');
+        var fullKey = formatStatsdKey(metricType(key), prefix + 'topic.' + ns1 + '.channel.' + ns2 + '.' + key);
+        targets.push('sumSeries(' + fullKey + ')');
     } else if (typ === 'node') {
         var target = prefix + 'mem.' + key;
         if (key === 'gc_runs') {
             target = 'movingAverage(' + target + ',45)';
         }
+        var fullKey = formatStatsdKey(metricType(key), target);
         targets.push(target);
     } else if (typ === 'e2e') {
         targets = _.map(ns1['percentiles'], function(p) {
@@ -76,10 +98,12 @@ var genTargets = function(typ, node, ns1, ns2, key) {
             if (node === '*') {
                 t = 'averageSeries(' + t + ')';
             }
+            var fullKey = formatStatsdKey(metricType(key), t);
             return 'scale(' + t + ',0.000001)';
         });
     } else if (typ === 'counter') {
-        targets.push('sumSeries(' + prefix + 'topic.*.channel.*.' + key + ')');
+        var fullKey = formatStatsdKey(metricType(key), prefix + 'topic.*.channel.*.' + key);
+        targets.push('sumSeries(' + fullKey + ')');
     }
     return targets;
 };