Merge pull request #155 from shermdog/RE-7014-cinext

(RE-7014) add statsd support

Gemfile

@@ -7,10 +7,12 @@ gem 'rbvmomi', '>= 1.8'
 gem 'redis', '>= 3.2'
 gem 'sinatra', '>= 1.4'
 gem 'net-ldap', '<= 0.12.1' # keep compatibility w/ jruby & mri-1.9.3
+gem 'statsd-ruby', '>= 1.3.0'
 
 # Test deps
 group :test do
   gem 'rack-test', '>= 0.6'
   gem 'rspec', '>= 3.2'
+  gem 'simplecov', '>= 0.11.2'
   gem 'yarjuf', '>= 2.0'
 end

lib/vmpooler.rb

@@ -7,6 +7,7 @@ module Vmpooler
   require 'rbvmomi'
   require 'redis'
   require 'sinatra/base'
+  require "statsd-ruby"
   require 'time'
   require 'timeout'
   require 'yaml'
@@ -52,6 +53,13 @@ def self.config(filepath='vmpooler.yaml')
       parsed_config[:graphite]['prefix'] ||= 'vmpooler'
     end
 
+    # statsd is an addition and my not be present in YAML configuration
+    if parsed_config[:statsd]
+      if parsed_config[:statsd]['server']
+        parsed_config[:statsd]['prefix'] ||= 'vmpooler'
+      end
+    end
+
     if parsed_config[:tagfilter]
       parsed_config[:tagfilter].keys.each do |tag|
         parsed_config[:tagfilter][tag] = Regexp.new(parsed_config[:tagfilter][tag])
@@ -79,6 +87,14 @@ def self.new_graphite(server)
     end
   end
 
+  def self.new_statsd(server, port)
+    if server.nil? || server.empty?
+      nil
+    else
+      Statsd.new server, port
+    end
+  end
+
   def self.pools(conf)
     conf[:pools]
   end

lib/vmpooler/api.rb

@@ -42,9 +42,10 @@ def initialize
     use Vmpooler::API::Reroute
     use Vmpooler::API::V1
 
-    def configure(config, redis, environment = :production)
+    def configure(config, redis, statsd, environment = :production)
       self.settings.set :config, config
       self.settings.set :redis, redis
+      self.settings.set :statsd, statsd
       self.settings.set :environment, environment
     end
 

lib/vmpooler/api/v1.rb

@@ -12,6 +12,16 @@ def backend
       Vmpooler::API.settings.redis
     end
 
+    def statsd
+      Vmpooler::API.settings.statsd
+    end
+
+    def statsd_prefix
+      if Vmpooler::API.settings.statsd
+        Vmpooler::API.settings.config[:statsd]['prefix'] ? Vmpooler::API.settings.config[:statsd]['prefix'] : 'vmpooler'
+      end
+    end
+
     def config
       Vmpooler::API.settings.config[:config]
     end
@@ -32,13 +42,16 @@ def alias_deref(hash)
       newhash = {}
 
       hash.each do |key, val|
+        if Vmpooler::API.settings.config[:alias][key]
+          key = Vmpooler::API.settings.config[:alias][key]
+        end
+
         if backend.exists('vmpooler__ready__' + key)
           newhash[key] = val
+        elsif backend.exists('vmpooler__empty__' + key)
+          newhash['empty'] = (newhash['empty'] || 0) + val.to_i
         else
-          if Vmpooler::API.settings.config[:alias][key]
-            newkey = Vmpooler::API.settings.config[:alias][key]
-            newhash[newkey] = val
-          end
+          newhash['invalid'] = (newhash['invalid'] || 0) + val.to_i
         end
       end
 
@@ -94,8 +107,10 @@ def atomically_allocate_vms(payload)
           vm = fetch_single_vm(template)
           if !vm
             failed = true
+            statsd.increment(statsd_prefix + '.checkout.fail.' + template, 1)
             break
           else
+            statsd.increment(statsd_prefix + '.checkout.success.' + template, 1)
             vms << [ template, vm ]
           end
         end
@@ -375,8 +390,16 @@ def atomically_allocate_vms(payload)
       content_type :json
       result = { 'ok' => false }
 
-      if jdata and !jdata.empty?
-        result = atomically_allocate_vms(jdata)
+      if jdata
+        empty = jdata.delete('empty')
+        invalid = jdata.delete('invalid')
+        statsd.increment(statsd_prefix + '.checkout.empty', empty) if empty
+        statsd.increment(statsd_prefix + '.checkout.invalid', invalid) if invalid
+        unless jdata.empty?
+          result = atomically_allocate_vms(jdata)
+        else
+          status 404
+        end
       else
         status 404
       end
@@ -400,8 +423,16 @@ def extract_templates_from_query_params(params)
       content_type :json
       result = { 'ok' => false }
 
-      if payload and !payload.empty?
-        result = atomically_allocate_vms(payload)
+      if payload
+        empty = payload.delete('empty')
+        invalid = payload.delete('invalid')
+        statsd.increment(statsd_prefix + '.checkout.empty', empty) if empty
+        statsd.increment(statsd_prefix + '.checkout.invalid', invalid) if invalid
+        unless payload.empty?
+          result = atomically_allocate_vms(payload)
+        else
+          status 404
+        end
       else
         status 404
       end

lib/vmpooler/pool_manager.rb

@@ -1,12 +1,15 @@
 module Vmpooler
   class PoolManager
-    def initialize(config, logger, redis, graphite=nil)
+    def initialize(config, logger, redis, graphite = nil, statsd = nil)
       $config = config
 
       # Load logger library
       $logger = logger
 
-      unless graphite.nil?
+      # statsd and graphite are mutex in the context of vmpooler
+      if statsd
+        $statsd = statsd
+      elsif graphite
         $graphite = graphite
       end
 
@@ -258,7 +261,8 @@ def clone_vm(template, folder, datastore, target)
         $redis.decr('vmpooler__tasks__clone')
 
         begin
-          $graphite.log($config[:graphite]['prefix'] + ".clone.#{vm['template']}", finish) if defined? $graphite
+          $statsd.timing($config[:statsd]['prefix'] + ".clone.#{vm['template']}", finish) if $statsd
+          $graphite.log($config[:graphite]['prefix'] + ".clone.#{vm['template']}", finish) if $graphite
         rescue
         end
       end
@@ -294,7 +298,7 @@ def destroy_vm(vm, pool)
 
           $logger.log('s', "[-] [#{pool}] '#{vm}' destroyed in #{finish} seconds")
 
-          $graphite.log($config[:graphite]['prefix'] + ".destroy.#{pool}", finish) if defined? $graphite
+          $graphite.log($config[:graphite]['prefix'] + ".destroy.#{pool}", finish) if $graphite
         end
       end
     end
@@ -565,7 +569,10 @@ def _check_pool(pool)
       total = $redis.scard('vmpooler__pending__' + pool['name']) + ready
 
       begin
-        if defined? $graphite
+        if $statsd
+          $statsd.gauge($config[:statsd]['prefix'] + '.ready.' + pool['name'], $redis.scard('vmpooler__ready__' + pool['name']))
+          $statsd.gauge($config[:statsd]['prefix'] + '.running.' + pool['name'], $redis.scard('vmpooler__running__' + pool['name']))
+        elsif $graphite
           $graphite.log($config[:graphite]['prefix'] + '.ready.' + pool['name'], $redis.scard('vmpooler__ready__' + pool['name']))
           $graphite.log($config[:graphite]['prefix'] + '.running.' + pool['name'], $redis.scard('vmpooler__running__' + pool['name']))
         end

lib/vmpooler/statsd.rb

@@ -0,0 +1,9 @@
+require 'rubygems' unless defined?(Gem)
+
+module Vmpooler
+  class Statsd
+    def initialize(server = 'statsd', port = 8125)
+      @server = Statsd.new(server, port)
+    end
+  end
+end

spec/spec_helper.rb

@@ -1,3 +1,7 @@
+require 'simplecov'
+SimpleCov.start do
+  add_filter '/spec/'
+end
 require 'helpers'
 require 'rbvmomi'
 require 'rspec'

spec/vmpooler/api/v1_spec.rb

@@ -180,6 +180,7 @@ def app()
 
   describe '/vm' do
     let(:redis)  { double('redis') }
+    let(:statsd)  { double('stats') }
     let(:prefix) { '/api/v1' }
     let(:config) { {
       config: {
@@ -190,20 +191,27 @@ def app()
         {'name' => 'pool1', 'size' => 5},
         {'name' => 'pool2', 'size' => 10}
       ],
-      alias: { 'poolone' => 'pool1' }
+      alias: { 'poolone' => 'pool1' },
+      statsd: { 'prefix' => 'vmpooler' }
     } }
 
     before do
       app.settings.set :config, config
       app.settings.set :redis, redis
+      app.settings.set :statsd, statsd
 
-      allow(redis).to receive(:exists).and_return '1'
+      allow(redis).to receive(:exists).with('vmpooler__token__abcdefghijklmnopqrstuvwxyz012345').and_return '1'
+      allow(redis).to receive(:exists).with('vmpooler__ready__pool1').and_return '1'
+      allow(redis).to receive(:exists).with('vmpooler__ready__pool2').and_return '1'
       allow(redis).to receive(:hget).with('vmpooler__token__abcdefghijklmnopqrstuvwxyz012345', 'user').and_return 'jdoe'
       allow(redis).to receive(:hset).and_return '1'
       allow(redis).to receive(:sadd).and_return '1'
       allow(redis).to receive(:scard).and_return '5'
       allow(redis).to receive(:spop).with('vmpooler__ready__pool1').and_return 'abcdefghijklmnop'
       allow(redis).to receive(:spop).with('vmpooler__ready__pool2').and_return 'qrstuvwxyz012345'
+      allow(statsd).to receive(:increment).with('vmpooler.checkout.success.pool1', 1)
+      allow(statsd).to receive(:increment).with('vmpooler.checkout.success.pool2', 1)
+      allow(statsd).to receive(:increment).with('vmpooler.checkout.fail.pool2', 1)
     end
 
     describe 'POST /vm' do
@@ -223,7 +231,7 @@ def app()
       end
 
       it 'returns a single VM for an alias' do
-        expect(redis).to receive(:exists).with("vmpooler__ready__poolone").and_return(false)
+        expect(redis).to receive(:exists).with("vmpooler__ready__pool1").and_return(1)
 
         post "#{prefix}/vm", '{"poolone":"1"}'
 
@@ -241,12 +249,22 @@ def app()
 
       it 'fails on nonexistent pools' do
         expect(redis).to receive(:exists).with("vmpooler__ready__poolpoolpool").and_return(false)
-
+        expect(redis).to receive(:exists).with("vmpooler__empty__poolpoolpool").and_return(false)
+        expect(statsd).to receive(:increment).with('vmpooler.checkout.invalid', 1)
         post "#{prefix}/vm", '{"poolpoolpool":"1"}'
 
         expect_json(ok = false, http = 404)
       end
 
+      it 'fails on empty pools' do
+        expect(redis).to receive(:exists).with("vmpooler__ready__emptypool").and_return(false)
+        expect(redis).to receive(:exists).with("vmpooler__empty__emptypool").and_return(true)
+        expect(statsd).to receive(:increment).with('vmpooler.checkout.empty', 1)
+        post "#{prefix}/vm", '{"emptypool":"1"}'
+
+        expect_json(ok = false, http = 404)
+      end
+
       it 'returns multiple VMs' do
         post "#{prefix}/vm", '{"pool1":"1","pool2":"1"}'
 
@@ -446,6 +464,7 @@ def app()
 
   describe '/vm/:template' do
     let(:redis)  { double('redis') }
+    let(:statsd)  { double('stats') }
     let(:prefix) { '/api/v1' }
     let(:config) { {
       config: {
@@ -456,20 +475,27 @@ def app()
         {'name' => 'pool1', 'size' => 5},
         {'name' => 'pool2', 'size' => 10}
       ],
-      alias: { 'poolone' => 'pool1' }
+      alias: { 'poolone' => 'pool1' },
+      statsd: { 'prefix' => 'vmpooler' }
     } }
 
     before do
       app.settings.set :config, config
       app.settings.set :redis, redis
+      app.settings.set :statsd, statsd
 
-      allow(redis).to receive(:exists).and_return '1'
+      allow(redis).to receive(:exists).with('vmpooler__token__abcdefghijklmnopqrstuvwxyz012345').and_return '1'
+      allow(redis).to receive(:exists).with('vmpooler__ready__pool1').and_return '1'
+      allow(redis).to receive(:exists).with('vmpooler__ready__pool2').and_return '1'
       allow(redis).to receive(:hget).with('vmpooler__token__abcdefghijklmnopqrstuvwxyz012345', 'user').and_return 'jdoe'
       allow(redis).to receive(:hset).and_return '1'
       allow(redis).to receive(:sadd).and_return '1'
       allow(redis).to receive(:scard).and_return '5'
       allow(redis).to receive(:spop).with('vmpooler__ready__pool1').and_return 'abcdefghijklmnop'
       allow(redis).to receive(:spop).with('vmpooler__ready__pool2').and_return 'qrstuvwxyz012345'
+      allow(statsd).to receive(:increment).with('vmpooler.checkout.success.pool1', 1)
+      allow(statsd).to receive(:increment).with('vmpooler.checkout.success.pool2', 1)
+      allow(statsd).to receive(:increment).with('vmpooler.checkout.fail.pool2', 1)
     end
 
     describe 'POST /vm/:template' do
@@ -489,7 +515,7 @@ def app()
       end
 
       it 'returns a single VM for an alias' do
-        expect(redis).to receive(:exists).with("vmpooler__ready__poolone").and_return(false)
+        expect(redis).to receive(:exists).with("vmpooler__ready__pool1").and_return(1)
 
         post "#{prefix}/vm/poolone", ''
 
@@ -507,12 +533,23 @@ def app()
 
       it 'fails on nonexistent pools' do
         expect(redis).to receive(:exists).with("vmpooler__ready__poolpoolpool").and_return(false)
+        expect(redis).to receive(:exists).with("vmpooler__empty__poolpoolpool").and_return(false)
+        expect(statsd).to receive(:increment).with('vmpooler.checkout.invalid', 1)
 
         post "#{prefix}/vm/poolpoolpool", ''
 
         expect_json(ok = false, http = 404)
       end
 
+      it 'fails on empty pools' do
+        expect(redis).to receive(:exists).with("vmpooler__ready__emptypool").and_return(false)
+        expect(redis).to receive(:exists).with("vmpooler__empty__emptypool").and_return(true)
+        expect(statsd).to receive(:increment).with('vmpooler.checkout.empty', 1)
+        post "#{prefix}/vm/emptypool", ''
+
+        expect_json(ok = false, http = 404)
+      end
+
       it 'returns multiple VMs' do
         post "#{prefix}/vm/pool1+pool2", ''