diff --git a/CHANGELOG.md b/CHANGELOG.md index e7e1b1abf..b7b479fbe 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,6 +17,9 @@ git logs & PR history. - Remove a failed VM from the ready queue (POOLER-133) - Begin checking ready VMs to ensure alive after 1 minute by default +### Added +- Add capability to ship VM usage metrics (POOLER-134) + # [0.2.2](https://github.com/puppetlabs/vmpooler/compare/0.2.1...0.2.2) ### Fixed diff --git a/docs/configuration.md b/docs/configuration.md index 0d534b7d4..d692f3209 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -156,6 +156,20 @@ When enabled in the global configuration then purging is enabled for all provide Expects a boolean value (optional; default: false) +### USAGE\_STATS + +Enable shipping of VM usage stats +When enabled a metric is emitted when a machine is destroyed. Tags are inspected and used to organize +shipped metrics if there is a jenkins\_build\_url tag set for the VM. +Without the jenkins\_build\_url tag set the metric will be sent as "usage.$user.$pool\_name". +When the jenkins\_build\_url tag is set the metric will be sent with additional data. Here is an example +based off of the following URL; +https://jenkins.example.com/job/platform\_puppet-agent-extra\_puppet-agent-integration-suite\_pr/RMM\_COMPONENT\_TO\_TEST\_NAME=puppet,SLAVE\_LABEL=beaker,TEST\_TARGET=redhat7-64a/824/ +"usage.$user.$instance.$value\_stream.$branch.$project.$job\_name.$component\_to\_test.$pool\_name", which translates to +"usage.$user.jenkins\_example\_com.platform.pr.puppet-agent-extra.puppet-agent-integration-suite.puppet.$pool\_name" +Expects a boolean value +(optional; default: false) + ## API options ### AUTH\_PROVIDER diff --git a/lib/vmpooler.rb b/lib/vmpooler.rb index a50be526e..4a013105b 100644 --- a/lib/vmpooler.rb +++ b/lib/vmpooler.rb @@ -66,6 +66,7 @@ def self.config(filepath = 'vmpooler.yaml') parsed_config[:config]['create_template_delta_disks'] = ENV['CREATE_TEMPLATE_DELTA_DISKS'] if ENV['CREATE_TEMPLATE_DELTA_DISKS'] parsed_config[:config]['experimental_features'] = ENV['EXPERIMENTAL_FEATURES'] if ENV['EXPERIMENTAL_FEATURES'] parsed_config[:config]['purge_unconfigured_folders'] = ENV['PURGE_UNCONFIGURED_FOLDERS'] if ENV['PURGE_UNCONFIGURED_FOLDERS'] + parsed_config[:config]['usage_stats'] = ENV['USAGE_STATS'] if ENV['USAGE_STATS'] parsed_config[:redis] = parsed_config[:redis] || {} parsed_config[:redis]['server'] = ENV['REDIS_SERVER'] || parsed_config[:redis]['server'] || 'localhost' diff --git a/lib/vmpooler/pool_manager.rb b/lib/vmpooler/pool_manager.rb index 3346da3cf..95f8bd87a 100644 --- a/lib/vmpooler/pool_manager.rb +++ b/lib/vmpooler/pool_manager.rb @@ -319,10 +319,66 @@ def _destroy_vm(vm, pool, provider) finish = format('%.2f', Time.now - start) $logger.log('s', "[-] [#{pool}] '#{vm}' destroyed in #{finish} seconds") $metrics.timing("destroy.#{pool}", finish) + get_vm_usage_labels(vm) end dereference_mutex(vm) end + def get_vm_usage_labels(vm) + return unless $config[:config]['usage_stats'] + checkout = $redis.hget("vmpooler__vm__#{vm}", 'checkout') + return if checkout.nil? + jenkins_build_url = $redis.hget("vmpooler__vm__#{vm}", 'tag:jenkins_build_url') + user = $redis.hget("vmpooler__vm__#{vm}", 'token:user') || 'unauthenticated' + poolname = $redis.hget("vmpooler__vm__#{vm}", "template") + + unless jenkins_build_url + $metrics.increment("usage.#{user}.#{poolname}") + return + end + + url_parts = jenkins_build_url.split('/')[2..-1] + instance = url_parts[0].gsub('.', '_') + value_stream_parts = url_parts[2].split('_') + value_stream = value_stream_parts.shift + branch = value_stream_parts.pop + project = value_stream_parts.shift + job_name = value_stream_parts.join('_') + build_metadata_parts = url_parts[3] + component_to_test = component_to_test('RMM_COMPONENT_TO_TEST_NAME', build_metadata_parts) + + metric_parts = [ + 'usage', + user, + instance, + value_stream, + branch, + project, + job_name, + component_to_test, + poolname + ] + + metric_parts = metric_parts.reject { |s| s.nil? } + + $metrics.increment(metric_parts.join('.')) + rescue => err + logger.log('d', "[!] [#{poolname}] failed while evaluating usage labels on '#{vm}' with an error: #{err}") + end + + def component_to_test(match, labels_string) + return if labels_string.nil? + labels_string_parts = labels_string.split(',') + labels_string_parts.each do |part| + key, value = part.split('=') + next if value.nil? + if key == match + return value + end + end + return + end + def purge_unused_vms_and_folders global_purge = $config[:config]['purge_unconfigured_folders'] providers = $config[:providers].keys diff --git a/spec/helpers.rb b/spec/helpers.rb index d9b691493..cff93e67e 100644 --- a/spec/helpers.rb +++ b/spec/helpers.rb @@ -45,10 +45,11 @@ def create_ready_vm(template, name, token = nil) redis.hset("vmpooler__vm__#{name}", "template", template) end -def create_running_vm(template, name, token = nil) - create_vm(name, token) +def create_running_vm(template, name, token = nil, user = nil) + create_vm(name, token, nil, user) redis.sadd("vmpooler__running__#{template}", name) - redis.hset("vmpooler__vm__#{name}", "template", template) + redis.hset("vmpooler__vm__#{name}", 'template', template) + redis.hset("vmpooler__vm__#{name}", 'checkout', Time.now) end def create_pending_vm(template, name, token = nil) @@ -57,10 +58,11 @@ def create_pending_vm(template, name, token = nil) redis.hset("vmpooler__vm__#{name}", "template", template) end -def create_vm(name, token = nil, redis_handle = nil) +def create_vm(name, token = nil, redis_handle = nil, user = nil) redis_db = redis_handle ? redis_handle : redis redis_db.hset("vmpooler__vm__#{name}", 'checkout', Time.now) redis_db.hset("vmpooler__vm__#{name}", 'token:token', token) if token + redis_db.hset("vmpooler__vm__#{name}", 'token:user', user) if user end def create_completed_vm(name, pool, active = false, redis_handle = nil) @@ -81,6 +83,11 @@ def create_migrating_vm(name, pool, redis_handle = nil) redis_db.sadd("vmpooler__migrating__#{pool}", name) end +def create_tag(vm, tag_name, tag_value, redis_handle = nil) + redis_db = redis_handle ? redis-handle : redis + redis_db.hset("vmpooler__vm__#{vm}", "tag:#{tag_name}", tag_value) +end + def add_vm_to_migration_set(name, redis_handle = nil) redis_db = redis_handle ? redis_handle : redis redis_db.sadd('vmpooler__migration', name) diff --git a/spec/unit/pool_manager_spec.rb b/spec/unit/pool_manager_spec.rb index e9d2d6710..995798b91 100644 --- a/spec/unit/pool_manager_spec.rb +++ b/spec/unit/pool_manager_spec.rb @@ -757,11 +757,18 @@ end it 'should emit a timing metric' do + allow(subject).to receive(:get_vm_usage_labels) expect(metrics).to receive(:timing).with("destroy.#{pool}", String) subject._destroy_vm(vm,pool,provider) end + it 'should check usage labels' do + expect(subject).to receive(:get_vm_usage_labels).with(vm) + + subject._destroy_vm(vm,pool,provider) + end + it 'should dereference the mutex' do expect(subject).to receive(:dereference_mutex) @@ -803,6 +810,171 @@ end end + describe '#get_vm_usage_labels' do + + let(:template) { 'pool1' } + let(:user) { 'vmpuser' } + let(:vm) { 'vm1' } + + context 'when label evaluation is disabled' do + it 'should do nothing' do + subject.get_vm_usage_labels(vm) + end + end + + context 'when label evaluation is enabled' do + + before(:each) do + config[:config]['usage_stats'] = true + end + + context 'when a VM has not been checked out' do + before(:each) do + create_ready_vm(template, vm) + end + + it 'should return' do + expect(subject).to receive(:get_vm_usage_labels).and_return(nil) + + subject.get_vm_usage_labels(vm) + end + end + + context 'when a VM has been checked out' do + + context 'without auth' do + + before(:each) do + create_running_vm(template, vm) + end + + it 'should emit a metric' do + expect(metrics).to receive(:increment).with("usage.unauthenticated.#{template}") + + subject.get_vm_usage_labels(vm) + end + end + + context 'with auth' do + + before(:each) do + create_running_vm(template, vm, token, user) + end + + it 'should emit a metric' do + expect(metrics).to receive(:increment).with("usage.#{user}.#{template}") + + subject.get_vm_usage_labels(vm) + end + + context 'with a jenkins_build_url label' do + let(:jenkins_build_url) { 'https://jenkins.example.com/job/enterprise_pe-acceptance-tests_integration-system_pe_full-agent-upgrade_weekend_2018.1.x/LAYOUT=centos6-64mcd-ubuntu1404-32f-64f,LEGACY_AGENT_VERSION=NONE,PLATFORM=NOTUSED,SCM_BRANCH=2018.1.x,UPGRADE_FROM=2018.1.0,UPGRADE_TO_VERSION=NONE,label=beaker/222/' } + let(:url_parts) { jenkins_build_url.split('/')[2..-1] } + let(:instance) { url_parts[0].gsub('.', '_') } + let(:value_stream_parts) { url_parts[2].split('_') } + let(:value_stream) { value_stream_parts.shift } + let(:branch) { value_stream_parts.pop } + let(:project) { value_stream_parts.shift } + let(:job_name) { value_stream_parts.join('_') } + + before(:each) do + create_tag(vm, 'jenkins_build_url', jenkins_build_url) + end + + it 'should emit a metric with information from the URL' do + expect(metrics).to receive(:increment).with("usage.#{user}.#{instance}.#{value_stream}.#{branch}.#{project}.#{job_name}.#{template}") + + subject.get_vm_usage_labels(vm) + end + end + + context 'with a jenkins_build_url that contains RMM_COMPONENT_TO_TEST_NAME' do + let(:jenkins_build_url) { 'https://jenkins.example.com/job/platform_puppet-agent-extra_puppet-agent-integration-suite_pr/RMM_COMPONENT_TO_TEST_NAME=puppet,SLAVE_LABEL=beaker,TEST_TARGET=redhat7-64a/824/' } + let(:url_parts) { jenkins_build_url.split('/')[2..-1] } + let(:instance) { url_parts[0].gsub('.', '_') } + let(:value_stream_parts) { url_parts[2].split('_') } + let(:value_stream) { value_stream_parts.shift } + let(:branch) { value_stream_parts.pop } + let(:project) { value_stream_parts.shift } + let(:job_name) { value_stream_parts.join('_') } + let(:build_metadata) { url_parts[3] } + let(:build_component) { subject.component_to_test('RMM_COMPONENT_TO_TEST_NAME', build_metadata) } + + before(:each) do + create_tag(vm, 'jenkins_build_url', jenkins_build_url) + end + + it 'should emit a metric with information from the URL' do + expect(metrics).to receive(:increment).with("usage.#{user}.#{instance}.#{value_stream}.#{branch}.#{project}.#{job_name}.#{build_component}.#{template}") + + subject.get_vm_usage_labels(vm) + end + + context 'when there is no matrix job information' do + + let(:jenkins_build_url) { 'https://jenkins.example.com/job/platform_puppet-agent-extra_puppet-agent-integration-suite_pr/824/' } + let(:url_parts) { jenkins_build_url.split('/')[2..-1] } + let(:instance) { url_parts[0].gsub('.', '_') } + let(:value_stream_parts) { url_parts[2].split('_') } + let(:value_stream) { value_stream_parts.shift } + let(:branch) { value_stream_parts.pop } + let(:project) { value_stream_parts.shift } + let(:job_name) { value_stream_parts.join('_') } + + before(:each) do + create_tag(vm, 'jenkins_build_url', jenkins_build_url) + end + + it 'should emit a metric with information from the URL without a build_component' do + expect(metrics).to receive(:increment).with("usage.#{user}.#{instance}.#{value_stream}.#{branch}.#{project}.#{job_name}.#{template}") + + subject.get_vm_usage_labels(vm) + end + end + end + + end + end + end + end + + describe '#component_to_test' do + let(:matching_key) { 'LABEL_ONE' } + let(:matching_value) { 'test' } + let(:labels_string) { "#{matching_key}=#{matching_value},LABEL_TWO=test2,LABEL_THREE=test3" } + let(:nonmatrix_string) { 'test,stuff,and,things' } + + context 'when string contains a matching key' do + it 'should print the corresponding value' do + expect(subject.component_to_test(matching_key, labels_string)).to eq(matching_value) + end + + context 'when match contains no value' do + it 'should return nil' do + expect(subject.component_to_test(matching_key, matching_key)).to be nil + end + end + end + + context 'when string contains no key value pairs' do + it 'should return' do + expect(subject.component_to_test(matching_key, nonmatrix_string)).to be nil + end + end + + context 'when labels_string is a job number' do + it 'should return nil' do + expect(subject.component_to_test(matching_key, '25')).to be nil + end + end + + context 'when labels_string is nil' do + it 'should return nil' do + expect(subject.component_to_test(matching_key, nil)).to be nil + end + end + end + describe '#purge_unused_vms_and_folders' do let(:config) { YAML.load(<<-EOT --- diff --git a/spec/unit/providers_spec.rb b/spec/unit/providers_spec.rb index 6c3b35a0c..0ccc7f044 100644 --- a/spec/unit/providers_spec.rb +++ b/spec/unit/providers_spec.rb @@ -17,7 +17,7 @@ File.join(project_root_dir, 'lib', 'vmpooler', 'providers', 'dummy.rb'), File.join(project_root_dir, 'lib', 'vmpooler', 'providers', 'vsphere.rb') ] - expect(Vmpooler::Providers.load_all_providers).to eq(p) + expect(Vmpooler::Providers.load_all_providers).to match_array(p) end it '#installed_providers' do @@ -43,7 +43,7 @@ File.join(project_root_dir, 'lib', 'vmpooler', 'providers', 'dummy.rb'), File.join(project_root_dir, 'lib', 'vmpooler', 'providers', 'vsphere.rb') ] - expect(providers.load_from_gems).to eq(p) + expect(providers.load_from_gems).to match_array(p) end diff --git a/vmpooler.yaml.example b/vmpooler.yaml.example index 1480972c1..c52bcd773 100644 --- a/vmpooler.yaml.example +++ b/vmpooler.yaml.example @@ -485,6 +485,19 @@ # Expects a hash value # (optional) # +# - usage_stats +# Enable shipping of VM usage stats +# When enabled a metric is emitted when a machine is destroyed. Tags are inspected and used to organize +# shipped metrics if there is a jenkins_build_url tag set for the VM. +# Without the jenkins_build_url tag set the metric will be sent as "usage.$user.$pool_name". +# When the jenkins_build_url tag is set the metric will be sent with additional data. Here is an example +# based off of the following URL, and requested by the user ABS; +# https://jenkins.example.com/job/platform_puppet-agent-extra_puppet-agent-integration-suite_pr/RMM_COMPONENT_TO_TEST_NAME=puppet,SLAVE_LABEL=beaker,TEST_TARGET=redhat7-64a/824/ +# "usage.$user.$instance.$value_stream.$branch.$project.$job_name.$component_to_test.$pool_name", which translates to +# "usage.$user.jenkins_example_com.platform.pr.puppet-agent-extra.puppet-agent-integration-suite.puppet.$pool_name" +# Expects a boolean value +# (optional; default: false) +# # Example: :config: