diff --git a/lib/vmpooler/pool_manager.rb b/lib/vmpooler/pool_manager.rb index 15e2dfb7b..b45908962 100644 --- a/lib/vmpooler/pool_manager.rb +++ b/lib/vmpooler/pool_manager.rb @@ -287,6 +287,53 @@ def _destroy_vm(vm, pool, provider) end end + def purge_unused_vms_and_folders + global_purge = $config[:config]['purge_unconfigured_folders'] + providers = $config[:providers].keys + providers.each do |provider| + provider_purge = $config[:providers][provider]['purge_unconfigured_folders'] + provider_purge = global_purge if provider_purge.nil? + if provider_purge + Thread.new do + begin + purge_vms_and_folders($providers[provider.to_s]) + rescue => err + $logger.log('s', "[!] failed while purging provider #{provider.to_s} VMs and folders with an error: #{err}") + end + end + end + end + return + end + + # Return a list of pool folders + def pool_folders(provider) + provider_name = provider.name + folders = {} + $config[:pools].each do |pool| + next unless pool['provider'] == provider_name + folder_parts = pool['folder'].split('/') + datacenter = provider.get_target_datacenter_from_config(pool['name']) + folders[folder_parts.pop] = "#{datacenter}/vm/#{folder_parts.join('/')}" + end + folders + end + + def get_base_folders(folders) + base = [] + folders.each do |key, value| + base << value + end + base.uniq + end + + def purge_vms_and_folders(provider) + configured_folders = pool_folders(provider) + base_folders = get_base_folders(configured_folders) + whitelist = provider.provider_config['folder_whitelist'] + provider.purge_unconfigured_folders(base_folders, configured_folders, whitelist) + end + def create_vm_disk(pool_name, vm, disk_size, provider) Thread.new do begin @@ -961,6 +1008,8 @@ def execute!(maxloop = 0, loop_delay = 1) end end + purge_unused_vms_and_folders + loop_count = 1 loop do if !$threads['disk_manager'] diff --git a/lib/vmpooler/providers/base.rb b/lib/vmpooler/providers/base.rb index 71b281d8d..d30564f1b 100644 --- a/lib/vmpooler/providers/base.rb +++ b/lib/vmpooler/providers/base.rb @@ -225,6 +225,18 @@ def vm_exists?(pool_name, vm_name) def create_template_delta_disks(pool) raise("#{self.class.name} does not implement create_template_delta_disks") end + + # inputs + # [String] provider_name : Name of the provider + # returns + # Hash of folders + def get_target_datacenter_from_config(provider_name) + raise("#{self.class.name} does not implement get_target_datacenter_from_config") + end + + def purge_unconfigured_folders(base_folders, configured_folders, whitelist) + raise("#{self.class.name} does not implement purge_unconfigured_folders") + end end end end diff --git a/lib/vmpooler/providers/vsphere.rb b/lib/vmpooler/providers/vsphere.rb index d734054aa..9860514f6 100644 --- a/lib/vmpooler/providers/vsphere.rb +++ b/lib/vmpooler/providers/vsphere.rb @@ -42,6 +42,109 @@ def name 'vsphere' end + def folder_configured?(folder_title, base_folder, configured_folders, whitelist) + if whitelist + return true if whitelist.include?(folder_title) + end + return false unless configured_folders.keys.include?(folder_title) + return false unless configured_folders[folder_title] == base_folder + return true + end + + def destroy_vm_and_log(vm_name, vm_object, pool, data_ttl) + try = 0 if try.nil? + max_tries = 3 + $redis.srem("vmpooler__completed__#{pool}", vm_name) + $redis.hdel("vmpooler__active__#{pool}", vm_name) + $redis.hset("vmpooler__vm__#{vm_name}", 'destroy', Time.now) + + # Auto-expire metadata key + $redis.expire('vmpooler__vm__' + vm_name, (data_ttl * 60 * 60)) + + start = Time.now + + if vm_object.is_a? RbVmomi::VIM::Folder + logger.log('s', "[!] [#{pool}] '#{vm_name}' is a folder, bailing on destroying") + raise('Expected VM, but received a folder object') + end + vm_object.PowerOffVM_Task.wait_for_completion if vm_object.runtime && vm_object.runtime.powerState && vm_object.runtime.powerState == 'poweredOn' + vm_object.Destroy_Task.wait_for_completion + + finish = format('%.2f', Time.now - start) + logger.log('s', "[-] [#{pool}] '#{vm_name}' destroyed in #{finish} seconds") + metrics.timing("destroy.#{pool}", finish) + rescue RuntimeError + raise + rescue => err + try += 1 + logger.log('s', "[!] [#{pool}] failed to destroy '#{vm_name}' with an error: #{err}") + try >= max_tries ? raise : retry + end + + def destroy_folder_and_children(folder_object) + vms = {} + data_ttl = $config[:redis]['data_ttl'].to_i + folder_name = folder_object.name + unless folder_object.childEntity.count == 0 + folder_object.childEntity.each do |vm| + vms[vm.name] = vm + end + + vms.each do |vm_name, vm_object| + destroy_vm_and_log(vm_name, vm_object, folder_name, data_ttl) + end + end + destroy_folder(folder_object) + end + + def destroy_folder(folder_object) + try = 0 if try.nil? + max_tries = 3 + logger.log('s', "[-] [#{folder_object.name}] removing unconfigured folder") + folder_object.Destroy_Task.wait_for_completion + rescue + try += 1 + try >= max_tries ? raise : retry + end + + def purge_unconfigured_folders(base_folders, configured_folders, whitelist) + @connection_pool.with_metrics do |pool_object| + connection = ensured_vsphere_connection(pool_object) + + base_folders.each do |base_folder| + folder_children = get_folder_children(base_folder, connection) + unless folder_children.empty? + folder_children.each do |folder_hash| + folder_hash.each do |folder_title, folder_object| + unless folder_configured?(folder_title, base_folder, configured_folders, whitelist) + destroy_folder_and_children(folder_object) + end + end + end + end + end + end + end + + def get_folder_children(folder_name, connection) + folders = [] + + propSpecs = { + :entity => self, + :inventoryPath => folder_name + } + folder_object = connection.searchIndex.FindByInventoryPath(propSpecs) + + return folders if folder_object.nil? + + folder_object.childEntity.each do |folder| + next unless folder.is_a? RbVmomi::VIM::Folder + folders << { folder.name => folder } + end + + folders + end + def vms_in_pool(pool_name) vms = [] @connection_pool.with_metrics do |pool_object| diff --git a/spec/unit/pool_manager_spec.rb b/spec/unit/pool_manager_spec.rb index 97196f0ad..969dfbb84 100644 --- a/spec/unit/pool_manager_spec.rb +++ b/spec/unit/pool_manager_spec.rb @@ -802,6 +802,140 @@ end end + describe '#purge_unused_vms_and_folders' do + let(:config) { YAML.load(<<-EOT +--- +:config: {} +:providers: + :mock: {} +:pools: + - name: '#{pool}' + size: 1 +EOT + ) + } + + it 'should return when purging is not enabled' do + expect(subject.purge_unused_vms_and_folders).to be_nil + end + + context 'with purging enabled globally' do + before(:each) do + config[:config]['purge_unconfigured_folders'] = true + expect(Thread).to receive(:new).and_yield + end + + it 'should run a purge for each provider' do + expect(subject).to receive(:purge_vms_and_folders) + + subject.purge_unused_vms_and_folders + end + + it 'should log when purging fails' do + expect(subject).to receive(:purge_vms_and_folders).and_raise(RuntimeError,'MockError') + expect(logger).to receive(:log).with('s', '[!] failed while purging provider mock VMs and folders with an error: MockError') + + subject.purge_unused_vms_and_folders + end + end + + context 'with purging enabled on the provider' do + before(:each) do + config[:providers][:mock]['purge_unconfigured_folders'] = true + expect(Thread).to receive(:new).and_yield + end + + it 'should run a purge for the provider' do + expect(subject).to receive(:purge_vms_and_folders) + + subject.purge_unused_vms_and_folders + end + end + end + + describe '#pool_folders' do + let(:folder_name) { 'myinstance' } + let(:folder_base) { 'vmpooler' } + let(:folder) { [folder_base,folder_name].join('/') } + let(:datacenter) { 'dc1' } + let(:provider_name) { 'mock_provider' } + let(:expected_response) { + { + folder_name => "#{datacenter}/vm/#{folder_base}" + } + } + let(:config) { YAML.load(<<-EOT +--- +:providers: + :mock: +:pools: + - name: '#{pool}' + folder: '#{folder}' + size: 1 + datacenter: '#{datacenter}' + provider: '#{provider_name}' + - name: '#{pool}2' + folder: '#{folder}' + size: 1 + datacenter: '#{datacenter}' + provider: '#{provider_name}2' +EOT + ) + } + + it 'should return a list of pool folders' do + expect(provider).to receive(:get_target_datacenter_from_config).with(pool).and_return(datacenter) + + expect(subject.pool_folders(provider)).to eq(expected_response) + end + + it 'should raise an error when the provider fails to get the datacenter' do + expect(provider).to receive(:get_target_datacenter_from_config).with(pool).and_raise('mockerror') + + expect{ subject.pool_folders(provider) }.to raise_error(RuntimeError, 'mockerror') + end + end + + describe '#purge_vms_and_folders' do + let(:folder_name) { 'myinstance' } + let(:folder_base) { 'vmpooler' } + let(:datacenter) { 'dc1' } + let(:full_folder_path) { "#{datacenter}/vm/folder_base" } + let(:configured_folders) { { folder_name => full_folder_path } } + let(:base_folders) { [ full_folder_path ] } + let(:folder) { [folder_base,folder_name].join('/') } + let(:provider_name) { 'mock_provider' } + let(:whitelist) { nil } + let(:config) { YAML.load(<<-EOT +--- +:config: {} +:providers: + :mock_provider: {} +:pools: + - name: '#{pool}' + folder: '#{folder}' + size: 1 + datacenter: '#{datacenter}' + provider: '#{provider_name}' +EOT + ) + } + + it 'should run purge_unconfigured_folders' do + expect(subject).to receive(:pool_folders).and_return(configured_folders) + expect(provider).to receive(:purge_unconfigured_folders).with(base_folders, configured_folders, whitelist) + + subject.purge_vms_and_folders(provider) + end + + it 'should raise any errors' do + expect(subject).to receive(:pool_folders).and_return(configured_folders) + expect(provider).to receive(:purge_unconfigured_folders).with(base_folders, configured_folders, whitelist).and_raise('mockerror') + + expect{ subject.purge_vms_and_folders(provider) }.to raise_error(RuntimeError, 'mockerror') + end + end + describe '#create_vm_disk' do let(:provider) { double('provider') } let(:disk_size) { 15 } @@ -2080,6 +2214,8 @@ let(:config) { YAML.load(<<-EOT --- +:providers: + :vsphere: {} :pools: - name: #{pool} - name: 'dummy' diff --git a/spec/unit/providers/vsphere_spec.rb b/spec/unit/providers/vsphere_spec.rb index 12d32f800..c60c4d4e9 100644 --- a/spec/unit/providers/vsphere_spec.rb +++ b/spec/unit/providers/vsphere_spec.rb @@ -1,4 +1,5 @@ require 'spec_helper' +require 'mock_redis' RSpec::Matchers.define :relocation_spec_with_host do |value| match { |actual| actual[:spec].host == value } @@ -87,6 +88,289 @@ end end + describe '#folder_configured?' do + let(:folder_title) { 'folder1' } + let(:other_folder) { 'folder2' } + let(:base_folder) { 'dc1/vm/base' } + let(:configured_folders) { { folder_title => base_folder } } + let(:whitelist) { nil } + it 'should return true when configured_folders includes the folder_title' do + expect(subject.folder_configured?(folder_title, base_folder, configured_folders, whitelist)).to be true + end + + it 'should return false when title is not in configured_folders' do + expect(subject.folder_configured?(other_folder, base_folder, configured_folders, whitelist)).to be false + end + + context 'with another base folder' do + let(:base_folder) { 'dc2/vm/base' } + let(:configured_folders) { { folder_title => 'dc1/vm/base' } } + it 'should return false' do + expect(subject.folder_configured?(folder_title, base_folder, configured_folders, whitelist)).to be false + end + end + + context 'with a whitelist set' do + let(:whitelist) { [ other_folder ] } + it 'should return true' do + expect(subject.folder_configured?(other_folder, base_folder, configured_folders, whitelist)).to be true + end + end + + context 'with string whitelist value' do + let(:whitelist) { 'whitelist' } + it 'should raise an error' do + expect(whitelist).to receive(:include?).and_raise('mockerror') + + expect{ subject.folder_configured?(other_folder, base_folder, configured_folders, whitelist) }.to raise_error(RuntimeError, 'mockerror') + end + end + end + + describe '#destroy_vm_and_log' do + let(:vm_object) { mock_RbVmomi_VIM_VirtualMachine({ + :name => vmname, + :powerstate => 'poweredOn', + }) + } + let(:pool) { 'pool1' } + let(:power_off_task) { mock_RbVmomi_VIM_Task() } + let(:destroy_task) { mock_RbVmomi_VIM_Task() } + let(:data_ttl) { 1 } + let(:redis) { MockRedis.new } + let(:finish) { '0.00' } + let(:now) { Time.now } + + context 'when destroying a vm' do + before(:each) do + allow(power_off_task).to receive(:wait_for_completion) + allow(destroy_task).to receive(:wait_for_completion) + allow(vm_object).to receive(:PowerOffVM_Task).and_return(power_off_task) + allow(vm_object).to receive(:Destroy_Task).and_return(destroy_task) + $redis = redis + end + + it 'should remove redis data and expire the vm key' do + allow(Time).to receive(:now).and_return(now) + expect(redis).to receive(:srem).with("vmpooler__completed__#{pool}", vmname) + expect(redis).to receive(:hdel).with("vmpooler__active__#{pool}", vmname) + expect(redis).to receive(:hset).with("vmpooler__vm__#{vmname}", 'destroy', now) + expect(redis).to receive(:expire).with("vmpooler__vm__#{vmname}", data_ttl * 60 * 60) + + subject.destroy_vm_and_log(vmname, vm_object, pool, data_ttl) + end + + it 'should log a message that the vm is destroyed' do + expect(logger).to receive(:log).with('s', "[-] [#{pool}] '#{vmname}' destroyed in #{finish} seconds") + + subject.destroy_vm_and_log(vmname, vm_object, pool, data_ttl) + end + + it 'should record metrics' do + expect(metrics).to receive(:timing).with("destroy.#{pool}", finish) + + subject.destroy_vm_and_log(vmname, vm_object, pool, data_ttl) + end + + it 'should power off and destroy the vm' do + allow(destroy_task).to receive(:wait_for_completion) + expect(vm_object).to receive(:PowerOffVM_Task).and_return(power_off_task) + expect(vm_object).to receive(:Destroy_Task).and_return(destroy_task) + + subject.destroy_vm_and_log(vmname, vm_object, pool, data_ttl) + end + end + + context 'with a powered off vm' do + before(:each) do + vm_object.runtime.powerState = 'poweredOff' + end + + it 'should destroy the vm without attempting to power it off' do + allow(destroy_task).to receive(:wait_for_completion) + expect(vm_object).to_not receive(:PowerOffVM_Task) + expect(vm_object).to receive(:Destroy_Task).and_return(destroy_task) + + subject.destroy_vm_and_log(vmname, vm_object, pool, data_ttl) + end + end + + context 'with a folder object' do + let(:folder_object) { mock_RbVmomi_VIM_Folder({ :name => vmname }) } + + it 'should log that a folder object was received' do + expect(logger).to receive(:log).with('s', "[!] [#{pool}] '#{vmname}' is a folder, bailing on destroying") + + expect{ subject.destroy_vm_and_log(vmname, folder_object, pool, data_ttl) }.to raise_error(RuntimeError, 'Expected VM, but received a folder object') + end + + it 'should raise an error' do + expect{ subject.destroy_vm_and_log(vmname, folder_object, pool, data_ttl) }.to raise_error(RuntimeError, 'Expected VM, but received a folder object') + end + end + + context 'with an error that is not a RuntimeError' do + it 'should retry three times' do + expect(vm_object).to receive(:PowerOffVM_Task).and_throw(:powerofffailed, 'failed').exactly(3).times + + expect{ subject.destroy_vm_and_log(vmname, vm_object, pool, data_ttl) }.to raise_error(/failed/) + end + end + end + + describe '#destroy_folder_and_children' do + let(:data_ttl) { 1 } + let(:config) { + { + redis: { + 'data_ttl' => data_ttl + } + } + } + let(:foldername) { 'pool1' } + let(:folder_object) { mock_RbVmomi_VIM_Folder({ :name => foldername }) } + + before(:each) do + $config = config + end + + context 'with an empty folder' do + it 'should destroy the folder' do + expect(subject).to_not receive(:destroy_vm_and_log) + expect(subject).to receive(:destroy_folder).with(folder_object).and_return(nil) + + subject.destroy_folder_and_children(folder_object) + end + end + + context 'with a folder containing vms' do + let(:vm_object) { mock_RbVmomi_VIM_VirtualMachine({ :name => vmname }) } + before(:each) do + folder_object.childEntity << vm_object + end + + it 'should destroy the vms' do + allow(subject).to receive(:destroy_vm_and_log).and_return(nil) + allow(subject).to receive(:destroy_folder).and_return(nil) + expect(subject).to receive(:destroy_vm_and_log).with(vmname, vm_object, foldername, data_ttl) + + subject.destroy_folder_and_children(folder_object) + end + end + + it 'should raise any errors' do + expect(subject).to receive(:destroy_folder).and_throw('mockerror') + + expect{ subject.destroy_folder_and_children(folder_object) }.to raise_error(/mockerror/) + end + end + + describe '#destroy_folder' do + let(:foldername) { 'pool1' } + let(:folder_object) { mock_RbVmomi_VIM_Folder({ :name => foldername }) } + let(:destroy_task) { mock_RbVmomi_VIM_Task() } + + before(:each) do + allow(folder_object).to receive(:Destroy_Task).and_return(destroy_task) + allow(destroy_task).to receive(:wait_for_completion) + end + + it 'should destroy the folder' do + expect(folder_object).to receive(:Destroy_Task).and_return(destroy_task) + + subject.destroy_folder(folder_object) + end + + it 'should log that the folder is being destroyed' do + expect(logger).to receive(:log).with('s', "[-] [#{foldername}] removing unconfigured folder") + + subject.destroy_folder(folder_object) + end + + it 'should retry three times when failing' do + expect(folder_object).to receive(:Destroy_Task).and_throw('mockerror').exactly(3).times + + expect{ subject.destroy_folder(folder_object) }.to raise_error(/mockerror/) + end + end + + describe '#purge_unconfigured_folders' do + let(:folder_title) { 'folder1' } + let(:base_folder) { 'dc1/vm/base' } + let(:folder_object) { mock_RbVmomi_VIM_Folder({ :name => base_folder }) } + let(:child_folder) { mock_RbVmomi_VIM_Folder({ :name => folder_title }) } + let(:whitelist) { nil } + let(:base_folders) { [ base_folder ] } + let(:configured_folders) { { folder_title => base_folder } } + let(:folder_children) { [ folder_title => child_folder ] } + let(:empty_list) { [] } + + before(:each) do + allow(subject).to receive(:connect_to_vsphere).and_return(connection) + end + + context 'with an empty folder' do + it 'should not attempt to destroy any folders' do + expect(subject).to receive(:get_folder_children).with(base_folder, connection).and_return(empty_list) + expect(subject).to_not receive(:destroy_folder_and_children) + + subject.purge_unconfigured_folders(base_folders, configured_folders, whitelist) + end + end + + it 'should retrieve the folder children' do + expect(subject).to receive(:get_folder_children).with(base_folder, connection).and_return(folder_children) + allow(subject).to receive(:folder_configured?).and_return(true) + + subject.purge_unconfigured_folders(base_folders, configured_folders, whitelist) + end + + context 'with a folder that is not configured' do + before(:each) do + expect(subject).to receive(:get_folder_children).with(base_folder, connection).and_return(folder_children) + allow(subject).to receive(:folder_configured?).and_return(false) + end + + it 'should destroy the folder and children' do + expect(subject).to receive(:destroy_folder_and_children).with(child_folder).and_return(nil) + + subject.purge_unconfigured_folders(base_folders, configured_folders, whitelist) + end + end + + it 'should raise any errors' do + expect(subject).to receive(:get_folder_children).and_throw('mockerror') + + expect{ subject.purge_unconfigured_folders(base_folders, configured_folders, whitelist) }.to raise_error(/mockerror/) + end + end + + describe '#get_folder_children' do + let(:base_folder) { 'dc1/vm/base' } + let(:base_folder_object) { mock_RbVmomi_VIM_Folder({ :name => base_folder }) } + let(:foldername) { 'folder1' } + let(:folder_object) { mock_RbVmomi_VIM_Folder({ :name => foldername }) } + let(:folder_return) { [ { foldername => folder_object } ] } + + before(:each) do + base_folder_object.childEntity << folder_object + end + + it 'should return an array of configured folder hashes' do + expect(connection.searchIndex).to receive(:FindByInventoryPath).and_return(base_folder_object) + + result = subject.get_folder_children(foldername, connection) + + expect(result).to eq(folder_return) + end + + it 'should raise any errors' do + expect(connection.searchIndex).to receive(:FindByInventoryPath).and_throw('mockerror') + + expect{ subject.get_folder_children(foldername, connection) }.to raise_error(/mockerror/) + end + end + describe '#vms_in_pool' do let(:folder_object) { mock_RbVmomi_VIM_Folder({ :name => 'pool1'}) } let(:pool_config) { config[:pools][0] } diff --git a/vmpooler.yaml.example b/vmpooler.yaml.example index 03e0e388f..5137aa857 100644 --- a/vmpooler.yaml.example +++ b/vmpooler.yaml.example @@ -11,6 +11,22 @@ # For multiple providers, specify one of the supported backing services (vsphere or dummy) # (optional: will default to it's parent :key: name eg. 'vsphere') # +# - purge_unconfigured_folders +# Enable purging of VMs and folders detected within the base folder path that are not configured for the provider +# Only a single layer of folders and their child VMs are evaluated from detected base folder paths +# Nested child folders will not be destroyed. An optional whitelist can be provided to exclude folders +# A base folder path for 'vmpooler/redhat-7' would be 'vmpooler' +# Setting this on the provider will enable folder purging for the provider +# Expects a boolean value +# (optional; default: false) +# +# - folder_whitelist +# Specify folders that are within the base folder path, not in the configuration, and should not be destroyed +# To exclude 'vmpooler/myfolder' from being destroyed when the base path is 'vmpooler' you would specify 'myfolder' in the whitelist +# This option is only evaluated when 'purge_unconfigured_folders' is enabled +# Expects an array of strings specifying the whitelisted folders by name +# (optional; default: nil) +# # If you want to support more than one provider with different parameters (server, username or passwords) you have to specify the # backing service in the provider_class configuration parameter for example 'vsphere' or 'dummy'. Each pool can specify # the provider to use. @@ -298,6 +314,7 @@ # # - base # The base DN used for LDAP searches. +# This can be a string providing a single DN, or an array of DNs to search. # # - user_object # The LDAP object-type used to designate a user object. @@ -448,6 +465,14 @@ # Expects a boolean value # (optional; default: false) # +# - purge_unconfigured_folders (vSphere Provider only) +# Enable purging of VMs and folders detected within the base folder path that are not configured for the provider +# Only a single layer of folders and their child VMs are evaluated from detected base folder paths +# A base folder path for 'vmpooler/redhat-7' would be 'vmpooler' +# When enabled in the global configuration then purging is enabled for all providers +# Expects a boolean value +# (optional; default: false) +# # Example: :config: