(POOLER-66) Purge vms and folders no longer configured

lib/vmpooler/pool_manager.rb

@@ -287,6 +287,53 @@ def _destroy_vm(vm, pool, provider)
       end
     end
 
+    def purge_unused_vms_and_folders
+      global_purge = $config[:config]['purge_unconfigured_folders']
+      providers = $config[:providers].keys
+      providers.each do |provider|
+        provider_purge = $config[:providers][provider]['purge_unconfigured_folders']
+        provider_purge = global_purge if provider_purge.nil?
+        if provider_purge
+          Thread.new do
+            begin
+              purge_vms_and_folders($providers[provider.to_s])
+            rescue => err
+              $logger.log('s', "[!] failed while purging provider #{provider.to_s} VMs and folders with an error: #{err}")
+            end
+          end
+        end
+      end
+      return
+    end
+
+    # Return a list of pool folders
+    def pool_folders(provider)
+      provider_name = provider.name
+      folders = {}
+      $config[:pools].each do |pool|
+        next unless pool['provider'] == provider_name
+        folder_parts = pool['folder'].split('/')
+        datacenter = provider.get_target_datacenter_from_config(pool['name'])
+        folders[folder_parts.pop] = "#{datacenter}/vm/#{folder_parts.join('/')}"
+      end
+      folders
+    end
+
+    def get_base_folders(folders)
+      base = []
+      folders.each do |key, value|
+        base << value
+      end
+      base.uniq
+    end
+
+    def purge_vms_and_folders(provider)
+      configured_folders = pool_folders(provider)
+      base_folders = get_base_folders(configured_folders)
+      whitelist = provider.provider_config['folder_whitelist']
+      provider.purge_unconfigured_folders(base_folders, configured_folders, whitelist)
+    end
+
     def create_vm_disk(pool_name, vm, disk_size, provider)
       Thread.new do
         begin
@@ -961,6 +1008,8 @@ def execute!(maxloop = 0, loop_delay = 1)
         end
       end
 
+      purge_unused_vms_and_folders
+
       loop_count = 1
       loop do
         if !$threads['disk_manager']

lib/vmpooler/providers/base.rb

@@ -225,6 +225,18 @@ def vm_exists?(pool_name, vm_name)
         def create_template_delta_disks(pool)
           raise("#{self.class.name} does not implement create_template_delta_disks")
         end
+
+        # inputs
+        #   [String] provider_name : Name of the provider
+        # returns
+        #   Hash of folders
+        def get_target_datacenter_from_config(provider_name)
+          raise("#{self.class.name} does not implement get_target_datacenter_from_config")
+        end
+
+        def purge_unconfigured_folders(base_folders, configured_folders, whitelist)
+          raise("#{self.class.name} does not implement purge_unconfigured_folders")
+        end
       end
     end
   end

lib/vmpooler/providers/vsphere.rb

@@ -42,6 +42,109 @@ def name
           'vsphere'
         end
 
+        def folder_configured?(folder_title, base_folder, configured_folders, whitelist)
+          if whitelist
+            return true if whitelist.include?(folder_title)
+          end
+          return false unless configured_folders.keys.include?(folder_title)
+          return false unless configured_folders[folder_title] == base_folder
+          return true
+        end
+
+        def destroy_vm_and_log(vm_name, vm_object, pool, data_ttl)
+          try = 0 if try.nil?
+          max_tries = 3
+          $redis.srem("vmpooler__completed__#{pool}", vm_name)
+          $redis.hdel("vmpooler__active__#{pool}", vm_name)
+          $redis.hset("vmpooler__vm__#{vm_name}", 'destroy', Time.now)
+
+          # Auto-expire metadata key
+          $redis.expire('vmpooler__vm__' + vm_name, (data_ttl * 60 * 60))
+
+          start = Time.now
+
+          if vm_object.is_a? RbVmomi::VIM::Folder
+            logger.log('s', "[!] [#{pool}] '#{vm_name}' is a folder, bailing on destroying")
+            raise('Expected VM, but received a folder object')
+          end
+          vm_object.PowerOffVM_Task.wait_for_completion if vm_object.runtime && vm_object.runtime.powerState && vm_object.runtime.powerState == 'poweredOn'
+          vm_object.Destroy_Task.wait_for_completion
+
+          finish = format('%.2f', Time.now - start)
+          logger.log('s', "[-] [#{pool}] '#{vm_name}' destroyed in #{finish} seconds")
+          metrics.timing("destroy.#{pool}", finish)
+        rescue RuntimeError
+          raise
+        rescue => err
+          try += 1
+          logger.log('s', "[!] [#{pool}] failed to destroy '#{vm_name}' with an error: #{err}")
+          try >= max_tries ? raise : retry
+        end
+
+        def destroy_folder_and_children(folder_object)
+          vms = {}
+          data_ttl = $config[:redis]['data_ttl'].to_i
+          folder_name = folder_object.name
+          unless folder_object.childEntity.count == 0
+            folder_object.childEntity.each do |vm|
+              vms[vm.name] = vm
+            end
+
+            vms.each do |vm_name, vm_object|
+              destroy_vm_and_log(vm_name, vm_object, folder_name, data_ttl)
+            end
+          end
+          destroy_folder(folder_object)
+        end
+
+        def destroy_folder(folder_object)
+          try = 0 if try.nil?
+          max_tries = 3
+          logger.log('s', "[-] [#{folder_object.name}] removing unconfigured folder")
+          folder_object.Destroy_Task.wait_for_completion
+        rescue
+          try += 1
+          try >= max_tries ? raise : retry
+        end
+
+        def purge_unconfigured_folders(base_folders, configured_folders, whitelist)
+          @connection_pool.with_metrics do |pool_object|
+            connection = ensured_vsphere_connection(pool_object)
+
+            base_folders.each do |base_folder|
+              folder_children = get_folder_children(base_folder, connection)
+              unless folder_children.empty?
+                folder_children.each do |folder_hash|
+                  folder_hash.each do |folder_title, folder_object|
+                    unless folder_configured?(folder_title, base_folder, configured_folders, whitelist)
+                      destroy_folder_and_children(folder_object)
+                    end
+                  end
+                end
+              end
+            end
+          end
+        end
+
+        def get_folder_children(folder_name, connection)
+          folders = []
+
+          propSpecs = {
+            :entity => self,
+            :inventoryPath => folder_name
+          }
+          folder_object = connection.searchIndex.FindByInventoryPath(propSpecs)
+
+          return folders if folder_object.nil?
+
+          folder_object.childEntity.each do |folder|
+            next unless folder.is_a? RbVmomi::VIM::Folder
+            folders << { folder.name => folder }
+          end
+
+          folders
+        end
+
         def vms_in_pool(pool_name)
           vms = []
           @connection_pool.with_metrics do |pool_object|

spec/unit/pool_manager_spec.rb

@@ -802,6 +802,140 @@
     end
   end
 
+  describe '#purge_unused_vms_and_folders' do
+    let(:config) { YAML.load(<<-EOT
+---
+:config: {}
+:providers:
+  :mock: {}
+:pools:
+  - name: '#{pool}'
+    size: 1
+EOT
+      )
+    }
+
+    it 'should return when purging is not enabled' do
+      expect(subject.purge_unused_vms_and_folders).to be_nil
+    end
+
+    context 'with purging enabled globally' do
+      before(:each) do
+        config[:config]['purge_unconfigured_folders'] = true
+        expect(Thread).to receive(:new).and_yield
+      end
+
+      it 'should run a purge for each provider' do
+        expect(subject).to receive(:purge_vms_and_folders)
+
+        subject.purge_unused_vms_and_folders
+      end
+
+      it 'should log when purging fails' do
+        expect(subject).to receive(:purge_vms_and_folders).and_raise(RuntimeError,'MockError')
+        expect(logger).to receive(:log).with('s', '[!] failed while purging provider mock VMs and folders with an error: MockError')
+
+        subject.purge_unused_vms_and_folders
+      end
+    end
+
+    context 'with purging enabled on the provider' do
+      before(:each) do
+        config[:providers][:mock]['purge_unconfigured_folders'] = true
+        expect(Thread).to receive(:new).and_yield
+      end
+
+      it 'should run a purge for the provider' do
+        expect(subject).to receive(:purge_vms_and_folders)
+
+        subject.purge_unused_vms_and_folders
+      end
+    end
+  end
+
+  describe '#pool_folders' do
+    let(:folder_name) { 'myinstance' }
+    let(:folder_base) { 'vmpooler' }
+    let(:folder) { [folder_base,folder_name].join('/') }
+    let(:datacenter) { 'dc1' }
+    let(:provider_name) { 'mock_provider' }
+    let(:expected_response) {
+      {
+        folder_name => "#{datacenter}/vm/#{folder_base}"
+      }
+    }
+    let(:config) { YAML.load(<<-EOT
+---
+:providers:
+  :mock:
+:pools:
+  - name: '#{pool}'
+    folder: '#{folder}'
+    size: 1
+    datacenter: '#{datacenter}'
+    provider: '#{provider_name}'
+  - name: '#{pool}2'
+    folder: '#{folder}'
+    size: 1
+    datacenter: '#{datacenter}'
+    provider: '#{provider_name}2'
+EOT
+      )
+    }
+
+    it 'should return a list of pool folders' do
+      expect(provider).to receive(:get_target_datacenter_from_config).with(pool).and_return(datacenter)
+
+      expect(subject.pool_folders(provider)).to eq(expected_response)
+    end
+
+    it 'should raise an error when the provider fails to get the datacenter' do
+      expect(provider).to receive(:get_target_datacenter_from_config).with(pool).and_raise('mockerror')
+
+      expect{ subject.pool_folders(provider) }.to raise_error(RuntimeError, 'mockerror')
+    end
+  end
+
+  describe '#purge_vms_and_folders' do
+    let(:folder_name) { 'myinstance' }
+    let(:folder_base) { 'vmpooler' }
+    let(:datacenter) { 'dc1' }
+    let(:full_folder_path) { "#{datacenter}/vm/folder_base" }
+    let(:configured_folders) { { folder_name => full_folder_path } }
+    let(:base_folders) { [ full_folder_path ] }
+    let(:folder) { [folder_base,folder_name].join('/') }
+    let(:provider_name) { 'mock_provider' }
+    let(:whitelist) { nil }
+    let(:config) { YAML.load(<<-EOT
+---
+:config: {}
+:providers:
+  :mock_provider: {}
+:pools:
+  - name: '#{pool}'
+    folder: '#{folder}'
+    size: 1
+    datacenter: '#{datacenter}'
+    provider: '#{provider_name}'
+EOT
+      )
+    }
+
+    it 'should run purge_unconfigured_folders' do
+      expect(subject).to receive(:pool_folders).and_return(configured_folders)
+      expect(provider).to receive(:purge_unconfigured_folders).with(base_folders, configured_folders, whitelist)
+
+      subject.purge_vms_and_folders(provider)
+    end
+
+    it 'should raise any errors' do
+      expect(subject).to receive(:pool_folders).and_return(configured_folders)
+      expect(provider).to receive(:purge_unconfigured_folders).with(base_folders, configured_folders, whitelist).and_raise('mockerror')
+
+      expect{ subject.purge_vms_and_folders(provider) }.to raise_error(RuntimeError, 'mockerror')
+    end
+  end
+
   describe '#create_vm_disk' do
     let(:provider) { double('provider') }
     let(:disk_size) { 15 }
@@ -2080,6 +2214,8 @@
         let(:config) {
         YAML.load(<<-EOT
 ---
+:providers:
+  :vsphere: {}
 :pools:
   - name: #{pool}
   - name: 'dummy'