From fb053414a988e3c148b3712f723d54b5a7a3ee51 Mon Sep 17 00:00:00 2001 From: Gene Liverman Date: Thu, 14 Oct 2021 15:08:55 -0400 Subject: [PATCH] (DIO-2675) Undo pool size template overrides This implements a delete method for pooltemplate that removes the override hash from redis and then resets triggers a reset of the pool. It also implements a delete method for poolsize that removes the override hash from redis. Unlike pooltemplate, it does not reset the pool. --- docs/API.md | 42 +++++++++++ lib/vmpooler.rb | 5 ++ lib/vmpooler/api/v1.rb | 98 ++++++++++++++++++++++++++ lib/vmpooler/pool_manager.rb | 52 +++++++++++++- spec/integration/api/v1/config_spec.rb | 95 +++++++++++++++++++++++++ spec/unit/pool_manager_spec.rb | 98 ++++++++++++++++++++++++++ 6 files changed, 388 insertions(+), 2 deletions(-) diff --git a/docs/API.md b/docs/API.md index 038d4e9dd..02749e773 100644 --- a/docs/API.md +++ b/docs/API.md @@ -743,6 +743,27 @@ $ curl -X POST -H "Content-Type: application/json" -d '{"debian-7-i386":"2","deb } ``` +##### DELETE /config/poolsize/<pool> + +Delete an overridden pool size. This results in the values from VMPooler's config being used. + +Return codes: +* 200 - when nothing was changed but no error occurred +* 201 - size reset successful +* 401 - when not authorized +* 404 - pool does not exist +* 405 - The endpoint is disabled because experimental features are disabled + +``` +$ curl -X DELETE -u jdoe --url vmpooler.example.com/api/v1/poolsize/almalinux-8-x86_64 +Enter host password for user 'jdoe': +``` +```json +{ + "ok": true +} +``` + ##### POST /config/pooltemplate Change the template configured for a pool, and replenish the pool with instances built from the new template. @@ -775,6 +796,27 @@ $ curl -X POST -H "Content-Type: application/json" -d '{"debian-7-i386":"templat } ``` +##### DELETE /config/pooltemplate/<pool> + +Delete an overridden pool template. This results in the values from VMPooler's config being used. + +Return codes: +* 200 - when nothing was changed but no error occurred +* 201 - template reset successful +* 401 - when not authorized +* 404 - pool does not exist +* 405 - The endpoint is disabled because experimental features are disabled + +``` +$ curl -X DELETE -u jdoe --url vmpooler.example.com/api/v1/pooltemplate/almalinux-8-x86_64 +Enter host password for user 'jdoe': +``` +```json +{ + "ok": true +} +``` + ##### POST /poolreset Clear all pending and ready instances in a pool, and deploy replacements diff --git a/lib/vmpooler.rb b/lib/vmpooler.rb index 340389620..e8dd462eb 100644 --- a/lib/vmpooler.rb +++ b/lib/vmpooler.rb @@ -133,8 +133,13 @@ def self.config(filepath = 'vmpooler.yaml') parsed_config[:pools] = load_pools_from_redis(redis) end + # retain a copy of the pools that were observed at startup + serialized_pools = Marshal.dump(parsed_config[:pools]) + parsed_config[:pools_at_startup] = Marshal.load(serialized_pools) + # Create an index of pools by title parsed_config[:pool_index] = pool_index(parsed_config[:pools]) + parsed_config[:pool_index_at_startup] = pool_index(parsed_config[:pools_at_startup]) parsed_config[:pools].each do |pool| parsed_config[:pool_names] << pool['name'] diff --git a/lib/vmpooler/api/v1.rb b/lib/vmpooler/api/v1.rb index 2fffee53b..ae469f15d 100644 --- a/lib/vmpooler/api/v1.rb +++ b/lib/vmpooler/api/v1.rb @@ -28,6 +28,10 @@ def pools Vmpooler::API.settings.config[:pools] end + def pools_at_startup + Vmpooler::API.settings.config[:pools_at_startup] + end + def pool_exists?(template) Vmpooler::API.settings.config[:pool_names].include?(template) end @@ -289,6 +293,33 @@ def update_user_metrics(operation, vmname) puts 'd', "[!] [#{poolname}] failed while evaluating usage labels on '#{vmname}' with an error: #{e}" end + def reset_pool_size(poolname) + result = { 'ok' => false } + + pool_index_live = pool_index(pools) + pool_index_original = pool_index(pools_at_startup) + + pools_updated = 0 + sync_pool_sizes + + pool_size_now = pools[pool_index_live[poolname]]['size'] + pool_size_original = pools_at_startup[pool_index_original[poolname]]['size'] + result['pool_size_before_reset'] = pool_size_now + result['pool_size_before_overrides'] = pool_size_original + + unless pool_size_now.to_i == pool_size_original.to_i + pools[pool_index_live[poolname]]['size'] = pool_size_original.to_i + backend.hdel('vmpooler__config__poolsize', poolname) + backend.sadd('vmpooler__pool__undo_size_override', poolname) + pools_updated += 1 + status 201 + end + + status 200 unless pools_updated > 0 + result['ok'] = true + result + end + def update_pool_size(payload) result = { 'ok' => false } @@ -309,6 +340,33 @@ def update_pool_size(payload) result end + def reset_pool_template(poolname) + result = { 'ok' => false } + + pool_index_live = pool_index(pools) + pool_index_original = pool_index(pools_at_startup) + + pools_updated = 0 + sync_pool_templates + + template_now = pools[pool_index_live[poolname]]['template'] + template_original = pools_at_startup[pool_index_original[poolname]]['template'] + result['template_before_reset'] = template_now + result['template_before_overrides'] = template_original + + unless template_now == template_original + pools[pool_index_live[poolname]]['template'] = template_original + backend.hdel('vmpooler__config__template', poolname) + backend.sadd('vmpooler__pool__undo_template_override', poolname) + pools_updated += 1 + status 201 + end + + status 200 unless pools_updated > 0 + result['ok'] = true + result + end + def update_pool_template(payload) result = { 'ok' => false } @@ -1375,6 +1433,26 @@ def delete_ondemand_request(request_id) JSON.pretty_generate(result) end + delete "#{api_prefix}/config/poolsize/:pool/?" do + content_type :json + result = { 'ok' => false } + + if config['experimental_features'] + need_token! if Vmpooler::API.settings.config[:auth] + + if pool_exists?(params[:pool]) + result = reset_pool_size(params[:pool]) + else + metrics.increment('config.invalid.unknown') + status 404 + end + else + status 405 + end + + JSON.pretty_generate(result) + end + post "#{api_prefix}/config/poolsize/?" do content_type :json result = { 'ok' => false } @@ -1406,6 +1484,26 @@ def delete_ondemand_request(request_id) JSON.pretty_generate(result) end + delete "#{api_prefix}/config/pooltemplate/:pool/?" do + content_type :json + result = { 'ok' => false } + + if config['experimental_features'] + need_token! if Vmpooler::API.settings.config[:auth] + + if pool_exists?(params[:pool]) + result = reset_pool_template(params[:pool]) + else + metrics.increment('config.invalid.unknown') + status 404 + end + else + status 405 + end + + JSON.pretty_generate(result) + end + post "#{api_prefix}/config/pooltemplate/?" do content_type :json result = { 'ok' => false } diff --git a/lib/vmpooler/pool_manager.rb b/lib/vmpooler/pool_manager.rb index 668d1c0b9..24a42ef48 100644 --- a/lib/vmpooler/pool_manager.rb +++ b/lib/vmpooler/pool_manager.rb @@ -784,6 +784,10 @@ def time_passed?(_event, time) # - Fires when a pool reset is requested # - Additional options # :poolname + # :undo_override + # - Fires when a pool override removal is requested + # - Additional options + # :poolname # def sleep_with_wakeup_events(loop_delay, wakeup_period = 5, options = {}) exit_by = Time.now + loop_delay @@ -826,6 +830,11 @@ def sleep_with_wakeup_events(loop_delay, wakeup_period = 5, options = {}) break if pending end + if options[:undo_override] + break if redis.sismember('vmpooler__pool__undo_template_override', options[:poolname]) + break if redis.sismember('vmpooler__pool__undo_size_override', options[:poolname]) + end + if options[:pending_vm] pending_vm_count = redis.scard("vmpooler__pending__#{options[:poolname]}") break unless pending_vm_count == 0 @@ -880,7 +889,7 @@ def check_pool(pool, loop_delay = (loop_delay * loop_delay_decay).to_i loop_delay = loop_delay_max if loop_delay > loop_delay_max end - sleep_with_wakeup_events(loop_delay, loop_delay_min, pool_size_change: true, poolname: pool['name'], pool_template_change: true, clone_target_change: true, pending_vm: true, pool_reset: true) + sleep_with_wakeup_events(loop_delay, loop_delay_min, pool_size_change: true, poolname: pool['name'], pool_template_change: true, clone_target_change: true, pending_vm: true, pool_reset: true, undo_override: true) unless maxloop == 0 break if loop_count >= maxloop @@ -1044,11 +1053,14 @@ def update_pool_size(pool) break if poolsize.nil? poolsize = Integer(poolsize) - break if poolsize == pool['size'] + pool_size_now = pool['size'] + break if poolsize == pool_size_now mutex.synchronize do pool['size'] = poolsize end + + $logger.log('s', "[*] [#{pool['name']}] size updated from #{pool_size_now} to #{poolsize}") end end @@ -1066,6 +1078,39 @@ def reset_pool(pool) end end + def undo_override(pool, provider) + poolname = pool['name'] + mutex = pool_mutex(poolname) + return if mutex.locked? + + + @redis.with_metrics do |redis| + break unless redis.sismember('vmpooler__pool__undo_template_override', poolname) + + redis.srem('vmpooler__pool__undo_template_override', poolname) + template_now = pool['template'] + template_original = $config[:pools_at_startup][$config[:pool_index_at_startup][poolname]]['template'] + + mutex.synchronize do + update_pool_template(pool, provider, template_original, template_now, redis) + end + end + + @redis.with_metrics do |redis| + break unless redis.sismember('vmpooler__pool__undo_size_override', poolname) + + redis.srem('vmpooler__pool__undo_size_override', poolname) + pool_size_now = pool['size'] + pool_size_original = $config[:pools_at_startup][$config[:pool_index_at_startup][poolname]]['size'] + + mutex.synchronize do + pool['size'] = pool_size_original + end + + $logger.log('s', "[*] [#{poolname}] size updated from #{pool_size_now} to #{pool_size_original}") + end + end + def create_inventory(pool, provider, pool_check_response) inventory = {} begin @@ -1300,6 +1345,9 @@ def _check_pool(pool, provider) # Reset a pool when poolreset is requested from the API reset_pool(pool) + # Undo overrides submitted via the api + undo_override(pool, provider) + pool_check_response end diff --git a/spec/integration/api/v1/config_spec.rb b/spec/integration/api/v1/config_spec.rb index 79874c8ed..62968bd53 100644 --- a/spec/integration/api/v1/config_spec.rb +++ b/spec/integration/api/v1/config_spec.rb @@ -26,6 +26,10 @@ def app() {'name' => 'pool1', 'size' => 5, 'template' => 'templates/pool1', 'clone_target' => 'default_cluster'}, {'name' => 'pool2', 'size' => 10} ], + pools_at_startup: [ + {'name' => 'pool1', 'size' => 5, 'template' => 'templates/pool1', 'clone_target' => 'default_cluster'}, + {'name' => 'pool2', 'size' => 10} + ], statsd: { 'prefix' => 'stats_prefix'}, alias: { 'poolone' => 'pool1' }, pool_names: [ 'pool1', 'pool2', 'poolone' ] @@ -45,6 +49,47 @@ def app() create_token('abcdefghijklmnopqrstuvwxyz012345', 'jdoe', current_time) end + describe 'DELETE /config/pooltemplate/:pool' do + it 'resets a pool template' do + post "#{prefix}/config/pooltemplate", '{"pool1":"templates/new_template"}' + delete "#{prefix}/config/pooltemplate/pool1" + expect_json(ok = true, http = 201) + + expected = { + ok: true, + template_before_reset: 'templates/new_template', + template_before_overrides: 'templates/pool1' + } + + expect(last_response.body).to eq(JSON.pretty_generate(expected)) + end + + it 'succeeds when the pool has not been overridden' do + delete "#{prefix}/config/pooltemplate/pool1" + expect_json(ok = true, http = 200) + end + + it 'fails on nonexistent pools' do + delete "#{prefix}/config/pooltemplate/poolpoolpool" + expect_json(ok = false, http = 404) + end + + context 'with experimental features disabled' do + before(:each) do + config[:config]['experimental_features'] = false + end + + it 'should return 405' do + delete "#{prefix}/config/pooltemplate/pool1" + expect_json(ok = false, http = 405) + + expected = { ok: false } + expect(last_response.body).to eq(JSON.pretty_generate(expected)) + end + end + + end + describe 'POST /config/pooltemplate' do it 'updates a pool template' do post "#{prefix}/config/pooltemplate", '{"pool1":"templates/new_template"}' @@ -142,6 +187,56 @@ def app() end + describe 'DELETE /config/poolsize' do + it 'resets a pool size' do + post "#{prefix}/config/poolsize", '{"pool1":"2"}' + delete "#{prefix}/config/poolsize/pool1" + expect_json(ok = true, http = 201) + + expected = { + ok: true, + pool_size_before_reset: 2, + pool_size_before_overrides: 5 + } + + expect(last_response.body).to eq(JSON.pretty_generate(expected)) + end + + it 'fails when a specified pool does not exist' do + delete "#{prefix}/config/poolsize/pool10" + expect_json(ok = false, http = 404) + expected = { ok: false } + + expect(last_response.body).to eq(JSON.pretty_generate(expected)) + end + + it 'succeeds when a pool has not been overridden' do + delete "#{prefix}/config/poolsize/pool1" + expect_json(ok = true, http = 200) + expected = { + ok: true, + pool_size_before_reset: 5, + pool_size_before_overrides: 5 + } + + expect(last_response.body).to eq(JSON.pretty_generate(expected)) + end + + context 'with experimental features disabled' do + before(:each) do + config[:config]['experimental_features'] = false + end + + it 'should return 405' do + delete "#{prefix}/config/poolsize/pool1" + expect_json(ok = false, http = 405) + + expected = { ok: false } + expect(last_response.body).to eq(JSON.pretty_generate(expected)) + end + end + end + describe 'POST /config/poolsize' do it 'changes a pool size' do post "#{prefix}/config/poolsize", '{"pool1":"2"}' diff --git a/spec/unit/pool_manager_spec.rb b/spec/unit/pool_manager_spec.rb index cc5777b18..92ac47363 100644 --- a/spec/unit/pool_manager_spec.rb +++ b/spec/unit/pool_manager_spec.rb @@ -3159,6 +3159,54 @@ end end + describe 'with the undo_override wakeup option' do + let(:wakeup_option) {{ + :undo_override => true, + :poolname => pool + }} + + let(:wakeup_period) { -1 } # A negative number forces the wakeup evaluation to always occur + + context 'when a undoing a template override is requested' do + before(:each) do + redis_connection_pool.with do |redis| + redis.sadd('vmpooler__pool__undo_template_override', pool) + allow(redis).to receive(:hget) + end + end + + it 'should sleep until the undo override request is detected' do + redis_connection_pool.with do |redis| + expect(subject).to receive(:sleep).at_least(2).times + expect(subject).to receive(:sleep).at_most(3).times + expect(redis).to receive(:sismember).with('vmpooler__pool__undo_template_override', pool).and_return(false,false,true) + expect(redis).to receive(:sismember).with('vmpooler__pool__undo_size_override', pool).and_return(false,false) + end + + subject.sleep_with_wakeup_events(loop_delay, wakeup_period, wakeup_option) + end + end + + context 'when a undoing a size override is requested' do + before(:each) do + redis_connection_pool.with do |redis| + redis.sadd('vmpooler__pool__undo_size_override', pool) + allow(redis).to receive(:hget) + end + end + + it 'should sleep until the undo override request is detected' do + redis_connection_pool.with do |redis| + expect(subject).to receive(:sleep).exactly(3).times + expect(redis).to receive(:sismember).with('vmpooler__pool__undo_template_override', pool).and_return(false,false,false) + expect(redis).to receive(:sismember).with('vmpooler__pool__undo_size_override', pool).and_return(false,false,true) + end + + subject.sleep_with_wakeup_events(loop_delay, wakeup_period, wakeup_option) + end + end + end + describe 'with the pending_vm wakeup option' do let(:wakeup_option) {{ :pending_vm => true, @@ -3477,6 +3525,56 @@ end end + describe 'undo_override' do + let(:mutex) { Mutex.new } + let(:original_template) { 'templates/template1' } + let(:override_template) { 'templates/template2' } + let(:original_size) { 2 } + let(:override_size) { 10 } + let(:config) { YAML.load(<<-EOT +--- +:config: + task_limit: 5 +:providers: + :mock: +:pools: + - name: '#{pool}' + size: #{override_size} + template: '#{override_template}' +:pool_index: + '#{pool}': 0 +:pools_at_startup: + - name: '#{pool}' + size: #{original_size} + template: '#{original_template}' +:pool_index_at_startup: + '#{pool}': 0 +EOT + ) + } + + before(:each) do + redis_connection_pool.with do |redis| + redis.sadd('vmpooler__pool__undo_template_override', pool) + redis.sadd('vmpooler__pool__undo_size_override', pool) + # allow(redis).to receive(:hget) + end + end + + it 'should revert to the original template and pool size' do + redis_connection_pool.with do |redis| + expect(redis).to receive(:sismember).with('vmpooler__pool__undo_template_override', pool).and_return(true) + expect(redis).to receive(:srem).with('vmpooler__pool__undo_template_override', pool).and_return(true) + expect(subject).to receive(:update_pool_template).with(config[:pools][0], provider, original_template, override_template, redis) + + expect(redis).to receive(:sismember).with('vmpooler__pool__undo_size_override', pool).and_return(true) + expect(redis).to receive(:srem).with('vmpooler__pool__undo_size_override', pool).and_return(true) + end + + subject.undo_override(config[:pools][0], provider) + end + end + describe '#create_inventory' do it 'should log an error if one occurs' do