Merge pull request #365 from mattkirby/pooler_156

(POOLER-156) Detect redis connection failures

docker/docker-compose.yml

@@ -9,6 +9,9 @@ services:
       - type: bind
         source: ${PWD}/vmpooler.yaml
         target: /etc/vmpooler/vmpooler.yaml
+      - type: bind
+        source: ${PWD}/providers.yaml
+        target: /etc/vmpooler/providers.yaml
     ports:
       - "4567:4567"
     networks:
@@ -17,6 +20,9 @@ services:
       - VMPOOLER_DEBUG=true # for use of dummy auth
       - VMPOOLER_CONFIG_FILE=/etc/vmpooler/vmpooler.yaml
       - REDIS_SERVER=redislocal
+      - EXTRA_CONFIG=/etc/vmpooler/providers.yaml
+      - LOGFILE=/dev/stdout
+      - LDAP_PORT=636
     image: vmpooler-local
     depends_on:
       - redislocal

lib/vmpooler/pool_manager.rb

@@ -822,6 +822,8 @@ def check_pool(pool,
               loop_count += 1
             end
           end
+        rescue Redis::CannotConnectError
+          raise
         rescue StandardError => e
           $logger.log('s', "[!] [#{pool['name']}] Error while checking the pool: #{e}")
           raise
@@ -1310,6 +1312,9 @@ def execute!(maxloop = 0, loop_delay = 1)
           loop_count += 1
         end
       end
+    rescue Redis::CannotConnectError => e
+      $logger.log('s', "Cannot connect to the redis server: #{e}")
+      raise
     end
   end
 end

spec/unit/pool_manager_spec.rb

@@ -2792,6 +2792,16 @@
         subject.execute!(maxloop,0)
       end
     end
+
+    context 'when redis server connection is not available' do
+      let(:maxloop) { 2 }
+      it 'should log a failure and raise the error' do
+        expect(redis).to receive(:set).with('vmpooler__tasks__clone', 0).and_raise(Redis::CannotConnectError)
+        expect(logger).to receive(:log).with('s', 'Cannot connect to the redis server: Redis::CannotConnectError')
+
+        expect{subject.execute!(maxloop,0)}.to raise_error Redis::CannotConnectError
+      end
+    end
   end
 
   describe "#sleep_with_wakeup_events" do
@@ -2987,6 +2997,18 @@
       end
     end
 
+    context 'when redis connection fails' do
+      let(:maxloop) { 2 }
+      let(:loop_delay) { 1 }
+
+      it 'should raise the error' do
+        allow(subject).to receive(:_check_pool).and_raise(Redis::CannotConnectError)
+        expect(logger).to receive(:log).with('d', "[*] [#{pool}] starting worker thread")
+
+        expect{subject.check_pool(pool_object,maxloop,loop_delay,loop_delay)}.to raise_error Redis::CannotConnectError
+      end
+    end
+
     context 'delays between loops configured in the pool configuration' do
       let(:maxloop) { 2 }
       let(:loop_delay) { 1 }