Merge pull request openstack-charmers#53 from guoqiao/retry-with-tena…

…city

use tenacity for retries

zaza/openstack/charm_tests/mysql/tests.py

@@ -19,7 +19,7 @@
 import os
 import re
 import tempfile
-import time
+import tenacity
 
 import zaza.charm_lifecycle.utils as lifecycle_utils
 import zaza.model
@@ -477,6 +477,23 @@ def test_100_cold_start_bootstrap(self):
             states=test_config.get("target_deploy_status", {}))
 
 
+@tenacity.retry(
+    retry=tenacity.retry_if_result(lambda is_new: is_new is False),
+    wait=tenacity.wait_fixed(5),  # interval between retries
+    stop=tenacity.stop_after_attempt(10))  # retry times
+def retry_is_new_crm_master(test, old_crm_master):
+    """Check new crm master with retries.
+
+    Return True if a new crm master detected, retry 10 times if False.
+    """
+    new_crm_master = test.get_crm_master()
+    if new_crm_master and new_crm_master != old_crm_master:
+        logging.info(
+            "New crm_master unit detected on {}".format(new_crm_master))
+        return True
+    return False
+
+
 class PerconaClusterScaleTests(PerconaClusterBaseTest):
     """Percona Cluster scale tests."""
 
@@ -496,22 +513,9 @@ def test_100_kill_crm_master(self):
         zaza.model.run_on_unit(old_crm_master, cmd)
 
         logging.info("looking for the new crm_master")
-        i = 0
-        while i < 10:
-            i += 1
-            # XXX time.sleep roundup
-            # https://github.com/openstack-charmers/zaza-openstack-tests/issues/46
-            time.sleep(5)  # give some time to pacemaker to react
-            new_crm_master = self.get_crm_master()
-
-            if (new_crm_master and new_crm_master != old_crm_master):
-                logging.info(
-                    "New crm_master unit detected"
-                    " on {}".format(new_crm_master)
-                )
-                break
-        else:
-            assert False, "The crm_master didn't change"
+        self.assertTrue(
+            retry_is_new_crm_master(self, old_crm_master),
+            msg="The crm_master didn't change")
 
         # Check connectivity on the VIP
         # \ is required due to pep8 and parenthesis would make the assertion

zaza/openstack/charm_tests/vault/tests.py

@@ -17,16 +17,16 @@
 """Collection of tests for vault."""
 
 import contextlib
-import hvac
 import json
 import logging
-import time
 import unittest
 import uuid
 import tempfile
-import tenacity
 
 import requests
+import tenacity
+from hvac.exceptions import InternalServerError
+
 import zaza.charm_lifecycle.utils as lifecycle_utils
 import zaza.openstack.charm_tests.test_utils as test_utils
 import zaza.openstack.charm_tests.vault.utils as vault_utils
@@ -36,6 +36,21 @@
 import zaza.utilities.juju as juju_utils
 
 
+@tenacity.retry(
+    retry=tenacity.retry_if_exception_type(InternalServerError),
+    retry_error_callback=lambda retry_state: False,
+    wait=tenacity.wait_fixed(2),  # interval between retries
+    stop=tenacity.stop_after_attempt(10))  # retry 10 times
+def retry_hvac_client_authenticated(client):
+    """Check hvac client is authenticated with retry.
+
+    If is_authenticated() raise exception for all retries,
+    return False(which is done by `retry_error_callback`).
+    Otherwise, return whatever the returned value.
+    """
+    return client.hvac_client.is_authenticated()
+
+
 class BaseVaultTest(test_utils.OpenStackBaseTest):
     """Base class for vault tests."""
 
@@ -189,17 +204,7 @@ def test_csr(self):
     def test_all_clients_authenticated(self):
         """Check all vault clients are authenticated."""
         for client in self.clients:
-            for i in range(1, 10):
-                try:
-                    self.assertTrue(client.hvac_client.is_authenticated())
-                except hvac.exceptions.InternalServerError:
-                    # XXX time.sleep roundup
-                    # https://github.com/openstack-charmers/zaza-openstack-tests/issues/46
-                    time.sleep(2)
-                else:
-                    break
-            else:
-                self.assertTrue(client.hvac_client.is_authenticated())
+            self.assertTrue(retry_hvac_client_authenticated(client))
 
     def check_read(self, key, value):
         """Check reading the key from all vault units."""

zaza/openstack/charm_tests/vault/utils.py

@@ -20,9 +20,9 @@
 import hvac
 import requests
 import tempfile
-import time
 import urllib3
 import yaml
+import tenacity
 
 import collections
 
@@ -240,30 +240,26 @@ def extract_lead_unit_client(
                        .format(application_name))
 
 
+@tenacity.retry(
+    retry=tenacity.retry_if_exception_type((
+        ConnectionRefusedError,
+        urllib3.exceptions.NewConnectionError,
+        urllib3.exceptions.MaxRetryError,
+        requests.exceptions.ConnectionError)),
+    reraise=True,  # if all retries failed, reraise the last exception
+    wait=tenacity.wait_fixed(2),  # interval between retries
+    stop=tenacity.stop_after_attempt(10))  # retry 10 times
 def is_initialized(client):
     """Check if vault is initialized.
 
     :param client: Client to use to check if vault is initialized
     :type client: CharmVaultClient
     :returns: Whether vault is initialized
     :rtype: bool
+
+    Raise the last exception if no value returned after retries.
     """
-    initialized = False
-    for i in range(1, 10):
-        try:
-            initialized = client.hvac_client.is_initialized()
-        except (ConnectionRefusedError,
-                urllib3.exceptions.NewConnectionError,
-                urllib3.exceptions.MaxRetryError,
-                requests.exceptions.ConnectionError):
-            # XXX time.sleep roundup
-            # https://github.com/openstack-charmers/zaza-openstack-tests/issues/46
-            time.sleep(2)
-        else:
-            break
-    else:
-        raise Exception("Cannot connect")
-    return initialized
+    return client.hvac_client.is_initialized()
 
 
 def ensure_secret_backend(client):