From c63448709ca14cb14789d8dfcde968ddc4f3013e Mon Sep 17 00:00:00 2001 From: sh0ked Date: Thu, 16 Mar 2017 18:18:49 +0700 Subject: [PATCH] Changed work with endpoints and fixes --- config_template.py | 3 +- vmmaster/api/__init__.py | 15 +++--- vmmaster/webdriver/helpers.py | 5 +- vmpool/clone.py | 91 +++++++++++++++-------------------- 4 files changed, 53 insertions(+), 61 deletions(-) diff --git a/config_template.py b/config_template.py index 61dbed50..971473ac 100644 --- a/config_template.py +++ b/config_template.py @@ -46,7 +46,8 @@ class Config(object): OPENSTACK_PING_RETRY_COUNT = 3 OPENSTACK_DEFAULT_FLAVOR = '' OPENASTACK_VM_META_DATA = { - 'admin_pass': 'testPassw0rd.' + 'admin_pass': 'testPassw0rd.', + 'adminPass': 'testPassw0rd.' } VM_CREATE_CHECK_PAUSE = 5 diff --git a/vmmaster/api/__init__.py b/vmmaster/api/__init__.py index 931065c1..a1c68059 100644 --- a/vmmaster/api/__init__.py +++ b/vmmaster/api/__init__.py @@ -169,10 +169,13 @@ def delete_vm_from_pool(endpoint_name): if endpoint: try: - endpoint.delete() + if endpoint.is_preloaded(): + endpoint.delete() + else: + endpoint.delete(try_to_rebuild=False) result = "Endpoint %s was deleted" % endpoint_name except Exception, e: - log.info("Cannot delete vm %s through api method" % endpoint.name) + log.info("Cannot delete vm %s through api method" % endpoint_name) result = "Got error during deleting vm %s. " \ "\n\n %s" % (endpoint_name, e.message) @@ -186,11 +189,11 @@ def delete_all_vm_from_pool(): for endpoint in current_app.pool.pool + current_app.pool.using: try: - endpoint.delete() - results.append(endpoint) + delete_vm_from_pool(endpoint.name) + results.append(endpoint.name) except: log.info("Cannot delete vm %s through api method" % endpoint.name) - failed.append(endpoint) + failed.append(endpoint.name) return render_json(result="This endpoints were deleted from " - "pool: %s" % results, code=200) + "pool: %s" % results, code=200) \ No newline at end of file diff --git a/vmmaster/webdriver/helpers.py b/vmmaster/webdriver/helpers.py index cb77bdb1..88448517 100644 --- a/vmmaster/webdriver/helpers.py +++ b/vmmaster/webdriver/helpers.py @@ -182,8 +182,8 @@ def check_to_exist_ip(session, tries=10, timeout=5): def get_endpoint(session_id, dc): _endpoint = None attempt = 0 - attempts = getattr(config, "GET_ENDPOINT_WAIT_TIME_INCREMENT", - constants.GET_ENDPOINT_WAIT_TIME_INCREMENT) + attempts = getattr(config, "GET_ENDPOINT_ATTEMPTS", + constants.GET_ENDPOINT_ATTEMPTS) wait_time = 0 wait_time_increment = getattr(config, "GET_ENDPOINT_WAIT_TIME_INCREMENT", constants.GET_ENDPOINT_WAIT_TIME_INCREMENT) @@ -203,6 +203,7 @@ def get_endpoint(session_id, dc): % (attempt, session_id, str(e))) if hasattr(_endpoint, "ready"): if not _endpoint.ready: + _endpoint.delete() _endpoint = None if attempt < attempts: time.sleep(wait_time) diff --git a/vmpool/clone.py b/vmpool/clone.py index c13502d1..3f4b4011 100644 --- a/vmpool/clone.py +++ b/vmpool/clone.py @@ -15,7 +15,6 @@ from core import dumpxml from core import utils -from core import constants from core.exceptions import libvirtError, CreationException from core.config import config from core.utils import network_utils @@ -275,24 +274,13 @@ def _wait_for_activated_service(self, method=None): config.VM_CREATE_CHECK_ATTEMPTS, config.VM_CREATE_CHECK_PAUSE config_ping_retry_count, config_ping_timeout = \ config.OPENSTACK_PING_RETRY_COUNT, config.PING_TIMEOUT - rebuild_attempts = getattr(config, "ENDPOINT_REBUILD_ATTEMPTS", - constants.ENDPOINT_REBUILD_ATTEMPTS) create_check_retry = 1 ping_retry = 1 - rebuild_retry = 1 while True: - try: - server = self.nova_client.servers.find(name=self.name) - except Exception as e: - log.exception( - "Can't find vm %s in openstack. Error: %s" % - (self.name, e.message) - ) - server = None - - if server is not None and server.status.lower() in \ + server = self.get_vm(self.name) + if server and server.status.lower() in \ ('build', 'rebuild'): log.info("Virtual Machine %s is spawning..." % self.name) @@ -306,6 +294,10 @@ def _wait_for_activated_service(self, method=None): time.sleep(config_create_check_pause) elif self.vm_has_created(): + if server.status.lower() in 'error': + log.error("VM %s was errored. Rebuilding..." % server.name) + self.rebuild() + break if method is not None: method() if self.ping_vm(): @@ -314,16 +306,14 @@ def _wait_for_activated_service(self, method=None): if ping_retry > config_ping_retry_count: p = config_ping_retry_count * config_ping_timeout log.info("VM %s pings more than %s seconds..." % (self.name, p)) - if rebuild_retry < rebuild_attempts: - raise CreationException("VM %s was created but does not ping" % self.name) - log.warn("Try to rebuild vm %s. Attempt %s" % (self.name, rebuild_retry)) - rebuild_retry += 1 - self.rebuild() + self.delete(try_to_rebuild=True) break ping_retry += 1 else: - raise CreationException("VM %s has not been created." % self.name) + log.error("VM %s has not been created." % self.name) + self.delete(try_to_rebuild=False) + break @property def image(self): @@ -374,15 +364,8 @@ def get_network_id(self): # create new network def vm_has_created(self): - try: - server = self.nova_client.servers.find(name=self.name) - except Exception as e: - log.exception( - "An error occurred during addition ip for vm %s: %s" % - (self.name, e.message)) - server = None - - if server is not None: + server = self.get_vm(self.name) + if server: if server.status.lower() == 'active': if getattr(server, 'addresses', None) is not None: return True @@ -390,12 +373,15 @@ def vm_has_created(self): return False def check_vm_exist(self, server_name): + return True if self.get_vm(server_name) else False + + def get_vm(self, server_name): try: server = self.nova_client.servers.find(name=server_name) - return True if server.name == server_name else False - except Exception as e: - log.exception("VM does not exist. Error: %s" % e.message) - return False + return server if server.name == server_name else None + except: + log.exception("VM %s does not exist" % server_name) + return None def delete(self, try_to_rebuild=True): if try_to_rebuild and self.is_preloaded(): @@ -404,17 +390,15 @@ def delete(self, try_to_rebuild=True): self.ready = False self.pool.remove_vm(self) - if self.check_vm_exist(self.name): + + server = self.get_vm(self.name) + if server: try: - self.nova_client.servers.find(name=self.name).delete() - except Exception as e: - log.exception("Delete vm %s was FAILED. %s" % - (self.name, e.message)) - log.info("Deleted openstack clone: {clone}".format( - clone=self.name)) - else: - log.warn("VM {clone} can not be removed because " - "it does not exist".format(clone=self.name)) + server.delete() + except: + log.exception("Delete vm %s was FAILED." % self.name) + + log.info("Deleted openstack clone: {0}".format(self.name)) VirtualMachine.delete(self) def rebuild(self): @@ -425,12 +409,15 @@ def rebuild(self): self.pool.pool.append(self) self.ready = False - try: - self.nova_client.servers.find(name=self.name).rebuild(self.image) - self._wait_for_activated_service(lambda: log.info( - "Rebuilded openstack clone: {clone}".format(clone=self.name))) - except Exception as e: - log.exception( - "Rebuild vm %s was FAILED. %s" % (self.name, e.message) - ) - self.delete(try_to_rebuild=False) + server = self.get_vm(self.name) + if server: + try: + server.rebuild(self.image) + self._wait_for_activated_service( + lambda: log.info( + "Rebuild vm %s was successful" % self.name + ) + ) + except: + log.exception("Rebuild vm %s was FAILED." % self.name) + self.delete(try_to_rebuild=False)