diff --git a/.travis.yml b/.travis.yml index 71c1d2f778..045d634f78 100644 --- a/.travis.yml +++ b/.travis.yml @@ -23,25 +23,25 @@ matrix: - >- NOSEOPTS="--verbose --with-timer" SETUPOPTS="" PYLINTOPTS="--rcfile=2.7.pylintrc" - PYLINTFILES="azurelinuxagent/daemon azurelinuxagent/agent.py azurelinuxagent/ga setup.py makepkg.py" + PYLINTFILES="azurelinuxagent/common/ azurelinuxagent/daemon azurelinuxagent/agent.py azurelinuxagent/ga setup.py makepkg.py" - python: 3.4 env: - >- NOSEOPTS="--verbose --with-timer" SETUPOPTS="" PYLINTOPTS="--rcfile=2.7.pylintrc" - PYLINTFILES="azurelinuxagent/daemon azurelinuxagent/agent.py azurelinuxagent/ga setup.py makepkg.py" + PYLINTFILES="azurelinuxagent/common/ azurelinuxagent/daemon azurelinuxagent/agent.py azurelinuxagent/ga setup.py makepkg.py" - python: 3.6 env: - >- NOSEOPTS="--verbose --with-timer" SETUPOPTS="" PYLINTOPTS="--rcfile=3.6.pylintrc" - PYLINTFILES="azurelinuxagent/daemon azurelinuxagent/agent.py azurelinuxagent/ga setup.py makepkg.py" + PYLINTFILES="azurelinuxagent/common/ azurelinuxagent/daemon azurelinuxagent/agent.py azurelinuxagent/ga setup.py makepkg.py" - python: 3.7 env: - >- NOSEOPTS="--verbose --with-timer" SETUPOPTS="" PYLINTOPTS="--rcfile=3.6.pylintrc" - PYLINTFILES="azurelinuxagent/daemon azurelinuxagent/agent.py azurelinuxagent/ga setup.py makepkg.py" + PYLINTFILES="azurelinuxagent/common/ azurelinuxagent/daemon azurelinuxagent/agent.py azurelinuxagent/ga setup.py makepkg.py" - python: 3.8 env: - >- @@ -50,7 +50,7 @@ matrix: --cover-package=azurelinuxagent --cover-xml" SETUPOPTS="" PYLINTOPTS="--rcfile=3.6.pylintrc" - PYLINTFILES="azurelinuxagent/daemon azurelinuxagent/agent.py azurelinuxagent/ga setup.py makepkg.py" + PYLINTFILES="azurelinuxagent/common/ azurelinuxagent/daemon azurelinuxagent/agent.py azurelinuxagent/ga setup.py makepkg.py" install: - pip install -r requirements.txt diff --git a/azurelinuxagent/common/AgentGlobals.py b/azurelinuxagent/common/AgentGlobals.py index 287dc48642..fb3e8021c8 100644 --- a/azurelinuxagent/common/AgentGlobals.py +++ b/azurelinuxagent/common/AgentGlobals.py @@ -1,4 +1,4 @@ -# Microsoft Azure Linux Agent +# Microsoft Azure Linux Agent # pylint: disable=C0103 # # Copyright 2020 Microsoft Corporation # diff --git a/azurelinuxagent/common/cgroup.py b/azurelinuxagent/common/cgroup.py index 912423c0c2..b5477f1795 100644 --- a/azurelinuxagent/common/cgroup.py +++ b/azurelinuxagent/common/cgroup.py @@ -29,21 +29,21 @@ MetricValue = namedtuple('Metric', ['category', 'counter', 'instance', 'value']) -class MetricsCategory(object): +class MetricsCategory(object): # pylint: disable=R0903 MEMORY_CATEGORY = "Memory" CPU_CATEGORY = "CPU" -class MetricsCounter(object): +class MetricsCounter(object): # pylint: disable=R0903 PROCESSOR_PERCENT_TIME = "% Processor Time" TOTAL_MEM_USAGE = "Total Memory Usage" MAX_MEM_USAGE = "Max Memory Usage" -re_user_system_times = re.compile(r'user (\d+)\nsystem (\d+)\n') +re_user_system_times = re.compile(r'user (\d+)\nsystem (\d+)\n') # pylint: disable=invalid-name -class CGroupContollers(object): +class CGroupContollers(object): # pylint: disable=R0903 CPU = "cpu" MEMORY = "memory" @@ -105,8 +105,8 @@ def _get_parameters(self, parameter_name, first_line_only=False): parameter_filename = self._get_cgroup_file(parameter_name) logger.error("File {0} is empty but should not be".format(parameter_filename)) raise CGroupsException("File {0} is empty but should not be".format(parameter_filename)) - except Exception as e: - if isinstance(e, (IOError, OSError)) and e.errno == errno.ENOENT: + except Exception as e: # pylint: disable=C0103 + if isinstance(e, (IOError, OSError)) and e.errno == errno.ENOENT: # pylint: disable=E1101 raise e parameter_filename = self._get_cgroup_file(parameter_name) raise CGroupsException("Exception while attempting to read {0}".format(parameter_filename), e) @@ -116,8 +116,8 @@ def is_active(self): try: tasks = self._get_parameters("tasks") if tasks: - return len(tasks) != 0 - except (IOError, OSError) as e: + return len(tasks) != 0 # pylint: disable=len-as-condition + except (IOError, OSError) as e: # pylint: disable=C0103 if e.errno == errno.ENOENT: # only suppressing file not found exceptions. pass @@ -125,7 +125,7 @@ def is_active(self): logger.periodic_warn(logger.EVERY_HALF_HOUR, 'Could not get list of tasks from "tasks" file in the cgroup: {0}.' ' Internal error: {1}'.format(self.path, ustr(e))) - except CGroupsException as e: + except CGroupsException as e: # pylint: disable=C0103 logger.periodic_warn(logger.EVERY_HALF_HOUR, 'Could not get list of tasks from "tasks" file in the cgroup: {0}.' ' Internal error: {1}'.format(self.path, ustr(e))) @@ -138,7 +138,7 @@ def get_tracked_processes(self): procs = [] try: procs = self._get_parameters("cgroup.procs") - except (IOError, OSError) as e: + except (IOError, OSError) as e: # pylint: disable=C0103 if e.errno == errno.ENOENT: # only suppressing file not found exceptions. pass @@ -146,7 +146,7 @@ def get_tracked_processes(self): logger.periodic_warn(logger.EVERY_HALF_HOUR, 'Could not get list of procs from "cgroup.procs" file in the cgroup: {0}.' ' Internal error: {1}'.format(self.path, ustr(e))) - except CGroupsException as e: + except CGroupsException as e: # pylint: disable=C0103 logger.periodic_warn(logger.EVERY_HALF_HOUR, 'Could not get list of tasks from "cgroup.procs" file in the cgroup: {0}.' ' Internal error: {1}'.format(self.path, ustr(e))) @@ -183,8 +183,8 @@ def _get_cpu_ticks(self, allow_no_such_file_or_directory_error=False): """ try: cpu_stat = self._get_file_contents('cpuacct.stat') - except Exception as e: - if not isinstance(e, (IOError, OSError)) or e.errno != errno.ENOENT: + except Exception as e: # pylint: disable=C0103 + if not isinstance(e, (IOError, OSError)) or e.errno != errno.ENOENT: # pylint: disable=E1101 raise CGroupsException("Failed to read cpuacct.stat: {0}".format(ustr(e))) if not allow_no_such_file_or_directory_error: raise e @@ -264,8 +264,8 @@ def get_memory_usage(self): usage = None try: usage = self._get_parameters('memory.usage_in_bytes', first_line_only=True) - except Exception as e: - if isinstance(e, (IOError, OSError)) and e.errno == errno.ENOENT: + except Exception as e: # pylint: disable=C0103 + if isinstance(e, (IOError, OSError)) and e.errno == errno.ENOENT: # pylint: disable=E1101 raise raise CGroupsException("Exception while attempting to read {0}".format("memory.usage_in_bytes"), e) @@ -281,8 +281,8 @@ def get_max_memory_usage(self): usage = None try: usage = self._get_parameters('memory.max_usage_in_bytes', first_line_only=True) - except Exception as e: - if isinstance(e, (IOError, OSError)) and e.errno == errno.ENOENT: + except Exception as e: # pylint: disable=C0103 + if isinstance(e, (IOError, OSError)) and e.errno == errno.ENOENT: # pylint: disable=E1101 raise raise CGroupsException("Exception while attempting to read {0}".format("memory.usage_in_bytes"), e) diff --git a/azurelinuxagent/common/cgroupapi.py b/azurelinuxagent/common/cgroupapi.py index 576e3af126..4a0f88fd2f 100644 --- a/azurelinuxagent/common/cgroupapi.py +++ b/azurelinuxagent/common/cgroupapi.py @@ -59,7 +59,7 @@ def remove_extension_cgroups(self, extension_name): def get_extension_cgroups(self, extension_name): raise NotImplementedError() - def start_extension_command(self, extension_name, command, timeout, shell, cwd, env, stdout, stderr, error_code): + def start_extension_command(self, extension_name, command, timeout, shell, cwd, env, stdout, stderr, error_code): # pylint: disable=R0913 raise NotImplementedError() def cleanup_legacy_cgroups(self): @@ -80,7 +80,7 @@ def track_cgroups(extension_cgroups): try: for cgroup in extension_cgroups: CGroupsTelemetry.track_cgroup(cgroup) - except Exception as e: + except Exception as e: # pylint: disable=C0103 logger.warn("Cannot add cgroup '{0}' to tracking list; resource usage will not be tracked. " "Error: {1}".format(cgroup.path, ustr(e))) @@ -119,7 +119,7 @@ def _foreach_controller(operation, message): logger.warn('Cgroup controller "{0}" is not mounted. {1}', controller, message) else: operation(controller) - except Exception as e: + except Exception as e: # pylint: disable=C0103 logger.warn('Error in cgroup controller "{0}": {1}. {2}', controller, ustr(e), message) @staticmethod @@ -175,9 +175,9 @@ def _try_mkdir(path): if not os.path.isdir(path): try: os.makedirs(path, 0o755) - except OSError as e: + except OSError as e: # pylint: disable=C0103 if e.errno == errno.EEXIST: - if not os.path.isdir(path): + if not os.path.isdir(path): # pylint: disable=R1720 raise CGroupsException("Create directory for cgroup {0}: normal file already exists with that name".format(path)) else: pass # There was a race to create the directory, but it's there now, and that's fine @@ -225,9 +225,9 @@ def cgroup_path(tail=""): if not os.path.exists(path): fileutil.mkdir(path) osutil.mount(device='cgroup_root', - mount_point=path, - option="-t tmpfs", - chk_err=False) + mount_point=path, # pylint: disable=C0330 + option="-t tmpfs", # pylint: disable=C0330 + chk_err=False) # pylint: disable=C0330 elif not os.path.isdir(cgroup_path()): logger.error("Could not mount cgroups: ordinary file at {0}", path) return @@ -241,9 +241,9 @@ def cgroup_path(tail=""): if not os.path.exists(target_path): fileutil.mkdir(target_path) osutil.mount(device=controller, - mount_point=target_path, - option="-t cgroup -o {0}".format(controller), - chk_err=False) + mount_point=target_path, # pylint: disable=C0330 + option="-t cgroup -o {0}".format(controller), # pylint: disable=C0330 + chk_err=False) # pylint: disable=C0330 if controller == 'cpu,cpuacct': cpu_mounted = True except Exception as exception: @@ -258,11 +258,11 @@ def cgroup_path(tail=""): if not os.path.exists(target_path): os.symlink(cgroup_path('cpu,cpuacct'), target_path) - except OSError as oe: + except OSError as oe: # pylint: disable=C0103 # log a warning for read-only file systems logger.warn("Could not mount cgroups: {0}", ustr(oe)) raise - except Exception as e: + except Exception as e: # pylint: disable=C0103 logger.error("Could not mount cgroups: {0}", ustr(e)) raise @@ -302,7 +302,7 @@ def create_cgroup(controller): self._foreach_controller(create_cgroup, 'Failed to create a cgroup for the VM Agent; resource usage will not be tracked') - if len(cgroups) == 0: + if len(cgroups) == 0: # pylint: disable=len-as-condition raise CGroupsException("Failed to create any cgroup for the VM Agent") return cgroups @@ -371,7 +371,7 @@ def get_cgroup(controller): return cgroups - def start_extension_command(self, extension_name, command, timeout, shell, cwd, env, stdout, stderr, + def start_extension_command(self, extension_name, command, timeout, shell, cwd, env, stdout, stderr, # pylint: disable=R0913 error_code=ExtensionErrorCodes.PluginUnknownFailure): """ Starts a command (install/enable/etc) for an extension and adds the command's PID to the extension's cgroup @@ -384,6 +384,7 @@ def start_extension_command(self, extension_name, command, timeout, shell, cwd, :param stderr: File object to redirect stderr to :param error_code: Extension error code to raise in case of error """ + # pylint: disable=too-many-locals try: extension_cgroups = self.create_extension_cgroups(extension_name) except Exception as exception: @@ -405,11 +406,11 @@ def pre_exec_function(): "Resource usage will not be tracked. Error: {2}".format(pid, extension_name, ustr(exception))) - except Exception as e: + except Exception as e: # pylint: disable=C0103 logger.warn("Failed to add extension {0} to its cgroup. Resource usage will not be tracked. " "Error: {1}".format(extension_name, ustr(e))) - process = subprocess.Popen(command, + process = subprocess.Popen(command, # pylint: disable=W1509 shell=shell, cwd=cwd, env=env, @@ -542,7 +543,7 @@ def get_unit_property(unit_name, property_name): output = shellutil.run_command(["systemctl", "show", unit_name, "--property", property_name]) match = re.match("[^=]+=(?P.+)", output) if match is None: - raise ValueError("Can't find property {0} of {1}", property_name, unit_name) + raise ValueError("Can't find property {0} of {1}", property_name, unit_name) # pylint: disable=W0715 return match.group('value') @staticmethod @@ -552,7 +553,7 @@ def create_and_start_unit(unit_filename, unit_contents): fileutil.write_file(unit_path, unit_contents) shellutil.run_command(["systemctl", "daemon-reload"]) shellutil.run_command(["systemctl", "start", unit_filename]) - except Exception as e: + except Exception as e: # pylint: disable=C0103 raise CGroupsException("Failed to create and start {0}. Error: {1}".format(unit_filename, ustr(e))) @staticmethod @@ -575,7 +576,7 @@ def create_extension_cgroups_root(self): logger.info("Created slice for walinuxagent extensions {0}".format(unit_filename)) def create_extension_cgroups(self, extension_name): - # TODO: The slice created by this function is not used currently. We need to create the extension scopes within + # TODO: The slice created by this function is not used currently. We need to create the extension scopes within # pylint: disable=W0511 # this slice and use the slice to monitor the cgroups. Also see comment in get_extension_cgroups. # the slice. unit_contents = """ @@ -605,11 +606,11 @@ def remove_extension_cgroups(self, extension_name): shellutil.run_command(["systemctl", "stop", unit_filename]) fileutil.rm_files(unit_path) shellutil.run_command(["systemctl", "daemon-reload"]) - except Exception as e: + except Exception as e: # pylint: disable=C0103 raise CGroupsException("Failed to remove {0}. Error: {1}".format(unit_filename, ustr(e))) def get_extension_cgroups(self, extension_name): - # TODO: The slice returned by this function is not used currently. We need to create the extension scopes within + # TODO: The slice returned by this function is not used currently. We need to create the extension scopes within # pylint: disable=W0511 # this slice and use the slice to monitor the cgroups. Also see comment in create_extension_cgroups. slice_name = self._get_extension_cgroup_name(extension_name) @@ -646,7 +647,7 @@ def get_processes_in_cgroup(cgroup_path): processes = [] for line in output.splitlines(): - match = re.match('[^\d]*(?P\d+)\s+(?P.+)', line) + match = re.match('[^\d]*(?P\d+)\s+(?P.+)', line) # pylint: disable=W1401 if match is not None: processes.append((match.group('pid'), match.group('command'))) @@ -659,11 +660,11 @@ def _is_systemd_failure(scope_name, stderr): unit_not_found = "Unit {0} not found.".format(scope_name) return unit_not_found in stderr or scope_name not in stderr - def start_extension_command(self, extension_name, command, timeout, shell, cwd, env, stdout, stderr, + def start_extension_command(self, extension_name, command, timeout, shell, cwd, env, stdout, stderr, # pylint: disable=R0913,R0914 error_code=ExtensionErrorCodes.PluginUnknownFailure): scope = "{0}_{1}".format(self._get_extension_cgroup_name(extension_name), uuid.uuid4()) - process = subprocess.Popen( + process = subprocess.Popen( # pylint: disable=W1509 "systemd-run --unit={0} --scope {1}".format(scope, command), shell=shell, cwd=cwd, @@ -694,11 +695,11 @@ def start_extension_command(self, extension_name, command, timeout, shell, cwd, memory_cgroup_path = os.path.join(memory_cgroup_mountpoint, cgroup_relative_path) CGroupsTelemetry.track_cgroup(MemoryCgroup(extension_name, memory_cgroup_path)) - except IOError as e: + except IOError as e: # pylint: disable=C0103 if e.errno == 2: # 'No such file or directory' logger.info("The extension command already completed; will not track resource usage") logger.info("Failed to start tracking resource usage for the extension: {0}", ustr(e)) - except Exception as e: + except Exception as e: # pylint: disable=C0103 logger.info("Failed to start tracking resource usage for the extension: {0}", ustr(e)) # Wait for process completion or timeout @@ -709,19 +710,22 @@ def start_extension_command(self, extension_name, command, timeout, shell, cwd, stdout=stdout, stderr=stderr, error_code=error_code) - except ExtensionError as e: + except ExtensionError as e: # pylint: disable=C0103 # The extension didn't terminate successfully. Determine whether it was due to systemd errors or # extension errors. systemd_failure = self._is_systemd_failure(scope, stderr) process_output = read_output(stdout, stderr) - if not systemd_failure: + if not systemd_failure: # pylint: disable=R1720 # There was an extension error; it either timed out or returned a non-zero exit code. Re-raise the error raise else: # There was an issue with systemd-run. We need to log it and retry the extension without systemd. - err_msg = 'Systemd process exited with code %s and output %s' % (e.exit_code, process_output) \ - if isinstance(e, ExtensionOperationError) else "Systemd timed-out, output: %s" % process_output + if isinstance(e, ExtensionOperationError): + err_msg = 'Systemd process exited with code %s and output %s' % (e.exit_code, process_output) # pylint: disable=no-member + else: + err_msg = "Systemd timed-out, output: %s" % process_output + event_msg = 'Failed to run systemd-run for unit {0}.scope. ' \ 'Will retry invoking the extension without systemd. ' \ 'Systemd-run error: {1}'.format(scope, err_msg) @@ -735,7 +739,7 @@ def start_extension_command(self, extension_name, command, timeout, shell, cwd, # Try invoking the process again, this time without systemd-run logger.info('Extension invocation using systemd failed, falling back to regular invocation ' 'without cgroups tracking.') - process = subprocess.Popen(command, + process = subprocess.Popen(command, # pylint: disable=W1509 shell=shell, cwd=cwd, env=env, diff --git a/azurelinuxagent/common/cgroupconfigurator.py b/azurelinuxagent/common/cgroupconfigurator.py index ac45d3ce12..b4c9d2b85c 100644 --- a/azurelinuxagent/common/cgroupconfigurator.py +++ b/azurelinuxagent/common/cgroupconfigurator.py @@ -35,7 +35,7 @@ class CGroupConfigurator(object): NOTE: with the exception of start_extension_command, none of the methods in this class raise exceptions (cgroup operations should not block extensions) """ - class __impl(object): + class __impl(object): # pylint: disable=R0902,C0103 def __init__(self): self._initialized = False self._cgroups_supported = False @@ -46,7 +46,8 @@ def __init__(self): self._get_processes_in_agent_cgroup_last_error = None self._get_processes_in_agent_cgroup_error_count = 0 - def initialize(self): + def initialize(self): # pylint: disable=R0912 + # pylint: disable=too-many-locals try: if self._initialized: return @@ -80,7 +81,7 @@ def log_cgroup_warn(format_string, *args): logger.warn(message) add_event(op=WALAEventOperation.CGroupsInfo, message=message, is_success=False, log_event=False) - log_cgroup_info("systemd version: {0}", self._cgroups_api.get_systemd_version()) + log_cgroup_info("systemd version: {0}", self._cgroups_api.get_systemd_version()) # pylint: disable=E1101 # # Older versions of the daemon (2.2.31-2.2.40) wrote their PID to /sys/fs/cgroup/{cpu,memory}/WALinuxAgent/WALinuxAgent. When running @@ -95,7 +96,7 @@ def log_cgroup_warn(format_string, *args): # # check v1 controllers # - cpu_controller_root, memory_controller_root = self._cgroups_api.get_cgroup_mount_points() + cpu_controller_root, memory_controller_root = self._cgroups_api.get_cgroup_mount_points() # pylint: disable=E1101 if cpu_controller_root is not None: logger.info("The CPU cgroup controller is mounted at {0}", cpu_controller_root) @@ -110,25 +111,25 @@ def log_cgroup_warn(format_string, *args): # # check v2 controllers # - cgroup2_mountpoint, cgroup2_controllers = self._cgroups_api.get_cgroup2_controllers() + cgroup2_mountpoint, cgroup2_controllers = self._cgroups_api.get_cgroup2_controllers() # pylint: disable=E1101 if cgroup2_mountpoint is not None: log_cgroup_warn("cgroups v2 mounted at {0}. Controllers: [{1}]", cgroup2_mountpoint, cgroup2_controllers) # # check the cgroups for the agent # - agent_unit_name = self._cgroups_api.get_agent_unit_name() - cpu_cgroup_relative_path, memory_cgroup_relative_path = self._cgroups_api.get_process_cgroup_relative_paths("self") + agent_unit_name = self._cgroups_api.get_agent_unit_name() # pylint: disable=E1101 + cpu_cgroup_relative_path, memory_cgroup_relative_path = self._cgroups_api.get_process_cgroup_relative_paths("self") # pylint: disable=E1101 if cpu_cgroup_relative_path is None: log_cgroup_warn("The agent's process is not within a CPU cgroup") else: - cpu_accounting = self._cgroups_api.get_unit_property(agent_unit_name, "CPUAccounting") + cpu_accounting = self._cgroups_api.get_unit_property(agent_unit_name, "CPUAccounting") # pylint: disable=E1101 log_cgroup_info('CPUAccounting: {0}', cpu_accounting) if memory_cgroup_relative_path is None: log_cgroup_warn("The agent's process is not within a memory cgroup") else: - memory_accounting = self._cgroups_api.get_unit_property(agent_unit_name, "MemoryAccounting") + memory_accounting = self._cgroups_api.get_unit_property(agent_unit_name, "MemoryAccounting") # pylint: disable=E1101 log_cgroup_info('MemoryAccounting: {0}', memory_accounting) # @@ -150,7 +151,7 @@ def log_cgroup_warn(format_string, *args): log_cgroup_info("Agent cgroups: CPU: {0} -- MEMORY: {1}", self._agent_cpu_cgroup_path, self._agent_memory_cgroup_path) - except Exception as e: + except Exception as e: # pylint: disable=C0103 message = "Error initializing cgroups: {0}".format(ustr(e)) logger.warn(message) add_event(op=WALAEventOperation.CGroupsInitialize, is_success=False, message=message, log_event=False) @@ -182,12 +183,12 @@ def _invoke_cgroup_operation(self, operation, error_message, on_error=None): try: return operation() - except Exception as e: + except Exception as e: # pylint: disable=C0103 logger.warn("{0} Error: {1}".format(error_message, ustr(e))) if on_error is not None: try: on_error(e) - except Exception as ex: + except Exception as ex: # pylint: disable=W0612 logger.warn("CGroupConfigurator._invoke_cgroup_operation: {0}".format(ustr(e))) def create_extension_cgroups_root(self): @@ -213,7 +214,7 @@ def remove_extension_cgroups(self, name): Deletes the cgroup for the given extension """ def __impl(): - cgroups = self._cgroups_api.remove_extension_cgroups(name) + cgroups = self._cgroups_api.remove_extension_cgroups(name) # pylint: disable=E1111 return cgroups self._invoke_cgroup_operation(__impl, "Failed to delete cgroups for extension '{0}'.".format(name)) @@ -227,7 +228,7 @@ def get_processes_in_agent_cgroup(self): def __impl(): if self._agent_cpu_cgroup_path is None: return [] - return self._cgroups_api.get_processes_in_cgroup(self._agent_cpu_cgroup_path) + return self._cgroups_api.get_processes_in_cgroup(self._agent_cpu_cgroup_path) # pylint: disable=E1101 def __on_error(exception): # @@ -242,7 +243,7 @@ def __on_error(exception): return self._invoke_cgroup_operation(__impl, "Failed to list the processes in the agent's cgroup.", on_error=__on_error) - def start_extension_command(self, extension_name, command, timeout, shell, cwd, env, stdout, stderr, + def start_extension_command(self, extension_name, command, timeout, shell, cwd, env, stdout, stderr, # pylint: disable=R0913 error_code=ExtensionErrorCodes.PluginUnknownFailure): """ Starts a command (install/enable/etc) for an extension and adds the command's PID to the extension's cgroup @@ -257,7 +258,7 @@ def start_extension_command(self, extension_name, command, timeout, shell, cwd, :param error_code: Extension error code to raise in case of error """ if not self.enabled(): - process = subprocess.Popen(command, + process = subprocess.Popen(command, # pylint: disable=W1509 shell=shell, cwd=cwd, env=env, @@ -273,14 +274,14 @@ def start_extension_command(self, extension_name, command, timeout, shell, cwd, error_code=error_code) else: process_output = self._cgroups_api.start_extension_command(extension_name, - command, - timeout, - shell=shell, - cwd=cwd, - env=env, - stdout=stdout, - stderr=stderr, - error_code=error_code) + command, # pylint: disable=C0330 + timeout, # pylint: disable=C0330 + shell=shell, # pylint: disable=C0330 + cwd=cwd, # pylint: disable=C0330 + env=env, # pylint: disable=C0330 + stdout=stdout, # pylint: disable=C0330 + stderr=stderr, # pylint: disable=C0330 + error_code=error_code) # pylint: disable=C0330 return process_output @@ -326,7 +327,7 @@ def is_agent_process(command_line): # r"^ifdown .+ && ifup .+", ] - for p in patterns: + for p in patterns: # pylint: disable=C0103 if re.match(p, command_line) is not None: return True return False diff --git a/azurelinuxagent/common/cgroupstelemetry.py b/azurelinuxagent/common/cgroupstelemetry.py index eab38b78c5..c6c79085e1 100644 --- a/azurelinuxagent/common/cgroupstelemetry.py +++ b/azurelinuxagent/common/cgroupstelemetry.py @@ -43,12 +43,12 @@ def get_process_info_summary(process_id): # details from those files. try: process_cmdline = ProcessInfo.get_proc_cmdline(process_id) if not None else DEFAULT_PROCESS_COMMANDLINE - except Exception as e: + except Exception as e: # pylint: disable=C0103 logger.periodic_info(EVERY_SIX_HOURS, "[PERIODIC] {0}", ustr(e)) try: process_name = ProcessInfo.get_proc_name(process_id) if not None else DEFAULT_PROCESS_NAME - except Exception as e: + except Exception as e: # pylint: disable=C0103 logger.periodic_info(EVERY_SIX_HOURS, "[PERIODIC] {0}", ustr(e)) return process_id + DELIM + process_name + DELIM + process_cmdline @@ -97,12 +97,12 @@ def poll_all_tracked(): for cgroup in CGroupsTelemetry._tracked[:]: try: metrics.extend(cgroup.get_tracked_metrics()) - except Exception as e: + except Exception as e: # pylint: disable=C0103 # There can be scenarios when the CGroup has been deleted by the time we are fetching the values # from it. This would raise IOError with file entry not found (ERRNO: 2). We do not want to log # every occurrences of such case as it would be very verbose. We do want to log all the other # exceptions which could occur, which is why we do a periodic log for all the other errors. - if not isinstance(e, (IOError, OSError)) or e.errno != errno.ENOENT: + if not isinstance(e, (IOError, OSError)) or e.errno != errno.ENOENT: # pylint: disable=E1101 logger.periodic_warn(logger.EVERY_HOUR, '[PERIODIC] Could not collect metrics for cgroup ' '{0}. Error : {1}'.format(cgroup.name, ustr(e))) if not cgroup.is_active(): diff --git a/azurelinuxagent/common/conf.py b/azurelinuxagent/common/conf.py index bb2e92e52c..931ec73e6b 100644 --- a/azurelinuxagent/common/conf.py +++ b/azurelinuxagent/common/conf.py @@ -19,11 +19,11 @@ """ Module conf loads and parses configuration file -""" +""" # pylint: disable=W0105 import os import os.path -import azurelinuxagent.common.utils.fileutil as fileutil +from azurelinuxagent.common.utils.fileutil import read_file #pylint: disable=R0401 from azurelinuxagent.common.exception import AgentConfigError DISABLE_AGENT_FILE = 'disable_agent' @@ -55,7 +55,7 @@ def get(self, key, default_val): def get_switch(self, key, default_val): val = self.values.get(key) - if val is not None and val.lower() == 'y': + if val is not None and val.lower() == 'y': # pylint: disable=R1705 return True elif val is not None and val.lower() == 'n': return False @@ -77,11 +77,11 @@ def load_conf_from_file(conf_file_path, conf=__conf__): """ Load conf file from: conf_file_path """ - if os.path.isfile(conf_file_path) == False: + if os.path.isfile(conf_file_path) == False: # pylint: disable=C0121 raise AgentConfigError(("Missing configuration in {0}" "").format(conf_file_path)) try: - content = fileutil.read_file(conf_file_path) + content = read_file(conf_file_path) conf.load(content) except IOError as err: raise AgentConfigError(("Failed to load conf file:{0}, {1}" @@ -285,12 +285,12 @@ def get_ssh_key_glob(conf=__conf__): def get_ssh_key_private_path(conf=__conf__): return os.path.join(get_ssh_dir(conf), - 'ssh_host_{0}_key'.format(get_ssh_host_keypair_type(conf))) + 'ssh_host_{0}_key'.format(get_ssh_host_keypair_type(conf))) # pylint: disable=C0330 def get_ssh_key_public_path(conf=__conf__): return os.path.join(get_ssh_dir(conf), - 'ssh_host_{0}_key.pub'.format(get_ssh_host_keypair_type(conf))) + 'ssh_host_{0}_key.pub'.format(get_ssh_host_keypair_type(conf))) # pylint: disable=C0330 def get_root_device_scsi_timeout(conf=__conf__): @@ -445,4 +445,4 @@ def get_cgroups_enforce_limits(conf=__conf__): def get_cgroups_excluded(conf=__conf__): excluded_value = conf.get("CGroups.Excluded", "customscript, runcommand") - return [s for s in [i.strip().lower() for i in excluded_value.split(',')] if len(s) > 0] if excluded_value else [] + return [s for s in [i.strip().lower() for i in excluded_value.split(',')] if len(s) > 0] if excluded_value else [] # pylint: disable=len-as-condition diff --git a/azurelinuxagent/common/datacontract.py b/azurelinuxagent/common/datacontract.py index c69bebc956..c8e0e85b5e 100644 --- a/azurelinuxagent/common/datacontract.py +++ b/azurelinuxagent/common/datacontract.py @@ -20,17 +20,19 @@ from azurelinuxagent.common.exception import ProtocolError import azurelinuxagent.common.logger as logger +# pylint: disable=W0105 """ Base class for data contracts between guest and host and utilities to manipulate the properties in those contracts -""" +""" +# pylint: enable=W0105 -class DataContract(object): +class DataContract(object): # pylint: disable=R0903 pass class DataContractList(list): - def __init__(self, item_cls): + def __init__(self, item_cls): # pylint: disable=W0231 self.item_cls = item_cls @@ -43,7 +45,7 @@ def validate_param(name, val, expected_type): def set_properties(name, obj, data): - if isinstance(obj, DataContract): + if isinstance(obj, DataContract): # pylint: disable=R1705 validate_param("Property '{0}'".format(name), data, dict) for prob_name, prob_val in data.items(): prob_full_name = "{0}.{1}".format(name, prob_name) @@ -67,7 +69,7 @@ def set_properties(name, obj, data): def get_properties(obj): - if isinstance(obj, DataContract): + if isinstance(obj, DataContract): # pylint: disable=R1705 data = {} props = vars(obj) for prob_name, prob in list(props.items()): diff --git a/azurelinuxagent/common/dhcp.py b/azurelinuxagent/common/dhcp.py index 5974965608..2d7f6d8da5 100644 --- a/azurelinuxagent/common/dhcp.py +++ b/azurelinuxagent/common/dhcp.py @@ -30,7 +30,7 @@ # the kernel routing table representation of 168.63.129.16 KNOWN_WIRESERVER_IP_ENTRY = '10813FA8' -from azurelinuxagent.common.utils.restutil import KNOWN_WIRESERVER_IP +from azurelinuxagent.common.utils.restutil import KNOWN_WIRESERVER_IP # pylint: disable=C0413 def get_dhcp_handler(): @@ -67,7 +67,7 @@ def wait_for_network(self): Wait for network stack to be initialized. """ ipv4 = self.osutil.get_ip4_addr() - while ipv4 == '' or ipv4 == '0.0.0.0': + while ipv4 == '' or ipv4 == '0.0.0.0': # pylint: disable=R1714 logger.info("Waiting for network.") time.sleep(10) logger.info("Try to start network interface.") @@ -96,7 +96,7 @@ def wireserver_route_exists(self): logger.info("Route to {0} exists".format(KNOWN_WIRESERVER_IP)) else: logger.warn("No route exists to {0}".format(KNOWN_WIRESERVER_IP)) - except Exception as e: + except Exception as e: # pylint: disable=C0103 logger.error( "Could not determine whether route exists to {0}: {1}".format( KNOWN_WIRESERVER_IP, e)) @@ -116,7 +116,7 @@ def dhcp_cache_exists(self): exists = False logger.info("Checking for dhcp lease cache") - cached_endpoint = self.osutil.get_dhcp_lease_endpoint() + cached_endpoint = self.osutil.get_dhcp_lease_endpoint() # pylint: disable=E1128 if cached_endpoint is not None: self.endpoint = cached_endpoint exists = True @@ -142,7 +142,7 @@ def _send_dhcp_req(self, request): response = socket_send(request) validate_dhcp_resp(request, response) return response - except DhcpError as e: + except DhcpError as e: # pylint: disable=C0103 logger.warn("Failed to send DHCP request: {0}", e) time.sleep(duration) return None @@ -157,11 +157,14 @@ def send_dhcp_req(self): self.endpoint = KNOWN_WIRESERVER_IP return + # pylint: disable=W0105 """ Build dhcp request with mac addr Configure route to allow dhcp traffic Stop dhcp service if necessary - """ + """ + # pylint: enable=W0105 + logger.info("Send dhcp request") mac_addr = self.osutil.get_mac_addr() @@ -194,7 +197,7 @@ def send_dhcp_req(self): self.endpoint, self.gateway, self.routes = parse_dhcp_resp(resp) -def validate_dhcp_resp(request, response): +def validate_dhcp_resp(request, response): # pylint: disable=R1710 bytes_recv = len(response) if bytes_recv < 0xF6: logger.error("HandleDhcpResponse: Too few bytes received:{0}", @@ -228,7 +231,7 @@ def validate_dhcp_resp(request, response): "doesn't match the request") -def parse_route(response, option, i, length, bytes_recv): +def parse_route(response, option, i, length, bytes_recv): # pylint: disable=W0613 # http://msdn.microsoft.com/en-us/library/cc227282%28PROT.10%29.aspx logger.verbose("Routes at offset: {0} with length:{1}", hex(i), hex(length)) @@ -254,7 +257,7 @@ def parse_route(response, option, i, length, bytes_recv): def parse_ip_addr(response, option, i, length, bytes_recv): - if i + 5 < bytes_recv: + if i + 5 < bytes_recv: # pylint: disable=R1705 if length != 4: logger.error("Endpoint or Default Gateway not 4 bytes") return None @@ -290,7 +293,7 @@ def parse_dhcp_resp(response): length = str_to_ord(response[i + 1]) logger.verbose("DHCP option {0} at offset:{1} with length:{2}", hex(option), hex(i), hex(length)) - if option == 255: + if option == 255: # pylint: disable=R1723 logger.verbose("DHCP packet ended at offset:{0}", hex(i)) break elif option == 249: @@ -324,7 +327,7 @@ def socket_send(request): "entering recv") response = sock.recv(1024) return response - except IOError as e: + except IOError as e: # pylint: disable=C0103 raise DhcpError("{0}".format(e)) finally: if sock is not None: @@ -371,11 +374,11 @@ def build_dhcp_request(mac_addr, request_broadcast): # Opcode = 1 # HardwareAddressType = 1 (ethernet/MAC) # HardwareAddressLength = 6 (ethernet/MAC/48 bits) - for a in range(0, 3): + for a in range(0, 3): # pylint: disable=C0103 request[a] = [1, 1, 6][a] # fill in transaction id (random number to ensure response matches request) - for a in range(0, 4): + for a in range(0, 4): # pylint: disable=C0103 request[4 + a] = str_to_ord(trans_id[a]) logger.verbose("BuildDhcpRequest: transactionId:%s,%04X" % ( @@ -386,10 +389,10 @@ def build_dhcp_request(mac_addr, request_broadcast): # set broadcast flag to true to request the dhcp server # to respond to a boradcast address, # this is useful when user dhclient fails. - request[0x0A] = 0x80; + request[0x0A] = 0x80; # pylint: disable=W0301 # fill in ClientHardwareAddress - for a in range(0, 6): + for a in range(0, 6): # pylint: disable=C0103 request[0x1C + a] = str_to_ord(mac_addr[a]) # DHCP Magic Cookie: 99, 130, 83, 99 @@ -397,7 +400,7 @@ def build_dhcp_request(mac_addr, request_broadcast): # MessageTypeLength = 1 # MessageType = DHCPDISCOVER # End = 255 DHCP_END - for a in range(0, 8): + for a in range(0, 8): # pylint: disable=C0103 request[0xEC + a] = [99, 130, 83, 99, 53, 1, 1, 255][a] return array.array("B", request) diff --git a/azurelinuxagent/common/event.py b/azurelinuxagent/common/event.py index 122364cfba..e55fe6f7f4 100644 --- a/azurelinuxagent/common/event.py +++ b/azurelinuxagent/common/event.py @@ -66,7 +66,7 @@ def send_logs_to_telemetry(): return SEND_LOGS_TO_TELEMETRY -class WALAEventOperation: +class WALAEventOperation: # pylint: disable=R0903,no-init ActivateResourceDisk = "ActivateResourceDisk" AgentBlacklisted = "AgentBlacklisted" AgentEnabled = "AgentEnabled" @@ -144,10 +144,10 @@ def clear(self): self._status = {} self._save() - def event_marked(self, name, version, op): + def event_marked(self, name, version, op): # pylint: disable=C0103 return self._event_name(name, version, op) in self._status - def event_succeeded(self, name, version, op): + def event_succeeded(self, name, version, op): # pylint: disable=C0103 event = self._event_name(name, version, op) if event not in self._status: return True @@ -157,35 +157,35 @@ def initialize(self, status_dir=conf.get_lib_dir()): self._path = os.path.join(status_dir, EventStatus.EVENT_STATUS_FILE) self._load() - def mark_event_status(self, name, version, op, status): + def mark_event_status(self, name, version, op, status): # pylint: disable=C0103 event = self._event_name(name, version, op) self._status[event] = (status is True) self._save() - def _event_name(self, name, version, op): + def _event_name(self, name, version, op): # pylint: disable=C0103 return "{0}-{1}-{2}".format(name, version, op) def _load(self): try: self._status = {} if os.path.isfile(self._path): - with open(self._path, 'r') as f: + with open(self._path, 'r') as f: # pylint: disable=C0103 self._status = json.load(f) - except Exception as e: + except Exception as e: # pylint: disable=C0103 logger.warn("Exception occurred loading event status: {0}".format(e)) self._status = {} def _save(self): try: - with open(self._path, 'w') as f: + with open(self._path, 'w') as f: # pylint: disable=C0103 json.dump(self._status, f) - except Exception as e: + except Exception as e: # pylint: disable=C0103 logger.warn("Exception occurred saving event status: {0}".format(e)) __event_status__ = EventStatus() __event_status_operations__ = [ - WALAEventOperation.ReportStatus + WALAEventOperation.ReportStatus # pylint: disable=bad-continuation ] @@ -203,7 +203,7 @@ def parse_event(data_str): return parse_json_event(data_str) except ValueError: return parse_xml_event(data_str) - except Exception as e: + except Exception as e: # pylint: disable=C0103 raise EventError("Error parsing event: {0}".format(ustr(e))) @@ -232,11 +232,11 @@ def parse_xml_event(data_str): event.parameters.append(parse_xml_param(param_node)) event.file_type = "xml" return event - except Exception as e: + except Exception as e: # pylint: disable=C0103 raise ValueError(ustr(e)) -def _encode_message(op, message): +def _encode_message(op, message): # pylint: disable=C0103 """ Gzip and base64 encode a message based on the operation. @@ -257,7 +257,7 @@ def _encode_message(op, message): :return: gzip'ed and base64 encoded message, or the original message """ - if len(message) == 0: + if len(message) == 0: # pylint: disable=len-as-condition return message if op not in SHOULD_ENCODE_MESSAGE_OP: @@ -271,8 +271,8 @@ def _encode_message(op, message): return "<>" -def _log_event(name, op, message, duration, is_success=True): - global _EVENT_MSG +def _log_event(name, op, message, duration, is_success=True): # pylint: disable=C0103 + global _EVENT_MSG # pylint: disable=W0603 message = _encode_message(op, message) if not is_success: @@ -334,7 +334,7 @@ def _get_os_version(): def _get_ram(osutil): try: return osutil.get_total_mem() - except OSUtilError as e: + except OSUtilError as e: # pylint: disable=C0103 logger.warn("Failed to get RAM info; will be missing from telemetry: {0}", ustr(e)) return 0 @@ -342,7 +342,7 @@ def _get_ram(osutil): def _get_processors(osutil): try: return osutil.get_processor_cores() - except OSUtilError as e: + except OSUtilError as e: # pylint: disable=C0103 logger.warn("Failed to get Processors info; will be missing from telemetry: {0}", ustr(e)) return 0 @@ -352,7 +352,7 @@ def initialize_vminfo_common_parameters(self, protocol): """ # create an index of the event parameters for faster updates parameters = {} - for p in self._common_parameters: + for p in self._common_parameters: # pylint: disable=C0103 parameters[p.name] = p try: @@ -360,7 +360,7 @@ def initialize_vminfo_common_parameters(self, protocol): parameters[CommonTelemetryEventSchema.TenantName].value = vminfo.tenantName parameters[CommonTelemetryEventSchema.RoleName].value = vminfo.roleName parameters[CommonTelemetryEventSchema.RoleInstanceName].value = vminfo.roleInstanceName - except Exception as e: + except Exception as e: # pylint: disable=C0103 logger.warn("Failed to get VM info from goal state; will be missing from telemetry: {0}", ustr(e)) try: @@ -371,7 +371,7 @@ def initialize_vminfo_common_parameters(self, protocol): parameters[CommonTelemetryEventSchema.ResourceGroupName].value = imds_info.resourceGroupName parameters[CommonTelemetryEventSchema.VMId].value = imds_info.vmId parameters[CommonTelemetryEventSchema.ImageOrigin].value = int(imds_info.image_origin) - except Exception as e: + except Exception as e: # pylint: disable=C0103 logger.warn("Failed to get IMDS info; will be missing from telemetry: {0}", ustr(e)) def save_event(self, data): @@ -381,7 +381,7 @@ def save_event(self, data): try: fileutil.mkdir(self.event_dir, mode=0o700) - except (IOError, OSError) as e: + except (IOError, OSError) as e: # pylint: disable=C0103 msg = "Failed to create events folder {0}. Error: {1}".format(self.event_dir, ustr(e)) raise EventError(msg) @@ -395,7 +395,7 @@ def save_event(self, data): oldest_files = existing_events[:-999] for event_file in oldest_files: os.remove(os.path.join(self.event_dir, event_file)) - except (IOError, OSError) as e: + except (IOError, OSError) as e: # pylint: disable=C0103 msg = "Failed to remove old events from events folder {0}. Error: {1}".format(self.event_dir, ustr(e)) raise EventError(msg) @@ -405,27 +405,27 @@ def save_event(self, data): with open(filename + ".tmp", 'wb+') as hfile: hfile.write(data.encode("utf-8")) os.rename(filename + ".tmp", filename + AGENT_EVENT_FILE_EXTENSION) - except (IOError, OSError) as e: + except (IOError, OSError) as e: # pylint: disable=C0103 msg = "Failed to write events to file: {0}".format(e) raise EventError(msg) def reset_periodic(self): self.periodic_events = {} - def is_period_elapsed(self, delta, h): + def is_period_elapsed(self, delta, h): # pylint: disable=C0103 return h not in self.periodic_events or \ (self.periodic_events[h] + delta) <= datetime.now() - def add_periodic(self, delta, name, op=WALAEventOperation.Unknown, is_success=True, duration=0, + def add_periodic(self, delta, name, op=WALAEventOperation.Unknown, is_success=True, duration=0, # pylint: disable=R0913,C0103 version=str(CURRENT_VERSION), message="", log_event=True, force=False): - h = hash(name + op + ustr(is_success) + message) + h = hash(name + op + ustr(is_success) + message) # pylint: disable=C0103 if force or self.is_period_elapsed(delta, h): self.add_event(name, op=op, is_success=is_success, duration=duration, version=version, message=message, log_event=log_event) self.periodic_events[h] = datetime.now() - def add_event(self, name, op=WALAEventOperation.Unknown, is_success=True, duration=0, version=str(CURRENT_VERSION), + def add_event(self, name, op=WALAEventOperation.Unknown, is_success=True, duration=0, version=str(CURRENT_VERSION), # pylint: disable=R0913,C0103 message="", log_event=True): if (not is_success) and log_event: @@ -443,7 +443,7 @@ def add_event(self, name, op=WALAEventOperation.Unknown, is_success=True, durati data = get_properties(event) try: self.save_event(json.dumps(data)) - except EventError as e: + except EventError as e: # pylint: disable=C0103 logger.periodic_error(logger.EVERY_FIFTEEN_MINUTES, "[PERIODIC] {0}".format(ustr(e))) def add_log_event(self, level, message): @@ -461,7 +461,7 @@ def add_log_event(self, level, message): except EventError: pass - def add_metric(self, category, counter, instance, value, log_event=False): + def add_metric(self, category, counter, instance, value, log_event=False): # pylint: disable=R0913 """ Create and save an event which contains a telemetry event. @@ -472,7 +472,7 @@ def add_metric(self, category, counter, instance, value, log_event=False): :param bool log_event: If true, log the collected metric in the agent log """ if log_event: - from azurelinuxagent.common.version import AGENT_NAME + from azurelinuxagent.common.version import AGENT_NAME # pylint: disable=W0404,W0621 message = "Metric {0}/{1} [{2}] = {3}".format(category, counter, instance, value) _log_event(AGENT_NAME, "METRIC", message, 0) @@ -486,7 +486,7 @@ def add_metric(self, category, counter, instance, value, log_event=False): data = get_properties(event) try: self.save_event(json.dumps(data)) - except EventError as e: + except EventError as e: # pylint: disable=C0103 logger.periodic_error(logger.EVERY_FIFTEEN_MINUTES, "[PERIODIC] {0}".format(ustr(e))) @staticmethod @@ -520,12 +520,12 @@ def _clean_up_message(message): # Parsing the log messages containing levels in it extract_level_message = log_level_format_parser.search(message) - if extract_level_message: + if extract_level_message: # pylint: disable=R1705 return extract_level_message.group(2) # The message bit else: # Parsing the log messages without levels in it. extract_message = log_format_parser.search(message) - if extract_message: + if extract_message: # pylint: disable=R1705 return extract_message.group(1) # The message bit else: return message @@ -547,7 +547,7 @@ def add_common_event_parameters(self, event, event_timestamp): # Currently only the GuestAgentExtensionEvents has these columns, the other tables dont have them so skipping # this data in those tables. common_params.extend([TelemetryEventParam(GuestAgentExtensionEventsSchema.ExtensionType, event.file_type), - TelemetryEventParam(GuestAgentExtensionEventsSchema.IsInternal, False)]) + TelemetryEventParam(GuestAgentExtensionEventsSchema.IsInternal, False)]) # pylint: disable=C0330 event.parameters.extend(common_params) event.parameters.extend(self._common_parameters) @@ -580,14 +580,14 @@ def _trim_extension_event_parameters(event): event.parameters = trimmed_params @staticmethod - def report_dropped_events_error(count, errors, op, max_errors_to_report): + def report_dropped_events_error(count, errors, op, max_errors_to_report): # pylint: disable=C0103 err_msg_format = "DroppedEventsCount: {0}\nReasons (first {1} errors): {2}" if count > 0: add_event(op=op, message=err_msg_format.format(count, max_errors_to_report, ', '.join(errors)), is_success=False) - def collect_events(self): + def collect_events(self): # pylint: disable=R0914 """ Retuns a list of events that need to be sent to the telemetry pipeline and deletes the corresponding files from the events directory. @@ -610,7 +610,7 @@ def collect_events(self): try: logger.verbose("Processing event file: {0}", event_file_path) - with open(event_file_path, "rb") as fd: + with open(event_file_path, "rb") as fd: # pylint: disable=C0103 event_data = fd.read().decode("utf-8") event = parse_event(event_data) @@ -633,11 +633,11 @@ def collect_events(self): event_list.events.append(event) finally: os.remove(event_file_path) - except UnicodeError as e: + except UnicodeError as e: # pylint: disable=C0103 unicode_error_count += 1 if len(unicode_errors) < max_collect_errors_to_report: unicode_errors.append(ustr(e)) - except Exception as e: + except Exception as e: # pylint: disable=C0103 collect_event_error_count += 1 if len(collect_event_errors) < max_collect_errors_to_report: collect_event_errors.append(ustr(e)) @@ -679,13 +679,13 @@ def elapsed_milliseconds(utc_start): if now < utc_start: return 0 - d = now - utc_start + d = now - utc_start # pylint: disable=C0103 return int(((d.days * 24 * 60 * 60 + d.seconds) * 1000) + \ (d.microseconds / 1000.0)) -def report_event(op, is_success=True, message='', log_event=True): - from azurelinuxagent.common.version import AGENT_NAME, CURRENT_VERSION +def report_event(op, is_success=True, message='', log_event=True): # pylint: disable=C0103 + from azurelinuxagent.common.version import AGENT_NAME, CURRENT_VERSION # pylint: disable=W0404,W0621 add_event(AGENT_NAME, version=str(CURRENT_VERSION), is_success=is_success, @@ -694,8 +694,8 @@ def report_event(op, is_success=True, message='', log_event=True): log_event=log_event) -def report_periodic(delta, op, is_success=True, message=''): - from azurelinuxagent.common.version import AGENT_NAME, CURRENT_VERSION +def report_periodic(delta, op, is_success=True, message=''): # pylint: disable=C0103 + from azurelinuxagent.common.version import AGENT_NAME, CURRENT_VERSION # pylint: disable=W0404,W0621 add_periodic(delta, AGENT_NAME, version=str(CURRENT_VERSION), is_success=is_success, @@ -703,7 +703,7 @@ def report_periodic(delta, op, is_success=True, message=''): op=op) -def report_metric(category, counter, instance, value, log_event=False, reporter=__event_logger__): +def report_metric(category, counter, instance, value, log_event=False, reporter=__event_logger__): # pylint: disable=R0913 """ Send a telemetry event reporting a single instance of a performance counter. :param str category: The category of the metric (cpu, memory, etc) @@ -714,7 +714,7 @@ def report_metric(category, counter, instance, value, log_event=False, reporter= :param EventLogger reporter: The EventLogger instance to which metric events should be sent """ if reporter.event_dir is None: - from azurelinuxagent.common.version import AGENT_NAME + from azurelinuxagent.common.version import AGENT_NAME # pylint: disable=W0404,W0621 logger.warn("Cannot report metric event -- Event reporter is not initialized.") message = "Metric {0}/{1} [{2}] = {3}".format(category, counter, instance, value) _log_event(AGENT_NAME, "METRIC", message, 0) @@ -730,7 +730,7 @@ def initialize_event_logger_vminfo_common_parameters(protocol, reporter=__event_ reporter.initialize_vminfo_common_parameters(protocol) -def add_event(name=AGENT_NAME, op=WALAEventOperation.Unknown, is_success=True, duration=0, version=str(CURRENT_VERSION), +def add_event(name=AGENT_NAME, op=WALAEventOperation.Unknown, is_success=True, duration=0, version=str(CURRENT_VERSION), # pylint: disable=R0913,C0103 message="", log_event=True, reporter=__event_logger__): if reporter.event_dir is None: logger.warn("Cannot add event -- Event reporter is not initialized.") @@ -763,7 +763,7 @@ def add_log_event(level, message, forced=False, reporter=__event_logger__): reporter.add_log_event(level, message) -def add_periodic(delta, name, op=WALAEventOperation.Unknown, is_success=True, duration=0, version=str(CURRENT_VERSION), +def add_periodic(delta, name, op=WALAEventOperation.Unknown, is_success=True, duration=0, version=str(CURRENT_VERSION), # pylint: disable=R0913,C0103 message="", log_event=True, force=False, reporter=__event_logger__): if reporter.event_dir is None: logger.warn("Cannot add periodic event -- Event reporter is not initialized.") @@ -778,12 +778,12 @@ def collect_events(reporter=__event_logger__): return reporter.collect_events() -def mark_event_status(name, version, op, status): +def mark_event_status(name, version, op, status): # pylint: disable=C0103 if op in __event_status_operations__: __event_status__.mark_event_status(name, version, op, status) -def should_emit_event(name, version, op, status): +def should_emit_event(name, version, op, status): # pylint: disable=C0103 return \ op not in __event_status_operations__ or \ __event_status__ is None or \ diff --git a/azurelinuxagent/common/exception.py b/azurelinuxagent/common/exception.py index f4e1f5021f..44f0ec31e6 100644 --- a/azurelinuxagent/common/exception.py +++ b/azurelinuxagent/common/exception.py @@ -87,7 +87,7 @@ class ExtensionUpdateError(ExtensionError): When failed to update an extension """ - def __init__(self, msg=None, inner=None, code=-1): + def __init__(self, msg=None, inner=None, code=-1): # pylint: disable=W0235 super(ExtensionUpdateError, self).__init__(msg, inner, code) @@ -96,7 +96,7 @@ class ExtensionDownloadError(ExtensionError): When failed to download and setup an extension """ - def __init__(self, msg=None, inner=None, code=-1): + def __init__(self, msg=None, inner=None, code=-1): # pylint: disable=W0235 super(ExtensionDownloadError, self).__init__(msg, inner, code) @@ -150,7 +150,7 @@ class ProtocolNotFoundError(ProtocolError): Azure protocol endpoint not found """ - def __init__(self, msg=None, inner=None): + def __init__(self, msg=None, inner=None): # pylint: disable=W0235 super(ProtocolNotFoundError, self).__init__(msg, inner) @@ -168,7 +168,7 @@ class InvalidContainerError(HttpError): Container id sent in the header is invalid """ - def __init__(self, msg=None, inner=None): + def __init__(self, msg=None, inner=None): # pylint: disable=W0235 super(InvalidContainerError, self).__init__(msg, inner) @@ -223,7 +223,7 @@ def __init__(self, msg=None, inner=None): super(InvalidExtensionEventError, self).__init__(msg, inner) -class ExtensionErrorCodes(object): +class ExtensionErrorCodes(object): # pylint: disable=R0903 """ Common Error codes used across by Compute RP for better understanding the cause and clarify common occurring errors diff --git a/azurelinuxagent/common/future.py b/azurelinuxagent/common/future.py index 577fb12e18..79e6e3ba12 100644 --- a/azurelinuxagent/common/future.py +++ b/azurelinuxagent/common/future.py @@ -6,38 +6,40 @@ # Note broken dependency handling to avoid potential backward # compatibility issues on different distributions try: - import distro + import distro # pylint: disable=E0401 except Exception: pass +# pylint: disable=W0105 """ Add alias for python2 and python3 libs and functions. -""" +""" +# pylint: enable=W0105 if sys.version_info[0] == 3: - import http.client as httpclient - from urllib.parse import urlparse + import http.client as httpclient # pylint: disable=W0611,import-error + from urllib.parse import urlparse # pylint: disable=W0611,import-error,no-name-in-module - """Rename Python3 str to ustr""" - ustr = str + """Rename Python3 str to ustr""" # pylint: disable=W0105 + ustr = str # pylint: disable=C0103 - bytebuffer = memoryview + bytebuffer = memoryview # pylint: disable=C0103 - from collections import OrderedDict + from collections import OrderedDict # pylint: disable=W0611 elif sys.version_info[0] == 2: - import httplib as httpclient - from urlparse import urlparse + import httplib as httpclient # pylint: disable=E0401,W0611 + from urlparse import urlparse # pylint: disable=E0401 - """Rename Python2 unicode to ustr""" - ustr = unicode + """Rename Python2 unicode to ustr""" # pylint: disable=W0105 + ustr = unicode # pylint: disable=E0602,invalid-name - bytebuffer = buffer + bytebuffer = buffer # pylint: disable=E0602,invalid-name if sys.version_info[1] >= 7: - from collections import OrderedDict # For Py 2.7+ + from collections import OrderedDict # For Py 2.7+ # pylint: disable=C0412 else: - from ordereddict import OrderedDict # Works only on 2.6 + from ordereddict import OrderedDict # Works only on 2.6 # pylint: disable=E0401 else: raise ImportError("Unknown python version: {0}".format(sys.version_info)) @@ -46,9 +48,9 @@ def get_linux_distribution(get_full_name, supported_dists): """Abstract platform.linux_distribution() call which is deprecated as of Python 3.5 and removed in Python 3.7""" try: - supported = platform._supported_dists + (supported_dists,) + supported = platform._supported_dists + (supported_dists,) # pylint: disable=W0212 osinfo = list( - platform.linux_distribution( + platform.linux_distribution( # pylint: disable=W1505 full_distribution_name=get_full_name, supported_dists=supported ) @@ -61,7 +63,7 @@ def get_linux_distribution(get_full_name, supported_dists): if not osinfo or osinfo == ['', '', '']: return get_linux_distribution_from_distro(get_full_name) - full_name = platform.linux_distribution()[0].strip() + full_name = platform.linux_distribution()[0].strip() # pylint: disable=W1505 osinfo.append(full_name) except AttributeError: return get_linux_distribution_from_distro(get_full_name) @@ -93,7 +95,7 @@ def get_openwrt_platform(): openwrt_version = re.compile(r"^DISTRIB_RELEASE=['\"](\d+\.\d+.\d+)['\"]") openwrt_product = re.compile(r"^DISTRIB_ID=['\"]([\w-]+)['\"]") - with open('/etc/openwrt_release', 'r') as fh: + with open('/etc/openwrt_release', 'r') as fh: # pylint: disable=C0103 content = fh.readlines() for line in content: version_matches = openwrt_version.match(line) diff --git a/azurelinuxagent/common/logcollector.py b/azurelinuxagent/common/logcollector.py index 4192db7983..86a1ba4c35 100644 --- a/azurelinuxagent/common/logcollector.py +++ b/azurelinuxagent/common/logcollector.py @@ -61,7 +61,7 @@ _LOGGER = logging.getLogger(__name__) -class LogCollector(object): +class LogCollector(object): # pylint: disable=R0903 _TRUNCATED_FILE_PREFIX = "truncated_" @@ -115,7 +115,7 @@ def _encode_command_output(output): process = subprocess.Popen(command, stdin=subprocess.PIPE, stdout=stdout, stderr=subprocess.PIPE, shell=False) stdout, stderr = process.communicate() return_code = process.returncode - except Exception as e: + except Exception as e: # pylint: disable=C0103 error_msg = u"Command [{0}] raised unexpected exception: [{1}]".format(format_command(command), ustr(e)) _LOGGER.error(error_msg) return @@ -146,7 +146,7 @@ def _expand_must_collect_files(): def _read_manifest_file(self): with open(self._manifest_file_path, "rb") as in_file: data = in_file.read() - if data is None: + if data is None: # pylint: disable=R1705 return None else: data = ustr(data, encoding="utf-8") @@ -173,7 +173,7 @@ def _convert_file_name_to_archive_name(file_name): # For non-truncated files: /var/log/waagent.log on disk becomes var/log/waagent.log in archive # (leading separator is removed by the archive). # For truncated files: /var/truncated/var/log/syslog.1 on disk becomes truncated_var_log_syslog.1 in archive. - if file_name.startswith(_TRUNCATED_FILES_DIR): + if file_name.startswith(_TRUNCATED_FILES_DIR): # pylint: disable=R1705 original_file_path = file_name[len(_TRUNCATED_FILES_DIR):].lstrip(os.path.sep) archive_file_name = LogCollector._TRUNCATED_FILE_PREFIX + original_file_path.replace(os.path.sep, "_") return archive_file_name @@ -195,9 +195,9 @@ def _remove_uncollected_truncated_files(files_to_collect): @staticmethod def _expand_parameters(manifest_data): - _LOGGER.info("Using {0} as $LIB_DIR".format(_AGENT_LIB_DIR)) - _LOGGER.info("Using {0} as $LOG_DIR".format(_EXTENSION_LOG_DIR)) - _LOGGER.info("Using {0} as $AGENT_LOG".format(_AGENT_LOG)) + _LOGGER.info("Using {0} as $LIB_DIR".format(_AGENT_LIB_DIR)) # pylint: disable=W1202 + _LOGGER.info("Using {0} as $LOG_DIR".format(_EXTENSION_LOG_DIR)) # pylint: disable=W1202 + _LOGGER.info("Using {0} as $AGENT_LOG".format(_AGENT_LOG)) # pylint: disable=W1202 new_manifest = [] for line in manifest_data: @@ -223,8 +223,8 @@ def _process_manifest_file(self): contents = entry.split(",") if len(contents) != 2: # If it's not a comment or an empty line, it's a malformed entry - if not entry.startswith("#") and len(entry.strip()) > 0: - _LOGGER.error("Couldn't parse \"{0}\"".format(entry)) + if not entry.startswith("#") and len(entry.strip()) > 0: # pylint: disable=len-as-condition + _LOGGER.error("Couldn't parse \"{0}\"".format(entry)) # pylint: disable=W1202 continue command, value = contents @@ -246,7 +246,7 @@ def _truncate_large_file(file_path): # Binary files cannot be truncated, don't include large binary files ext = os.path.splitext(file_path)[1] if ext in [".gz", ".zip", ".xz"]: - _LOGGER.warning("Discarding large binary file {0}".format(file_path)) + _LOGGER.warning("Discarding large binary file {0}".format(file_path)) # pylint: disable=W1202 return None truncated_file_path = os.path.join(_TRUNCATED_FILES_DIR, file_path.replace(os.path.sep, "_")) @@ -260,15 +260,15 @@ def _truncate_large_file(file_path): return truncated_file_path # Get the last N bytes of the file - with open(truncated_file_path, "w+") as fh: + with open(truncated_file_path, "w+") as fh: # pylint: disable=C0103 LogCollector._run_shell_command(["tail", "-c", str(_FILE_SIZE_LIMIT), file_path], stdout=fh) return truncated_file_path - except OSError as e: - _LOGGER.error("Failed to truncate large file: {0}".format(ustr(e))) + except OSError as e: # pylint: disable=C0103 + _LOGGER.error("Failed to truncate large file: {0}".format(ustr(e))) # pylint: disable=W1202 return None - def _get_file_priority(self, file): + def _get_file_priority(self, file): # pylint: disable=redefined-builtin # The sooner the file appears in the must collect list, the bigger its priority. # Priority is higher the lower the number (0 is highest priority). try: @@ -281,7 +281,7 @@ def _get_priority_files_list(self, file_list): # Given a list of files to collect, determine if they show up in the must collect list and build a priority # queue. The queue will determine the order in which the files are collected, highest priority files first. priority_file_queue = [] - for file in file_list: + for file in file_list: # pylint: disable=redefined-builtin priority = self._get_file_priority(file) heappush(priority_file_queue, (priority, file)) @@ -304,16 +304,16 @@ def _get_final_list_for_archive(self, priority_file_queue): if os.path.getsize(file_path) <= _FILE_SIZE_LIMIT: final_files_to_collect.append(file_path) - _LOGGER.info("Adding file {0}, size {1}b".format(file_path, file_size)) + _LOGGER.info("Adding file {0}, size {1}b".format(file_path, file_size)) # pylint: disable=W1202 else: truncated_file_path = self._truncate_large_file(file_path) if truncated_file_path: - _LOGGER.info("Adding truncated file {0}, size {1}b".format(truncated_file_path, file_size)) + _LOGGER.info("Adding truncated file {0}, size {1}b".format(truncated_file_path, file_size)) # pylint: disable=W1202 final_files_to_collect.append(truncated_file_path) total_uncompressed_size += file_size - _LOGGER.info("Uncompressed archive size is {0}b".format(total_uncompressed_size)) + _LOGGER.info("Uncompressed archive size is {0}b".format(total_uncompressed_size)) # pylint: disable=W1202 return final_files_to_collect @@ -345,17 +345,17 @@ def collect_logs(self): _LOGGER.info("### Creating compressed archive ###") with zipfile.ZipFile(_COMPRESSED_ARCHIVE_PATH, "w", compression=zipfile.ZIP_DEFLATED) as compressed_archive: - for file in files_to_collect: + for file in files_to_collect: # pylint: disable=redefined-builtin archive_file_name = LogCollector._convert_file_name_to_archive_name(file) compressed_archive.write(file, arcname=archive_file_name) compressed_archive_size = os.path.getsize(_COMPRESSED_ARCHIVE_PATH) - _LOGGER.info("Successfully compressed files. " + _LOGGER.info("Successfully compressed files. " # pylint: disable=W1202 "Compressed archive size is {0}b".format(compressed_archive_size)) compressed_archive.write(_OUTPUT_RESULTS_FILE_PATH, arcname="results.txt") return True - except Exception as e: + except Exception as e: # pylint: disable=C0103 msg = "Failed to collect logs: {0}".format(ustr(e)) _LOGGER.error(msg) diff --git a/azurelinuxagent/common/logger.py b/azurelinuxagent/common/logger.py index 2b1e8ab76d..4dd9a7f205 100644 --- a/azurelinuxagent/common/logger.py +++ b/azurelinuxagent/common/logger.py @@ -52,12 +52,12 @@ def reset_periodic(self): def set_prefix(self, prefix): self.prefix = prefix - def _is_period_elapsed(self, delta, h): + def _is_period_elapsed(self, delta, h): # pylint: disable=C0103 return h not in self.logger.periodic_messages or \ (self.logger.periodic_messages[h] + delta) <= datetime.now() def _periodic(self, delta, log_level_op, msg_format, *args): - h = hash(msg_format) + h = hash(msg_format) # pylint: disable=C0103 if self._is_period_elapsed(delta, h): log_level_op(msg_format, *args) self.logger.periodic_messages[h] = datetime.now() @@ -87,7 +87,7 @@ def error(self, msg_format, *args): self.log(LogLevel.ERROR, msg_format, *args) def log(self, level, msg_format, *args): - def write_log(log_appender): + def write_log(log_appender): # pylint: disable=W0612 """ The appender_lock flag is used to signal if the logger is currently in use. This prevents a subsequent log coming in due to writing of a log statement to be not written. @@ -125,9 +125,9 @@ def write_log(log_appender): log_appender.appender_lock = False # if msg_format is not unicode convert it to unicode - if type(msg_format) is not ustr: + if type(msg_format) is not ustr: # pylint: disable=C0123 msg_format = ustr(msg_format, errors="backslashreplace") - if len(args) > 0: + if len(args) > 0: # pylint: disable=len-as-condition msg = msg_format.format(*args) else: msg = msg_format @@ -145,7 +145,7 @@ def write_log(log_appender): for appender in self.appenders: appender.write(level, log_item) # - # TODO: we should actually call + # TODO: we should actually call # pylint: disable=W0511 # # write_log(appender) # @@ -158,7 +158,7 @@ def write_log(log_appender): for appender in self.logger.appenders: appender.write(level, log_item) # - # TODO: call write_log instead (see comment above) + # TODO: call write_log instead (see comment above) # pylint: disable=W0511 # def add_appender(self, appender_type, level, path): @@ -166,7 +166,7 @@ def add_appender(self, appender_type, level, path): self.appenders.append(appender) -class Appender(object): +class Appender(object): # pylint: disable=R0903 def __init__(self, level): self.appender_lock = False self.level = level @@ -175,7 +175,7 @@ def write(self, level, msg): pass -class ConsoleAppender(Appender): +class ConsoleAppender(Appender): # pylint: disable=R0903 def __init__(self, level, path): super(ConsoleAppender, self).__init__(level) self.path = path @@ -189,7 +189,7 @@ def write(self, level, msg): pass -class FileAppender(Appender): +class FileAppender(Appender): # pylint: disable=R0903 def __init__(self, level, path): super(FileAppender, self).__init__(level) self.path = path @@ -203,8 +203,8 @@ def write(self, level, msg): pass -class StdoutAppender(Appender): - def __init__(self, level): +class StdoutAppender(Appender): # pylint: disable=R0903 + def __init__(self, level): # pylint: disable=W0235 super(StdoutAppender, self).__init__(level) def write(self, level, msg): @@ -215,7 +215,7 @@ def write(self, level, msg): pass -class TelemetryAppender(Appender): +class TelemetryAppender(Appender): # pylint: disable=R0903 def __init__(self, level, event_func): super(TelemetryAppender, self).__init__(level) self.event_func = event_func @@ -232,7 +232,7 @@ def write(self, level, msg): DEFAULT_LOGGER = Logger() -class LogLevel(object): +class LogLevel(object): # pylint: disable=R0903 VERBOSE = 0 INFO = 1 WARNING = 2 @@ -245,7 +245,7 @@ class LogLevel(object): ] -class AppenderType(object): +class AppenderType(object): # pylint: disable=R0903 FILE = 0 CONSOLE = 1 STDOUT = 2 @@ -317,7 +317,7 @@ def log(level, msg_format, *args): def _create_logger_appender(appender_type, level=LogLevel.INFO, path=None): - if appender_type == AppenderType.CONSOLE: + if appender_type == AppenderType.CONSOLE: # pylint: disable=R1705 return ConsoleAppender(level, path) elif appender_type == AppenderType.FILE: return FileAppender(level, path) diff --git a/azurelinuxagent/common/osutil/alpine.py b/azurelinuxagent/common/osutil/alpine.py index eca67dfcd2..69a5de1912 100644 --- a/azurelinuxagent/common/osutil/alpine.py +++ b/azurelinuxagent/common/osutil/alpine.py @@ -34,11 +34,11 @@ def is_dhcp_enabled(self): def get_dhcp_pid(self): return self._get_dhcp_pid(["pidof", "dhcpcd"]) - def restart_if(self, ifname): + def restart_if(self, ifname): # pylint: disable=W0221 logger.info('restarting {} (sort of, actually SIGHUPing dhcpcd)'.format(ifname)) pid = self.get_dhcp_pid() - if pid != None: - ret = shellutil.run_get_output('kill -HUP {}'.format(pid)) + if pid != None: # pylint: disable=C0121 + ret = shellutil.run_get_output('kill -HUP {}'.format(pid)) # pylint: disable=W0612 def set_ssh_client_alive_interval(self): # Alpine will handle this. diff --git a/azurelinuxagent/common/osutil/arch.py b/azurelinuxagent/common/osutil/arch.py index a6ba001ab4..f25394666a 100644 --- a/azurelinuxagent/common/osutil/arch.py +++ b/azurelinuxagent/common/osutil/arch.py @@ -16,7 +16,7 @@ # Requires Python 2.6+ and Openssl 1.0+ # -import os +import os # pylint: disable=W0611 import azurelinuxagent.common.utils.shellutil as shellutil from azurelinuxagent.common.osutil.default import DefaultOSUtil @@ -32,7 +32,7 @@ def is_dhcp_enabled(self): def start_network(self): return shellutil.run("systemctl start systemd-networkd", chk_err=False) - def restart_if(self, iface): + def restart_if(self, iface): # pylint: disable=W0613,W0221 shellutil.run("systemctl restart systemd-networkd") def restart_ssh_service(self): diff --git a/azurelinuxagent/common/osutil/bigip.py b/azurelinuxagent/common/osutil/bigip.py index 61d3c695f9..916f53a60b 100644 --- a/azurelinuxagent/common/osutil/bigip.py +++ b/azurelinuxagent/common/osutil/bigip.py @@ -42,7 +42,7 @@ class BigIpOSUtil(DefaultOSUtil): - def __init__(self): + def __init__(self): # pylint: disable=W0235 super(BigIpOSUtil, self).__init__() def _wait_until_mcpd_is_initialized(self): @@ -58,10 +58,10 @@ def _wait_until_mcpd_is_initialized(self): :raises OSUtilError: Raises exception if mcpd does not come up within roughly 50 minutes (100 * 30 seconds) """ - for retries in range(1, 100): + for retries in range(1, 100): # pylint: disable=W0612 # Retry until mcpd completes startup: logger.info("Checking to see if mcpd is up") - rc = shellutil.run("/usr/bin/tmsh -a show sys mcp-state field-fmt 2>/dev/null | grep phase | grep running", chk_err=False) + rc = shellutil.run("/usr/bin/tmsh -a show sys mcp-state field-fmt 2>/dev/null | grep phase | grep running", chk_err=False) # pylint: disable=C0103 if rc == 0: logger.info("mcpd is up!") break @@ -76,7 +76,7 @@ def _wait_until_mcpd_is_initialized(self): def _save_sys_config(self): cmd = "/usr/bin/tmsh save sys config" - rc = shellutil.run(cmd) + rc = shellutil.run(cmd) # pylint: disable=C0103 if rc != 0: logger.error("WARNING: Cannot save sys config on 1st boot.") return rc @@ -223,7 +223,7 @@ def get_dvd_device(self, dev_dir='/dev'): return "/dev/{0}".format(dvd.group(0)) raise OSUtilError("Failed to get dvd device") - def mount_dvd(self, **kwargs): + def mount_dvd(self, **kwargs): # pylint: disable=W0221 """Mount the DVD containing the provisioningiso.iso file This is the _first_ hook that WAAgent provides for us, so this is the @@ -285,11 +285,11 @@ def get_first_if(self): iface = self._format_single_interface_name(sock, i) # Azure public was returning "lo:1" when deploying WAF - if b'lo' in iface: + if b'lo' in iface: # pylint: disable=R1724 continue else: break - return iface.decode('latin-1'), socket.inet_ntoa(sock[i+20:i+24]) + return iface.decode('latin-1'), socket.inet_ntoa(sock[i+20:i+24]) # pylint: disable=undefined-loop-variable def _format_single_interface_name(self, sock, offset): return sock[offset:offset+16].split(b'\0', 1)[0] @@ -315,9 +315,9 @@ def device_for_ide_port(self, port_id): :param port_id: :return: """ - for retries in range(1, 100): + for retries in range(1, 100): # pylint: disable=W0612 # Retry until devices are ready - if os.path.exists("/sys/bus/vmbus/devices/"): + if os.path.exists("/sys/bus/vmbus/devices/"): # pylint: disable=R1723 break else: time.sleep(10) diff --git a/azurelinuxagent/common/osutil/clearlinux.py b/azurelinuxagent/common/osutil/clearlinux.py index ca7d83a462..dd2decffed 100644 --- a/azurelinuxagent/common/osutil/clearlinux.py +++ b/azurelinuxagent/common/osutil/clearlinux.py @@ -16,21 +16,21 @@ # Requires Python 2.6+ and Openssl 1.0+ # -import os -import re -import pwd -import shutil -import socket -import array -import struct -import fcntl -import time -import base64 +import os # pylint: disable=W0611 +import re # pylint: disable=W0611 +import pwd # pylint: disable=W0611 +import shutil # pylint: disable=W0611 +import socket # pylint: disable=W0611 +import array # pylint: disable=W0611 +import struct # pylint: disable=W0611 +import fcntl # pylint: disable=W0611 +import time # pylint: disable=W0611 +import base64 # pylint: disable=W0611 import azurelinuxagent.common.conf as conf -import azurelinuxagent.common.logger as logger +import azurelinuxagent.common.logger as logger # pylint: disable=W0611 import azurelinuxagent.common.utils.fileutil as fileutil import azurelinuxagent.common.utils.shellutil as shellutil -import azurelinuxagent.common.utils.textutil as textutil +import azurelinuxagent.common.utils.textutil as textutil # pylint: disable=W0611 from azurelinuxagent.common.osutil.default import DefaultOSUtil class ClearLinuxUtil(DefaultOSUtil): @@ -46,7 +46,7 @@ def is_dhcp_enabled(self): def start_network(self) : return shellutil.run("systemctl start systemd-networkd", chk_err=False) - def restart_if(self, iface): + def restart_if(self, iface): # pylint: disable=W0613,W0221 shellutil.run("systemctl restart systemd-networkd") def restart_ssh_service(self): @@ -79,14 +79,14 @@ def del_root_password(self): passwd_content = fileutil.read_file(passwd_file_path) if not passwd_content: # Empty file is no better than no file - raise FileNotFoundError - except FileNotFoundError: + raise FileNotFoundError # pylint: disable=undefined-variable + except FileNotFoundError: # pylint: disable=undefined-variable new_passwd = ["root:*LOCK*:14600::::::"] else: passwd = passwd_content.split('\n') new_passwd = [x for x in passwd if not x.startswith("root:")] new_passwd.insert(0, "root:*LOCK*:14600::::::") fileutil.write_file(passwd_file_path, "\n".join(new_passwd)) - except IOError as e: - raise OSUtilError("Failed to delete root password:{0}".format(e)) - pass + except IOError as e: # pylint: disable=C0103 + raise OSUtilError("Failed to delete root password:{0}".format(e)) # pylint: disable=E0602 + pass # pylint: disable=W0107 diff --git a/azurelinuxagent/common/osutil/coreos.py b/azurelinuxagent/common/osutil/coreos.py index e8070cf9b0..5557fed8df 100644 --- a/azurelinuxagent/common/osutil/coreos.py +++ b/azurelinuxagent/common/osutil/coreos.py @@ -54,7 +54,7 @@ def is_dhcp_enabled(self): def start_network(self): return shellutil.run("systemctl start systemd-networkd", chk_err=False) - def restart_if(self, *dummy, **_): + def restart_if(self, *dummy, **_): # pylint: disable=W0613,W0222,arguments-differ shellutil.run("systemctl restart systemd-networkd") def restart_ssh_service(self): diff --git a/azurelinuxagent/common/osutil/debian.py b/azurelinuxagent/common/osutil/debian.py index 6e573ef33e..b9223282c3 100644 --- a/azurelinuxagent/common/osutil/debian.py +++ b/azurelinuxagent/common/osutil/debian.py @@ -16,20 +16,20 @@ # Requires Python 2.6+ and Openssl 1.0+ # -import os -import re -import pwd -import shutil -import socket -import array -import struct -import fcntl -import time -import base64 -import azurelinuxagent.common.logger as logger -import azurelinuxagent.common.utils.fileutil as fileutil +import os # pylint: disable=W0611 +import re # pylint: disable=W0611 +import pwd # pylint: disable=W0611 +import shutil # pylint: disable=W0611 +import socket # pylint: disable=W0611 +import array # pylint: disable=W0611 +import struct # pylint: disable=W0611 +import fcntl # pylint: disable=W0611 +import time # pylint: disable=W0611 +import base64 # pylint: disable=W0611 +import azurelinuxagent.common.logger as logger # pylint: disable=W0611 +import azurelinuxagent.common.utils.fileutil as fileutil # pylint: disable=W0611 import azurelinuxagent.common.utils.shellutil as shellutil -import azurelinuxagent.common.utils.textutil as textutil +import azurelinuxagent.common.utils.textutil as textutil # pylint: disable=W0611 from azurelinuxagent.common.osutil.default import DefaultOSUtil diff --git a/azurelinuxagent/common/osutil/default.py b/azurelinuxagent/common/osutil/default.py index 521776818e..48152b73bb 100644 --- a/azurelinuxagent/common/osutil/default.py +++ b/azurelinuxagent/common/osutil/default.py @@ -1,4 +1,4 @@ -# +# # pylint: disable=C0302 # Copyright 2018 Microsoft Corporation # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -31,15 +31,18 @@ import struct import sys import time -from pwd import getpwall +from pwd import getpwall # pylint: disable=ungrouped-imports import array +#pylint: disable=R0801 import azurelinuxagent.common.conf as conf import azurelinuxagent.common.logger as logger import azurelinuxagent.common.utils.fileutil as fileutil import azurelinuxagent.common.utils.shellutil as shellutil import azurelinuxagent.common.utils.textutil as textutil +#pylint: enable=R0801 + from azurelinuxagent.common.exception import OSUtilError from azurelinuxagent.common.future import ustr from azurelinuxagent.common.utils.cryptutil import CryptUtil @@ -56,7 +59,7 @@ if needed. """ -_IPTABLES_VERSION_PATTERN = re.compile("^[^\d\.]*([\d\.]+).*$") +_IPTABLES_VERSION_PATTERN = re.compile("^[^\d\.]*([\d\.]+).*$") # pylint: disable=W1401 _IPTABLES_LOCKING_VERSION = FlexibleVersion('1.4.21') @@ -74,7 +77,7 @@ def _get_iptables_version_command(): def _get_firewall_accept_command(wait, command, destination, owner_uid): - return _add_wait(wait, ["iptables", "-t", "security", command, "OUTPUT", "-d", destination, "-p", "tcp", "-m", "owner", "--uid-owner", str(owner_uid), "-j" "ACCEPT"]) + return _add_wait(wait, ["iptables", "-t", "security", command, "OUTPUT", "-d", destination, "-p", "tcp", "-m", "owner", "--uid-owner", str(owner_uid), "-j" "ACCEPT"]) # pylint: disable=W1404,W1403 def _get_firewall_drop_command(wait, command, destination): @@ -101,11 +104,11 @@ def _get_firewall_delete_owner_accept_command(wait, destination, owner_uid): def _get_firewall_delete_conntrack_drop_command(wait, destination): return _add_wait(wait, ["iptables", "-t", "security", "-D", "OUTPUT", "-d", destination, "-p", "tcp", "-m", "conntrack", "--ctstate", "INVALID,NEW", "-j", "DROP"]) -PACKET_PATTERN = "^\s*(\d+)\s+(\d+)\s+DROP\s+.*{0}[^\d]*$" +PACKET_PATTERN = "^\s*(\d+)\s+(\d+)\s+DROP\s+.*{0}[^\d]*$" # pylint: disable=W1401 ALL_CPUS_REGEX = re.compile('^cpu .*') -_enable_firewall = True +_enable_firewall = True # pylint: disable=C0103 DMIDECODE_CMD = 'dmidecode --string system-uuid' PRODUCT_ID_FILE = '/sys/class/dmi/id/product_uuid' @@ -118,12 +121,12 @@ def _get_firewall_delete_conntrack_drop_command(wait, destination): IOCTL_SIOCGIFHWADDR = 0x8927 IFNAMSIZ = 16 -IP_COMMAND_OUTPUT = re.compile('^\d+:\s+(\w+):\s+(.*)$') +IP_COMMAND_OUTPUT = re.compile('^\d+:\s+(\w+):\s+(.*)$') # pylint: disable=W1401 STORAGE_DEVICE_PATH = '/sys/bus/vmbus/devices/' GEN2_DEVICE_ID = 'f8b3781a-1e82-4818-a1c3-63d806ec15bb' -class DefaultOSUtil(object): +class DefaultOSUtil(object): # pylint: disable=R0904 def __init__(self): self.agent_conf_file_path = '/etc/waagent.conf' self.selinux = None @@ -137,7 +140,7 @@ def get_service_name(): def get_firewall_dropped_packets(self, dst_ip=None): # If a previous attempt failed, do not retry - global _enable_firewall + global _enable_firewall # pylint: disable=W0603,C0103 if not _enable_firewall: return 0 @@ -149,12 +152,12 @@ def get_firewall_dropped_packets(self, dst_ip=None): pattern = re.compile(PACKET_PATTERN.format(dst_ip)) for line in output.split('\n'): - m = pattern.match(line) + m = pattern.match(line) # pylint: disable=C0103 if m is not None: return int(m.group(1)) - except Exception as e: - if isinstance(e, CommandError) and e.returncode == 3: + except Exception as e: # pylint: disable=C0103 + if isinstance(e, CommandError) and e.returncode == 3: # pylint: disable=E1101 # Transient error that we ignore. This code fires every loop # of the daemon (60m), so we will get the value eventually. return 0 @@ -163,7 +166,7 @@ def get_firewall_dropped_packets(self, dst_ip=None): return 0 - except Exception as e: + except Exception as e: # pylint: disable=C0103 _enable_firewall = False logger.warn("Unable to retrieve firewall packets dropped" "{0}".format(ustr(e))) @@ -173,12 +176,12 @@ def get_firewall_will_wait(self): # Determine if iptables will serialize access try: output = shellutil.run_command(_get_iptables_version_command()) - except Exception as e: + except Exception as e: # pylint: disable=C0103 msg = "Unable to determine version of iptables: {0}".format(ustr(e)) logger.warn(msg) raise Exception(msg) - m = _IPTABLES_VERSION_PATTERN.match(output) + m = _IPTABLES_VERSION_PATTERN.match(output) # pylint: disable=C0103 if m is None: msg = "iptables did not return version information: {0}".format(output) logger.warn(msg) @@ -194,10 +197,10 @@ def _delete_rule(self, rule): Continually execute the delete operation until the return code is non-zero or the limit has been reached. """ - for i in range(1, 100): + for i in range(1, 100): # pylint: disable=W0612 try: - rc = shellutil.run_command(rule) - except CommandError as e: + rc = shellutil.run_command(rule) # pylint: disable=W0612,C0103 + except CommandError as e: # pylint: disable=C0103 if e.returncode == 1: return if e.returncode == 2: @@ -205,7 +208,7 @@ def _delete_rule(self, rule): def remove_firewall(self, dst_ip, uid): # If a previous attempt failed, do not retry - global _enable_firewall + global _enable_firewall # pylint: disable=W0603,C0103 if not _enable_firewall: return False @@ -220,7 +223,7 @@ def remove_firewall(self, dst_ip, uid): return True - except Exception as e: + except Exception as e: # pylint: disable=C0103 _enable_firewall = False logger.info("Unable to remove firewall -- " "no further attempts will be made: " @@ -229,7 +232,7 @@ def remove_firewall(self, dst_ip, uid): def enable_firewall(self, dst_ip, uid): # If a previous attempt failed, do not retry - global _enable_firewall + global _enable_firewall # pylint: disable=W0603,C0103 if not _enable_firewall: return False @@ -237,13 +240,13 @@ def enable_firewall(self, dst_ip, uid): wait = self.get_firewall_will_wait() # If the DROP rule exists, make no changes - firewall_established = False + firewall_established = False # pylint: disable=W0612 try: drop_rule = _get_firewall_drop_command(wait, "-C", dst_ip) shellutil.run_command(drop_rule) logger.verbose("Firewall appears established") return True - except CommandError as e: + except CommandError as e: # pylint: disable=C0103 if e.returncode == 2: self.remove_firewall(dst_ip, uid) msg = "please upgrade iptables to a version that supports the -C option" @@ -254,7 +257,7 @@ def enable_firewall(self, dst_ip, uid): try: accept_rule = _get_firewall_accept_command(wait, "-A", dst_ip, uid) shellutil.run_command(accept_rule) - except Exception as e: + except Exception as e: # pylint: disable=C0103 msg = "Unable to add ACCEPT firewall rule '{0}' - {1}".format(accept_rule, ustr(e)) logger.warn(msg) raise Exception(msg) @@ -262,7 +265,7 @@ def enable_firewall(self, dst_ip, uid): try: drop_rule = _get_firewall_drop_command(wait, "-A", dst_ip) shellutil.run_command(drop_rule) - except Exception as e: + except Exception as e: # pylint: disable=C0103 msg = "Unable to add DROP firewall rule '{0}' - {1}".format(drop_rule, ustr(e)) logger.warn(msg) raise Exception(msg) @@ -272,12 +275,12 @@ def enable_firewall(self, dst_ip, uid): try: output = shellutil.run_command(_get_firewall_list_command(wait)) logger.info("Firewall rules:\n{0}".format(output)) - except Exception as e: + except Exception as e: # pylint: disable=C0103 logger.warn("Listing firewall rules failed: {0}".format(ustr(e))) return True - except Exception as e: + except Exception as e: # pylint: disable=C0103 _enable_firewall = False logger.info("Unable to establish firewall -- " "no further attempts will be made: " @@ -285,7 +288,7 @@ def enable_firewall(self, dst_ip, uid): return False @staticmethod - def _correct_instance_id(id): + def _correct_instance_id(id): # pylint: disable=W0622,C0103 ''' Azure stores the instance ID with an incorrect byte ordering for the first parts. For example, the ID returned by the metadata service: @@ -305,11 +308,11 @@ def _correct_instance_id(id): parts = id.split('-') return '-'.join([ - textutil.swap_hexstring(parts[0], width=2), - textutil.swap_hexstring(parts[1], width=2), - textutil.swap_hexstring(parts[2], width=2), - parts[3], - parts[4] + textutil.swap_hexstring(parts[0], width=2), # pylint: disable=C0330 + textutil.swap_hexstring(parts[1], width=2), # pylint: disable=C0330 + textutil.swap_hexstring(parts[2], width=2), # pylint: disable=C0330 + parts[3], # pylint: disable=C0330 + parts[4] # pylint: disable=C0330 ]) def is_current_instance_id(self, id_that): @@ -334,10 +337,10 @@ def get_instance_id(self): If nothing works (for old VMs), return the empty string ''' if os.path.isfile(PRODUCT_ID_FILE): - s = fileutil.read_file(PRODUCT_ID_FILE).strip() + s = fileutil.read_file(PRODUCT_ID_FILE).strip() # pylint: disable=C0103 else: - rc, s = shellutil.run_get_output(DMIDECODE_CMD) + rc, s = shellutil.run_get_output(DMIDECODE_CMD) # pylint: disable=C0103 if rc != 0 or UUID_PATTERN.match(s) is None: return "" @@ -366,11 +369,11 @@ def is_sys_user(self, username): "/etc/login.defs") if uidmin_def is not None: uidmin = int(uidmin_def.split()[1]) - except IOError as e: + except IOError as e: # pylint: disable=W0612,C0103 pass - if uidmin == None: + if uidmin == None: # pylint: disable=C0121 uidmin = 100 - if userentry != None and userentry[2] < uidmin: + if userentry != None and userentry[2] < uidmin: # pylint: disable=C0121,R1705,R1703 return True else: return False @@ -436,7 +439,7 @@ def conf_sudoer(self, username, nopasswd=False, remove=False): sudoers = content.split("\n") sudoers = [x for x in sudoers if username not in x] fileutil.write_file(sudoers_wagent, "\n".join(sudoers)) - except IOError as e: + except IOError as e: # pylint: disable=C0103 raise OSUtilError("Failed to remove sudoer: {0}".format(e)) def del_root_password(self): @@ -447,7 +450,7 @@ def del_root_password(self): new_passwd = [x for x in passwd if not x.startswith("root:")] new_passwd.insert(0, "root:*LOCK*:14600::::::") fileutil.write_file(passwd_file_path, "\n".join(new_passwd)) - except IOError as e: + except IOError as e: # pylint: disable=C0103 raise OSUtilError("Failed to delete root password:{0}".format(e)) @staticmethod @@ -525,8 +528,8 @@ def is_selinux_system(self): """ Checks and sets self.selinux = True if SELinux is available on system. """ - if self.selinux == None: - if shellutil.run("which getenforce", chk_err=False) == 0: + if self.selinux == None: # pylint: disable=C0121 + if shellutil.run("which getenforce", chk_err=False) == 0: # pylint: disable=simplifiable-if-statement self.selinux = True else: self.selinux = False @@ -536,13 +539,13 @@ def is_selinux_enforcing(self): """ Calls shell command 'getenforce' and returns True if 'Enforcing'. """ - if self.is_selinux_system(): + if self.is_selinux_system(): # pylint: disable=R1705 output = shellutil.run_get_output("getenforce")[1] return output.startswith("Enforcing") else: return False - def set_selinux_context(self, path, con): + def set_selinux_context(self, path, con): # pylint: disable=R1710 """ Calls shell 'chcon' with 'path' and 'con' context. Returns exit result. @@ -576,7 +579,7 @@ def get_dvd_device(self, dev_dir='/dev'): raise OSUtilError(msg="Failed to get dvd device from {0}".format(dev_dir), inner=inner_detail) - def mount_dvd(self, + def mount_dvd(self, # pylint: disable=R0913 max_retry=6, chk_err=True, dvd_device=None, @@ -603,7 +606,7 @@ def mount_dvd(self, mount_point, option="-o ro -t udf,iso9660", chk_err=False) - if return_code == 0: + if return_code == 0: # pylint: disable=R1705 logger.info("Successfully mounted dvd") return else: @@ -622,7 +625,7 @@ def umount_dvd(self, chk_err=True, mount_point=None): mount_point = conf.get_dvd_mount_point() return_code = self.umount(mount_point, chk_err=chk_err) if chk_err and return_code != 0: - raise OSUtilError("Failed to unmount dvd device at {0}", + raise OSUtilError("Failed to unmount dvd device at {0}", # pylint: disable=W0715 mount_point) def eject_dvd(self, chk_err=True): @@ -634,7 +637,7 @@ def eject_dvd(self, chk_err=True): def try_load_atapiix_mod(self): try: self.load_atapiix_mod() - except Exception as e: + except Exception as e: # pylint: disable=C0103 logger.warn("Could not load ATAPI driver: {0}".format(e)) def load_atapiix_mod(self): @@ -649,7 +652,7 @@ def load_atapiix_mod(self): if not os.path.isfile(mod_path): raise Exception("Can't find module file:{0}".format(mod_path)) - ret, output = shellutil.run_get_output("insmod " + mod_path) + ret, output = shellutil.run_get_output("insmod " + mod_path) # pylint: disable=W0612 if ret != 0: raise Exception("Error calling insmod for ATAPI CD-ROM driver") if not self.is_atapiix_mod_loaded(max_retry=3): @@ -684,7 +687,7 @@ def allow_dhcp_broadcast(self): shellutil.run("iptables -I INPUT -p udp --dport 68 -j ACCEPT", chk_err=False) - def remove_rules_files(self, rules_files=__RULES_FILES__): + def remove_rules_files(self, rules_files=__RULES_FILES__): # pylint: disable=W0102 lib_dir = conf.get_lib_dir() for src in rules_files: file_name = fileutil.base_name(src) @@ -695,7 +698,7 @@ def remove_rules_files(self, rules_files=__RULES_FILES__): logger.warn("Move rules file {0} to {1}", file_name, dest) shutil.move(src, dest) - def restore_rules_files(self, rules_files=__RULES_FILES__): + def restore_rules_files(self, rules_files=__RULES_FILES__): # pylint: disable=W0102 lib_dir = conf.get_lib_dir() for dest in rules_files: filename = fileutil.base_name(dest) @@ -763,7 +766,7 @@ def _get_all_interfaces(self): ifaces = {} for i in range(0, array_size, struct_size): iface = ifconf_buff[i:i+IFNAMSIZ].split(b'\0', 1)[0] - if len(iface) > 0: + if len(iface) > 0: # pylint: disable=len-as-condition iface_name = iface.decode('latin-1') if iface_name not in ifaces: ifaces[iface_name] = socket.inet_ntoa(ifconf_buff[i+20:i+24]) @@ -780,7 +783,7 @@ def get_first_if(self): if primary in ifaces: return primary, ifaces[primary] - for iface_name in ifaces.keys(): + for iface_name in ifaces.keys(): # pylint: disable=C0201 if not self.is_loopback(iface_name): logger.info("Choosing non-primary [{0}]".format(iface_name)) return iface_name, ifaces[iface_name] @@ -788,7 +791,7 @@ def get_first_if(self): return '', '' @staticmethod - def _build_route_list(proc_net_route): + def _build_route_list(proc_net_route): # pylint: disable=R0914 """ Construct a list of network route entries :param list(str) proc_net_route: Route table lines, including headers, containing at least one route @@ -816,9 +819,9 @@ def _build_route_list(proc_net_route): route_list = [] for entry in proc_net_route[1:]: route = entry.split("\t") - if len(route) > 0: + if len(route) > 0: # pylint: disable=len-as-condition route_obj = RouteEntry(route[idx_iface], route[idx_dest], route[idx_gw], route[idx_mask], - route[idx_flags], route[idx_metric]) + route[idx_flags], route[idx_metric]) # pylint: disable=C0330 route_list.append(route_obj) return route_list @@ -834,7 +837,7 @@ def read_route_table(): try: with open('/proc/net/route') as routing_table: return list(map(str.strip, routing_table.readlines())) - except Exception as e: + except Exception as e: # pylint: disable=C0103 logger.error("Cannot read route table [{0}]", ustr(e)) return [] @@ -867,8 +870,8 @@ def get_primary_interface(self): :return: the interface which has the default route """ # from linux/route.h - RTF_GATEWAY = 0x02 - DEFAULT_DEST = "00000000" + RTF_GATEWAY = 0x02 # pylint: disable=C0103 + DEFAULT_DEST = "00000000" # pylint: disable=C0103 primary_interface = None @@ -882,7 +885,7 @@ def is_default(route): candidates = list(filter(is_default, DefaultOSUtil.get_list_of_routes(route_table))) - if len(candidates) > 0: + if len(candidates) > 0: # pylint: disable=len-as-condition def get_metric(route): return int(route.metric) primary_route = min(candidates, key=get_metric) @@ -915,7 +918,7 @@ def is_loopback(self, ifname): """ Determine if a named interface is loopback. """ - s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM, socket.IPPROTO_UDP) + s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM, socket.IPPROTO_UDP) # pylint: disable=C0103 ifname_buff = ifname + ('\0'*256) result = fcntl.ioctl(s.fileno(), IOCTL_SIOCGIFFLAGS, ifname_buff) flags, = struct.unpack('H', result[16:18]) @@ -936,7 +939,7 @@ def get_dhcp_lease_endpoint(self): return None @staticmethod - def get_endpoint_from_leases_path(pathglob): + def get_endpoint_from_leases_path(pathglob): # pylint: disable=R0912,R0914 """ Try to discover and decode the wireserver endpoint in the specified dhcp leases path. @@ -945,15 +948,15 @@ def get_endpoint_from_leases_path(pathglob): """ endpoint = None - HEADER_LEASE = "lease" - HEADER_OPTION_245 = "option unknown-245" - HEADER_EXPIRE = "expire" - FOOTER_LEASE = "}" - FORMAT_DATETIME = "%Y/%m/%d %H:%M:%S" + HEADER_LEASE = "lease" # pylint: disable=C0103 + HEADER_OPTION_245 = "option unknown-245" # pylint: disable=C0103 + HEADER_EXPIRE = "expire" # pylint: disable=C0103 + FOOTER_LEASE = "}" # pylint: disable=C0103 + FORMAT_DATETIME = "%Y/%m/%d %H:%M:%S" # pylint: disable=C0103 option_245_re = re.compile(r'\s*option\s+unknown-245\s+([0-9a-fA-F]+):([0-9a-fA-F]+):([0-9a-fA-F]+):([0-9a-fA-F]+);') logger.info("looking for leases in path [{0}]".format(pathglob)) - for lease_file in glob.glob(pathglob): + for lease_file in glob.glob(pathglob): # pylint: disable=R1702 leases = open(lease_file).read() if HEADER_OPTION_245 in leases: cached_endpoint = None @@ -972,7 +975,7 @@ def get_endpoint_from_leases_path(pathglob): expire_date = datetime.datetime.strptime(expire_string, FORMAT_DATETIME) if expire_date > datetime.datetime.utcnow(): expired = False - except: + except: # pylint: disable=W0702 logger.error("could not parse expiry token '{0}'".format(line)) elif FOOTER_LEASE in line: logger.info("dhcp entry:{0}, 245:{1}, expired:{2}".format( @@ -1004,7 +1007,7 @@ def is_missing_default_route(self): if route.startswith("0.0.0.0 ") or route.startswith("default "): return False return True - except CommandError as e: + except CommandError as e: # pylint: disable=C0103 logger.warn("Cannot get the routing table. {0} failed: {1}", ustr(route_cmd), ustr(e)) return False @@ -1065,7 +1068,7 @@ def unregister_agent_service(self): def restart_ssh_service(self): pass - def route_add(self, net, mask, gateway): + def route_add(self, net, mask, gateway): # pylint: disable=W0613 """ Add specified route """ @@ -1083,7 +1086,7 @@ def _text_to_pid_list(text): def _get_dhcp_pid(command): try: return DefaultOSUtil._text_to_pid_list(shellutil.run_command(command)) - except CommandError as exception: + except CommandError as exception: # pylint: disable=W0612 return [] def get_dhcp_pid(self): @@ -1194,10 +1197,10 @@ def search_for_resource_disk(gen1_device_prefix, gen2_device_id): device = None # We have to try device IDs for both Gen1 and Gen2 VMs. logger.info('Searching gen1 prefix {0} or gen2 {1}'.format(gen1_device_prefix, gen2_device_id)) - try: + try: # pylint: disable=R1702 for vmbus, guid in DefaultOSUtil._enumerate_device_id(): if guid.startswith(gen1_device_prefix) or guid == gen2_device_id: - for root, dirs, files in os.walk(STORAGE_DEVICE_PATH + vmbus): + for root, dirs, files in os.walk(STORAGE_DEVICE_PATH + vmbus): # pylint: disable=W0612 root_path_parts = root.split('/') # For Gen1 VMs we only have to check for the block dir in the # current device. But for Gen2 VMs all of the disks (sda, sdb, @@ -1206,14 +1209,14 @@ def search_for_resource_disk(gen1_device_prefix, gen2_device_id): # 0 - OS disk # 1 - Resource disk # 2 - CDROM - if root_path_parts[-1] == 'block' and ( + if root_path_parts[-1] == 'block' and ( # pylint: disable=R1705 guid != gen2_device_id or root_path_parts[-2].split(':')[-1] == '1'): device = dirs[0] return device else: # older distros - for d in dirs: + for d in dirs: # pylint: disable=C0103 if ':' in d and "block" == d.split(':')[0]: device = d.split(':')[1] return device @@ -1227,9 +1230,9 @@ def device_for_ide_port(self, port_id): """ if port_id > 3: return None - g0 = "00000000" + g0 = "00000000" # pylint: disable=C0103 if port_id > 1: - g0 = "00000001" + g0 = "00000001" # pylint: disable=C0103 port_id = port_id - 2 gen1_device_prefix = '{0}-000{1}'.format(g0, port_id) @@ -1280,7 +1283,7 @@ def check_pid_alive(self, pid): os.kill(pid, 0) except (ValueError, TypeError): return False - except OSError as e: + except OSError as e: # pylint: disable=C0103 if e.errno == errno.EPERM: return True return False @@ -1336,12 +1339,15 @@ def get_nic_state(self): state = {} status, output = shellutil.run_get_output("ip -a -o link", chk_err=False, log_cmd=False) + # pylint: disable=W1401 + # pylint: disable=W0105 """ 1: lo: mtu 65536 qdisc noqueue state UNKNOWN mode DEFAULT group default qlen 1000\ link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00 promiscuity 0 addrgenmode eui64 2: eth0: mtu 1500 qdisc mq state UP mode DEFAULT group default qlen 1000\ link/ether 00:0d:3a:30:c3:5a brd ff:ff:ff:ff:ff:ff promiscuity 0 addrgenmode eui64 3: docker0: mtu 1500 qdisc noqueue state DOWN mode DEFAULT group default \ link/ether 02:42:b5:d5:00:1d brd ff:ff:ff:ff:ff:ff promiscuity 0 \ bridge forward_delay 1500 hello_time 200 max_age 2000 ageing_time 30000 stp_state 0 priority 32768 vlan_filtering 0 vlan_protocol 802.1Q addrgenmode eui64 """ + # pylint: enable=W0105 if status != 0: logger.verbose("Could not fetch NIC link info; status {0}, {1}".format(status, output)) return {} @@ -1353,17 +1359,21 @@ def get_nic_state(self): state[name] = NetworkInterfaceCard(name, result.group(2)) self._update_nic_state(state, "ip -4 -a -o address", NetworkInterfaceCard.add_ipv4, "an IPv4 address") - """ + # pylint: disable=W0105 + """ # pylint: disable=W1401 1: lo inet 127.0.0.1/8 scope host lo\ valid_lft forever preferred_lft forever 2: eth0 inet 10.145.187.220/26 brd 10.145.187.255 scope global eth0\ valid_lft forever preferred_lft forever 3: docker0 inet 192.168.43.1/24 brd 192.168.43.255 scope global docker0\ valid_lft forever preferred_lft forever """ + # pylint: enable=W0105 self._update_nic_state(state, "ip -6 -a -o address", NetworkInterfaceCard.add_ipv6, "an IPv6 address") - """ + # pylint: disable=W0105 + """ # pylint: disable=W1401 1: lo inet6 ::1/128 scope host \ valid_lft forever preferred_lft forever 2: eth0 inet6 fe80::20d:3aff:fe30:c35a/64 scope link \ valid_lft forever preferred_lft forever - """ + """ + # pylint: enable=W0105 return state @@ -1409,12 +1419,12 @@ def _run_multiple_commands_without_raising(commands, log_error=True, continue_on break @staticmethod - def _run_command_raising_OSUtilError(cmd, err_msg, cmd_input=None): + def _run_command_raising_OSUtilError(cmd, err_msg, cmd_input=None): # pylint: disable=C0103 # This method runs shell command using the new secure shellutil.run_command and raises OSUtilErrors on failures. try: return shellutil.run_command(cmd, log_error=True, cmd_input=cmd_input) - except shellutil.CommandError as e: + except shellutil.CommandError as e: # pylint: disable=C0103 raise OSUtilError( "{0}, Retcode: {1}, Output: {2}, Error: {3}".format(err_msg, e.returncode, e.stdout, e.stderr)) - except Exception as e: + except Exception as e: # pylint: disable=C0103 raise OSUtilError("{0}, Retcode: {1}, Error: {2}".format(err_msg, -1, ustr(e))) diff --git a/azurelinuxagent/common/osutil/factory.py b/azurelinuxagent/common/osutil/factory.py index 69f8430052..dd2029ab56 100644 --- a/azurelinuxagent/common/osutil/factory.py +++ b/azurelinuxagent/common/osutil/factory.py @@ -16,7 +16,7 @@ # -from distutils.version import LooseVersion as Version +from distutils.version import LooseVersion as Version # pylint: disable=no-name-in-module, import-error import azurelinuxagent.common.logger as logger from azurelinuxagent.common.version import DISTRO_NAME, DISTRO_CODE_NAME, DISTRO_VERSION, DISTRO_FULL_NAME @@ -50,7 +50,7 @@ def get_osutil(distro_name=DISTRO_NAME, return _get_osutil(distro_name, distro_code_name, distro_version, distro_full_name) -def _get_osutil(distro_name, distro_code_name, distro_version, distro_full_name): +def _get_osutil(distro_name, distro_code_name, distro_version, distro_full_name): # pylint: disable=R0912,R0911 if distro_name == "arch": return ArchUtil() @@ -59,7 +59,7 @@ def _get_osutil(distro_name, distro_code_name, distro_version, distro_full_name) return ClearLinuxUtil() if distro_name == "ubuntu": - if Version(distro_version) in [Version("12.04"), Version("12.10")]: + if Version(distro_version) in [Version("12.04"), Version("12.10")]: # pylint: disable=R1705 return Ubuntu12OSUtil() elif Version(distro_version) in [Version("14.04"), Version("14.10")]: return Ubuntu14OSUtil() @@ -84,6 +84,7 @@ def _get_osutil(distro_name, distro_code_name, distro_version, distro_full_name) return CoreOSUtil() if distro_name in ("suse", "sles", "opensuse"): + # pylint: disable=R1705 if distro_full_name == 'SUSE Linux Enterprise Server' \ and Version(distro_version) < Version('12') \ or distro_full_name == 'openSUSE' and Version(distro_version) < Version('13.2'): @@ -92,15 +93,16 @@ def _get_osutil(distro_name, distro_code_name, distro_version, distro_full_name) return SUSEOSUtil() if distro_name == "debian": - if "sid" in distro_version or Version(distro_version) > Version("7"): + if "sid" in distro_version or Version(distro_version) > Version("7"): # pylint: disable=R1705 return DebianOSModernUtil() else: return DebianOSBaseUtil() + # pylint: disable=R1714 if distro_name == "redhat" \ or distro_name == "centos" \ or distro_name == "oracle": - if Version(distro_version) < Version("7"): + if Version(distro_version) < Version("7"): # pylint: disable=R1705 return Redhat6xOSUtil() else: return RedhatOSUtil() @@ -126,7 +128,7 @@ def _get_osutil(distro_name, distro_code_name, distro_version, distro_full_name) if distro_name == "nsbsd": return NSBSDOSUtil() - if distro_name == "openwrt": + if distro_name == "openwrt": # pylint: disable=R1705 return OpenWRTOSUtil() else: diff --git a/azurelinuxagent/common/osutil/freebsd.py b/azurelinuxagent/common/osutil/freebsd.py index 0686d3badf..b8c111ead8 100644 --- a/azurelinuxagent/common/osutil/freebsd.py +++ b/azurelinuxagent/common/osutil/freebsd.py @@ -22,13 +22,13 @@ import azurelinuxagent.common.utils.fileutil as fileutil import azurelinuxagent.common.utils.shellutil as shellutil import azurelinuxagent.common.utils.textutil as textutil -from azurelinuxagent.common.utils.networkutil import RouteEntry +from azurelinuxagent.common.utils.networkutil import RouteEntry # pylint: disable=W0611 import azurelinuxagent.common.logger as logger from azurelinuxagent.common.exception import OSUtilError from azurelinuxagent.common.osutil.default import DefaultOSUtil from azurelinuxagent.common.future import ustr -class FreeBSDOSUtil(DefaultOSUtil): +class FreeBSDOSUtil(DefaultOSUtil): # pylint: disable=R0904 def __init__(self): super(FreeBSDOSUtil, self).__init__() @@ -127,7 +127,7 @@ def read_route_table(): :rtype: list(str) """ - def _get_netstat_rn_ipv4_routes(): + def _get_netstat_rn_ipv4_routes(): # pylint: disable=too-many-locals """ Runs `netstat -rn -f inet` and parses its output and returns a list of routes where the key is the column name and the value is the value in the column, stripped of leading and trailing whitespace. @@ -237,10 +237,10 @@ def _ascii_route_flags_to_bitmask(ascii_route_flags): :rtype: int """ bitmask_flags = 0 - RTF_UP = 0x0001 - RTF_GATEWAY = 0x0002 - RTF_HOST = 0x0004 - RTF_DYNAMIC = 0x0010 + RTF_UP = 0x0001 # pylint: disable=C0103 + RTF_GATEWAY = 0x0002 # pylint: disable=C0103 + RTF_HOST = 0x0004 # pylint: disable=C0103 + RTF_DYNAMIC = 0x0010 # pylint: disable=C0103 if "U" in ascii_route_flags: bitmask_flags |= RTF_UP if "G" in ascii_route_flags: @@ -291,7 +291,7 @@ def _freebsd_netstat_rn_route_to_linux_proc_net_route(netstat_route): netstat_routes = _get_netstat_rn_ipv4_routes() # Make sure the `netstat -rn -f inet` contains columns for Netif, Destination, Gateway and Flags which are needed to convert # to the Linux Format - if len(netstat_routes) > 0: + if len(netstat_routes) > 0: # pylint: disable=len-as-condition missing_headers = [] if "Netif" not in netstat_routes[0]: missing_headers.append("Netif") @@ -311,7 +311,7 @@ def _freebsd_netstat_rn_route_to_linux_proc_net_route(netstat_route): except Exception: # Skip the route continue - except Exception as e: + except Exception as e: # pylint: disable=C0103 logger.error("Cannot read route table [{0}]", ustr(e)) return linux_style_route_file @@ -342,8 +342,8 @@ def get_primary_interface(self): the primary has the lowest Metric. :return: the interface which has the default route """ - RTF_GATEWAY = 0x0002 - DEFAULT_DEST = "00000000" + RTF_GATEWAY = 0x0002 # pylint: disable=C0103 + DEFAULT_DEST = "00000000" # pylint: disable=C0103 primary_interface = None @@ -357,7 +357,7 @@ def is_default(route): candidates = list(filter(is_default, self.get_list_of_routes(route_table))) - if len(candidates) > 0: + if len(candidates) > 0: # pylint: disable=len-as-condition def get_metric(route): return int(route.metric) primary_route = min(candidates, key=get_metric) @@ -399,14 +399,14 @@ def is_missing_default_route(self): specify the route manually to get it work in a VNET environment. SEE ALSO: man ip(4) IP_ONESBCAST, """ - RTF_GATEWAY = 0x0002 - DEFAULT_DEST = "00000000" + RTF_GATEWAY = 0x0002 # pylint: disable=C0103 + DEFAULT_DEST = "00000000" # pylint: disable=C0103 route_table = self.read_route_table() routes = self.get_list_of_routes(route_table) for route in routes: if (route.destination == DEFAULT_DEST) and (RTF_GATEWAY & route.flags): - return False + return False # pylint: disable=W0311 return True def is_dhcp_enabled(self): @@ -433,7 +433,7 @@ def eject_dvd(self, chk_err=True): if chk_err and retcode != 0: raise OSUtilError("Failed to eject dvd: ret={0}".format(retcode)) - def restart_if(self, ifname): + def restart_if(self, ifname): # pylint: disable=W0221 # Restart dhclient only to publish hostname shellutil.run("/etc/rc.d/dhclient restart {0}".format(ifname), chk_err=False) @@ -509,33 +509,39 @@ def device_for_ide_port(self, port_id): """ if port_id > 3: return None - g0 = "00000000" + g0 = "00000000" # pylint: disable=C0103 if port_id > 1: - g0 = "00000001" + g0 = "00000001" # pylint: disable=C0103 port_id = port_id - 2 err, output = shellutil.run_get_output('sysctl dev.storvsc | grep pnpinfo | grep deviceid=') if err: return None - g1 = "000" + ustr(port_id) + g1 = "000" + ustr(port_id) # pylint: disable=C0103 g0g1 = "{0}-{1}".format(g0, g1) + + # pylint: disable=W0105 """ search 'X' from 'dev.storvsc.X.%pnpinfo: classid=32412632-86cb-44a2-9b5c-50d1417354f5 deviceid=00000000-0001-8899-0000-000000000000' """ + # pylint: enable=W0105 + cmd_search_ide = "sysctl dev.storvsc | grep pnpinfo | grep deviceid={0}".format(g0g1) err, output = shellutil.run_get_output(cmd_search_ide) if err: return None cmd_extract_id = cmd_search_ide + "|awk -F . '{print $3}'" err, output = shellutil.run_get_output(cmd_extract_id) + # pylint: disable=W0105 """ try to search 'blkvscX' and 'storvscX' to find device name - """ + """ + # pylint: enable=W0105 output = output.rstrip() cmd_search_blkvsc = "camcontrol devlist -b | grep blkvsc{0} | awk '{{print $1}}'".format(output) err, output = shellutil.run_get_output(cmd_search_blkvsc) if err == 0: output = output.rstrip() - cmd_search_dev="camcontrol devlist | grep {0} | awk -F \( '{{print $2}}'|sed -e 's/.*(//'| sed -e 's/).*//'".format(output) + cmd_search_dev="camcontrol devlist | grep {0} | awk -F \( '{{print $2}}'|sed -e 's/.*(//'| sed -e 's/).*//'".format(output) # pylint: disable=W1401 err, output = shellutil.run_get_output(cmd_search_dev) if err == 0: for possible in output.rstrip().split(','): @@ -546,7 +552,7 @@ def device_for_ide_port(self, port_id): err, output = shellutil.run_get_output(cmd_search_storvsc) if err == 0: output = output.rstrip() - cmd_search_dev="camcontrol devlist | grep {0} | awk -F \( '{{print $2}}'|sed -e 's/.*(//'| sed -e 's/).*//'".format(output) + cmd_search_dev="camcontrol devlist | grep {0} | awk -F \( '{{print $2}}'|sed -e 's/.*(//'| sed -e 's/).*//'".format(output) # pylint: disable=W1401 err, output = shellutil.run_get_output(cmd_search_dev) if err == 0: for possible in output.rstrip().split(','): diff --git a/azurelinuxagent/common/osutil/gaia.py b/azurelinuxagent/common/osutil/gaia.py index 943032c7e6..20d80a911c 100644 --- a/azurelinuxagent/common/osutil/gaia.py +++ b/azurelinuxagent/common/osutil/gaia.py @@ -34,22 +34,22 @@ class GaiaOSUtil(DefaultOSUtil): - def __init__(self): + def __init__(self): # pylint: disable=W0235 super(GaiaOSUtil, self).__init__() def _run_clish(self, cmd): ret = 0 out = "" - for i in xrange(10): + for i in xrange(10): # pylint: disable=E0602,W0612 try: final_command = ["/bin/clish", "-s", "-c", "'{0}'".format(cmd)] out = shellutil.run_command(final_command, log_error=True) ret = 0 break - except shellutil.CommandError as e: + except shellutil.CommandError as e: # pylint: disable=C0103 ret = e.returncode out = e.stdout - except Exception as e: + except Exception as e: # pylint: disable=C0103 ret = -1 out = ustr(e) @@ -59,7 +59,7 @@ def _run_clish(self, cmd): time.sleep(2) return ret, out - def useradd(self, username, expiration=None): + def useradd(self, username, expiration=None): # pylint: disable=W0221 logger.warn('useradd is not supported on GAiA') def chpasswd(self, username, password, crypt_id=6, salt_len=10): @@ -76,7 +76,7 @@ def conf_sudoer(self, username, nopasswd=False, remove=False): def del_root_password(self): logger.info('del_root_password') - ret, out = self._run_clish('set user admin password-hash *LOCK*') + ret, out = self._run_clish('set user admin password-hash *LOCK*') # pylint: disable=W0612 if ret != 0: raise OSUtilError("Failed to delete root password") @@ -96,6 +96,7 @@ def deploy_ssh_keypair(self, username, keypair): username, (path, thumbprint)) def openssl_to_openssh(self, input_file, output_file): + # pylint: disable=too-many-locals cryptutil = CryptUtil(conf.get_openssl_cmd()) ret, out = shellutil.run_get_output( conf.get_openssl_cmd() + @@ -121,10 +122,10 @@ def openssl_to_openssh(self, input_file, output_file): def text_to_num(buf): if len(buf) == 1: return int(buf[0].split()[1]) - return long(''.join(buf[1:]), 16) + return long(''.join(buf[1:]), 16) # pylint: disable=E0602 - n = text_to_num(modulus) - e = text_to_num(exponent) + n = text_to_num(modulus) # pylint: disable=C0103 + e = text_to_num(exponent) # pylint: disable=C0103 keydata = bytearray() keydata.extend(struct.pack('>I', len('ssh-rsa'))) @@ -188,7 +189,7 @@ def route_add(self, net, mask, gateway): cidr = self._address_to_string(net) + '/' + self._get_prefix( self._address_to_string(mask)) - ret, out = self._run_clish( + ret, out = self._run_clish( # pylint: disable=W0612 'set static-route ' + cidr + ' nexthop gateway address ' + self._address_to_string(gateway) + ' on') diff --git a/azurelinuxagent/common/osutil/iosxe.py b/azurelinuxagent/common/osutil/iosxe.py index c395851afd..0c49484072 100644 --- a/azurelinuxagent/common/osutil/iosxe.py +++ b/azurelinuxagent/common/osutil/iosxe.py @@ -21,9 +21,10 @@ import azurelinuxagent.common.utils.shellutil as shellutil from azurelinuxagent.common.future import ustr from azurelinuxagent.common.osutil.default import DefaultOSUtil -from azurelinuxagent.common.osutil.redhat import Redhat6xOSUtil -from azurelinuxagent.common.utils import textutil +from azurelinuxagent.common.osutil.redhat import Redhat6xOSUtil # pylint: disable=W0611 +from azurelinuxagent.common.utils import textutil # pylint: disable=W0611 +# pylint: disable=W0105 ''' The IOSXE distribution is a variant of the Centos distribution, version 7.1. @@ -33,8 +34,10 @@ - no provisioning is performed - no DHCP-based services are available ''' +# pylint: enable=W0105 + class IosxeOSUtil(DefaultOSUtil): - def __init__(self): + def __init__(self): # pylint: disable=W0235 super(IosxeOSUtil, self).__init__() def set_hostname(self, hostname): @@ -46,7 +49,7 @@ def set_hostname(self, hostname): hostnamectl_cmd = ["hostnamectl", "set-hostname", hostname, "--static"] try: shellutil.run_command(hostnamectl_cmd) - except Exception as e: + except Exception as e: # pylint: disable=C0103 logger.warn("[{0}] failed with error: {1}, attempting fallback".format(' '.join(hostnamectl_cmd), ustr(e))) DefaultOSUtil.set_hostname(self, hostname) @@ -55,7 +58,7 @@ def publish_hostname(self, hostname): Restart NetworkManager first before publishing hostname """ shellutil.run("service NetworkManager restart") - super(RedhatOSUtil, self).publish_hostname(hostname) + super(RedhatOSUtil, self).publish_hostname(hostname) # pylint: disable=E0602 def register_agent_service(self): return shellutil.run("systemctl enable waagent", chk_err=False) @@ -76,13 +79,13 @@ def get_instance_id(self): If that is missing, then extracts from dmidecode If nothing works (for old VMs), return the empty string ''' - if os.path.isfile(PRODUCT_ID_FILE): + if os.path.isfile(PRODUCT_ID_FILE): # pylint: disable=E0602 try: - s = fileutil.read_file(PRODUCT_ID_FILE).strip() + s = fileutil.read_file(PRODUCT_ID_FILE).strip() # pylint: disable=E0602,C0103 return self._correct_instance_id(s.strip()) except IOError: pass - rc, s = shellutil.run_get_output(DMIDECODE_CMD) - if rc != 0 or UUID_PATTERN.match(s) is None: + rc, s = shellutil.run_get_output(DMIDECODE_CMD) # pylint: disable=E0602,C0103 + if rc != 0 or UUID_PATTERN.match(s) is None: # pylint: disable=E0602 return "" return self._correct_instance_id(s.strip()) diff --git a/azurelinuxagent/common/osutil/nsbsd.py b/azurelinuxagent/common/osutil/nsbsd.py index 4130da4381..17795beffc 100644 --- a/azurelinuxagent/common/osutil/nsbsd.py +++ b/azurelinuxagent/common/osutil/nsbsd.py @@ -19,7 +19,7 @@ import azurelinuxagent.common.logger as logger from azurelinuxagent.common.exception import OSUtilError from azurelinuxagent.common.osutil.freebsd import FreeBSDOSUtil -import os +import os # pylint: disable=C0411 class NSBSDOSUtil(FreeBSDOSUtil): @@ -39,13 +39,13 @@ def __init__(self): self.resolver = dns.resolver.Resolver() servers = [] cmd = "getconf /usr/Firewall/ConfigFiles/dns Servers | tail -n +2" - ret, output = shellutil.run_get_output(cmd) + ret, output = shellutil.run_get_output(cmd) # pylint: disable=W0612 for server in output.split("\n"): if server == '': break server = server[:-1] # remove last '=' cmd = "grep '{}' /etc/hosts".format(server) + " | awk '{print $1}'" - ret, ip = shellutil.run_get_output(cmd) + ret, ip = shellutil.run_get_output(cmd) # pylint: disable=C0103 servers.append(ip) self.resolver.nameservers = servers dns.resolver.override_system_resolver(self.resolver) @@ -72,7 +72,7 @@ def conf_sshd(self, disable_password): logger.info("{0} SSH password-based authentication methods." .format("Disabled" if disable_password else "Enabled")) - def useradd(self, username, expiration=None): + def useradd(self, username, expiration=None): # pylint: disable=R1711,W0221 """ Create user account with 'username' """ @@ -98,11 +98,11 @@ def deploy_ssh_pubkey(self, username, pubkey): """ Deploy authorized_key """ - path, thumbprint, value = pubkey + path, thumbprint, value = pubkey # pylint: disable=W0612 #overide parameters super(NSBSDOSUtil, self).deploy_ssh_pubkey('admin', - ["/usr/Firewall/.ssh/authorized_keys", thumbprint, value]) + ["/usr/Firewall/.ssh/authorized_keys", thumbprint, value]) # pylint: disable=C0330 def del_root_password(self): logger.warn("Root password deletion disabled") diff --git a/azurelinuxagent/common/osutil/openbsd.py b/azurelinuxagent/common/osutil/openbsd.py index 37fae5c873..ad2e933885 100644 --- a/azurelinuxagent/common/osutil/openbsd.py +++ b/azurelinuxagent/common/osutil/openbsd.py @@ -36,7 +36,7 @@ re.IGNORECASE) -class OpenBSDOSUtil(DefaultOSUtil): +class OpenBSDOSUtil(DefaultOSUtil): # pylint: disable=R0904 def __init__(self): super(OpenBSDOSUtil, self).__init__() @@ -146,21 +146,21 @@ def start_dhcp_service(self): def stop_dhcp_service(self): pass - def get_dhcp_lease_endpoint(self): + def get_dhcp_lease_endpoint(self): # pylint: disable=R0912,R0914 """ OpenBSD has a sligthly different lease file format. """ endpoint = None pathglob = '/var/db/dhclient.leases.{}'.format(self.get_first_if()[0]) - HEADER_LEASE = "lease" - HEADER_OPTION = "option option-245" - HEADER_EXPIRE = "expire" - FOOTER_LEASE = "}" - FORMAT_DATETIME = "%Y/%m/%d %H:%M:%S %Z" + HEADER_LEASE = "lease" # pylint: disable=C0103 + HEADER_OPTION = "option option-245" # pylint: disable=C0103 + HEADER_EXPIRE = "expire" # pylint: disable=C0103 + FOOTER_LEASE = "}" # pylint: disable=C0103 + FORMAT_DATETIME = "%Y/%m/%d %H:%M:%S %Z" # pylint: disable=C0103 logger.info("looking for leases in path [{0}]".format(pathglob)) - for lease_file in glob.glob(pathglob): + for lease_file in glob.glob(pathglob): # pylint: disable=R1702 leases = open(lease_file).read() if HEADER_OPTION in leases: cached_endpoint = None @@ -228,7 +228,7 @@ def get_dvd_device(self, dev_dir='/dev'): return "/dev/{0}".format(dvd.group(0)) raise OSUtilError("Failed to get DVD device") - def mount_dvd(self, + def mount_dvd(self, # pylint: disable=R0913 max_retry=6, chk_err=True, dvd_device=None, @@ -243,9 +243,9 @@ def mount_dvd(self, for retry in range(0, max_retry): retcode = self.mount(dvd_device, - mount_point, - option="-o ro -t udf", - chk_err=False) + mount_point, # pylint: disable=C0330 + option="-o ro -t udf", # pylint: disable=C0330 + chk_err=False) # pylint: disable=C0330 if retcode == 0: logger.info("Successfully mounted DVD") return @@ -293,7 +293,7 @@ def get_processor_cores(self): def set_scsi_disks_timeout(self, timeout): pass - def check_pid_alive(self, pid): + def check_pid_alive(self, pid): # pylint: disable=R1710 if not pid: return return shellutil.run('ps -p {0}'.format(pid), chk_err=False) == 0 diff --git a/azurelinuxagent/common/osutil/openwrt.py b/azurelinuxagent/common/osutil/openwrt.py index 0421da74de..d070878aad 100644 --- a/azurelinuxagent/common/osutil/openwrt.py +++ b/azurelinuxagent/common/osutil/openwrt.py @@ -31,7 +31,7 @@ def __init__(self): super(OpenWRTOSUtil, self).__init__() self.agent_conf_file_path = '/etc/waagent.conf' self.dhclient_name = 'udhcpc' - self.ip_command_output = re.compile('^\d+:\s+(\w+):\s+(.*)$') + self.ip_command_output = re.compile('^\d+:\s+(\w+):\s+(.*)$') # pylint: disable=W1401 self.jit_enabled = True def eject_dvd(self, chk_err=True): @@ -121,12 +121,12 @@ def stop_dhcp_service(self): def start_network(self) : return shellutil.run("/etc/init.d/network start", chk_err=True) - def restart_ssh_service(self): + def restart_ssh_service(self): # pylint: disable=R1710 # Since Dropbear is the default ssh server on OpenWRt, lets do a sanity check - if os.path.exists("/etc/init.d/sshd"): + if os.path.exists("/etc/init.d/sshd"): # pylint: disable=R1705 return shellutil.run("/etc/init.d/sshd restart", chk_err=True) else: - logger.warn("sshd service does not exists", username) + logger.warn("sshd service does not exists", username) # pylint: disable=E0602 def stop_agent_service(self): return shellutil.run("/etc/init.d/{0} stop".format(self.service_name), chk_err=True) diff --git a/azurelinuxagent/common/osutil/redhat.py b/azurelinuxagent/common/osutil/redhat.py index 8974eccb88..243833ce80 100644 --- a/azurelinuxagent/common/osutil/redhat.py +++ b/azurelinuxagent/common/osutil/redhat.py @@ -16,23 +16,23 @@ # Requires Python 2.6+ and Openssl 1.0+ # -import os -import re -import pwd -import shutil -import socket -import array -import struct -import fcntl -import time -import base64 +import os # pylint: disable=W0611 +import re # pylint: disable=W0611 +import pwd # pylint: disable=W0611 +import shutil # pylint: disable=W0611 +import socket # pylint: disable=W0611 +import array # pylint: disable=W0611 +import struct # pylint: disable=W0611 +import fcntl # pylint: disable=W0611 +import time # pylint: disable=W0611 +import base64 # pylint: disable=W0611 import azurelinuxagent.common.conf as conf import azurelinuxagent.common.logger as logger -from azurelinuxagent.common.future import ustr, bytebuffer +from azurelinuxagent.common.future import ustr, bytebuffer # pylint: disable=W0611 from azurelinuxagent.common.exception import OSUtilError, CryptError import azurelinuxagent.common.utils.fileutil as fileutil import azurelinuxagent.common.utils.shellutil as shellutil -import azurelinuxagent.common.utils.textutil as textutil +import azurelinuxagent.common.utils.textutil as textutil # pylint: disable=W0611 from azurelinuxagent.common.utils.cryptutil import CryptUtil from azurelinuxagent.common.osutil.default import DefaultOSUtil @@ -66,7 +66,7 @@ def openssl_to_openssh(self, input_file, output_file): try: cryptutil = CryptUtil(conf.get_openssl_cmd()) ssh_rsa_pubkey = cryptutil.asn1_to_ssh(pubkey) - except CryptError as e: + except CryptError as e: # pylint: disable=C0103 raise OSUtilError(ustr(e)) fileutil.append_file(output_file, ssh_rsa_pubkey) diff --git a/azurelinuxagent/common/osutil/suse.py b/azurelinuxagent/common/osutil/suse.py index 2192c67d41..3dbd697e9b 100644 --- a/azurelinuxagent/common/osutil/suse.py +++ b/azurelinuxagent/common/osutil/suse.py @@ -17,9 +17,9 @@ # import azurelinuxagent.common.utils.fileutil as fileutil -import azurelinuxagent.common.utils.shellutil as shellutil -from azurelinuxagent.common.exception import OSUtilError -from azurelinuxagent.common.future import ustr +import azurelinuxagent.common.utils.shellutil as shellutil # pylint: disable=W0611 +from azurelinuxagent.common.exception import OSUtilError # pylint: disable=W0611 +from azurelinuxagent.common.future import ustr # pylint: disable=W0611 from azurelinuxagent.common.osutil.default import DefaultOSUtil diff --git a/azurelinuxagent/common/osutil/ubuntu.py b/azurelinuxagent/common/osutil/ubuntu.py index 02953da93e..eaf4b7ffe7 100644 --- a/azurelinuxagent/common/osutil/ubuntu.py +++ b/azurelinuxagent/common/osutil/ubuntu.py @@ -55,7 +55,7 @@ def get_dhcp_lease_endpoint(self): class Ubuntu12OSUtil(Ubuntu14OSUtil): - def __init__(self): + def __init__(self): # pylint: disable=W0235 super(Ubuntu12OSUtil, self).__init__() # Override @@ -109,7 +109,7 @@ def stop_agent_service(self): class UbuntuOSUtil(Ubuntu16OSUtil): - def __init__(self): + def __init__(self): # pylint: disable=W0235 super(UbuntuOSUtil, self).__init__() def restart_if(self, ifname, retries=3, wait=5): diff --git a/azurelinuxagent/common/protocol/goal_state.py b/azurelinuxagent/common/protocol/goal_state.py index de0124c236..5d5a22b9c1 100644 --- a/azurelinuxagent/common/protocol/goal_state.py +++ b/azurelinuxagent/common/protocol/goal_state.py @@ -42,7 +42,7 @@ TRANSPORT_PRV_FILE_NAME = "TransportPrivate.pem" -class GoalState(object): +class GoalState(object): # pylint: disable=R0902 def __init__(self, wire_client, full_goal_state=False, base_incarnation=None): """ @@ -122,7 +122,7 @@ def __init__(self, wire_client, full_goal_state=False, base_incarnation=None): else: xml_text = wire_client.fetch_config(uri, wire_client.get_header_for_cert()) self.remote_access = RemoteAccess(xml_text) - except Exception as e: + except Exception as e: # pylint: disable=C0103 logger.warn("Fetching the goal state failed: {0}", ustr(e)) raise finally: @@ -151,7 +151,7 @@ def fetch_full_goal_state_if_incarnation_different_than(wire_client, incarnation return goal_state if goal_state.incarnation != incarnation else None -class HostingEnv(object): +class HostingEnv(object): # pylint: disable=R0903 def __init__(self, xml_text): self.xml_text = xml_text xml_doc = parse_doc(xml_text) @@ -163,13 +163,13 @@ def __init__(self, xml_text): self.deployment_name = getattrib(deployment, "name") -class SharedConfig(object): +class SharedConfig(object): # pylint: disable=R0903 def __init__(self, xml_text): self.xml_text = xml_text -class Certificates(object): - def __init__(self, xml_text): +class Certificates(object): # pylint: disable=R0903 + def __init__(self, xml_text): # pylint: disable=R0912,R0914 self.cert_list = CertList() # Save the certificates @@ -183,14 +183,14 @@ def __init__(self, xml_text): return # if the certificates format is not Pkcs7BlobWithPfxContents do not parse it - certificateFormat = findtext(xml_doc, "Format") + certificateFormat = findtext(xml_doc, "Format") # pylint: disable=C0103 if certificateFormat and certificateFormat != "Pkcs7BlobWithPfxContents": logger.warn("The Format is not Pkcs7BlobWithPfxContents. Format is " + certificateFormat) return cryptutil = CryptUtil(conf.get_openssl_cmd()) p7m_file = os.path.join(conf.get_lib_dir(), P7M_FILE_NAME) - p7m = ("MIME-Version:1.0\n" + p7m = ("MIME-Version:1.0\n" # pylint: disable=W1308 "Content-Disposition: attachment; filename=\"{0}\"\n" "Content-Type: application/x-pkcs7-mime; name=\"{1}\"\n" "Content-Transfer-Encoding: base64\n" @@ -207,8 +207,8 @@ def __init__(self, xml_text): # The parsing process use public key to match prv and crt. buf = [] - begin_crt = False - begin_prv = False + begin_crt = False # pylint: disable=W0612 + begin_prv = False # pylint: disable=W0612 prvs = {} thumbprints = {} index = 0 @@ -275,8 +275,8 @@ def _write_to_tmp_file(index, suffix, buf): return file_name -class ExtensionsConfig(object): - def __init__(self, xml_text): +class ExtensionsConfig(object): # pylint: disable=R0903 + def __init__(self, xml_text): # pylint: disable=R0914 self.xml_text = xml_text self.ext_handlers = ExtHandlerList() self.vmagent_manifests = VMAgentManifestList() @@ -299,7 +299,7 @@ def __init__(self, xml_text): manifest = VMAgentManifest() manifest.family = family for uri in uris: - manifestUri = VMAgentManifestUri(uri=gettext(uri)) + manifestUri = VMAgentManifestUri(uri=gettext(uri)) # pylint: disable=C0103 manifest.versionsManifestUris.append(manifestUri) self.vmagent_manifests.vmAgentManifests.append(manifest) @@ -336,7 +336,7 @@ def _parse_plugin(plugin): return ext_handler @staticmethod - def _parse_plugin_settings(ext_handler, plugin_settings): + def _parse_plugin_settings(ext_handler, plugin_settings): # pylint: disable=R0914 if plugin_settings is None: return @@ -344,17 +344,17 @@ def _parse_plugin_settings(ext_handler, plugin_settings): version = ext_handler.properties.version ext_handler_plugin_settings = [x for x in plugin_settings if getattrib(x, "name") == name] - if ext_handler_plugin_settings is None or len(ext_handler_plugin_settings) == 0: + if ext_handler_plugin_settings is None or len(ext_handler_plugin_settings) == 0: # pylint: disable=len-as-condition return settings = [x for x in ext_handler_plugin_settings if getattrib(x, "version") == version] if len(settings) != len(ext_handler_plugin_settings): msg = "ExtHandler PluginSettings Version Mismatch! Expected PluginSettings version: {0} for Handler: " \ "{1} but found versions: ({2})".format(version, name, ', '.join( - set([getattrib(x, "version") for x in ext_handler_plugin_settings]))) + set([getattrib(x, "version") for x in ext_handler_plugin_settings]))) # pylint: disable=C0330 add_event(AGENT_NAME, op=WALAEventOperation.PluginSettingsVersionMismatch, message=msg, log_event=False, is_success=False) - if len(settings) == 0: + if len(settings) == 0: # pylint: disable=len-as-condition # If there is no corresponding settings for the specific extension handler, we will not process it at all, # this is an unexpected error as we always expect both versions to be in sync. logger.error(msg) @@ -363,17 +363,17 @@ def _parse_plugin_settings(ext_handler, plugin_settings): runtime_settings = None runtime_settings_node = find(settings[0], "RuntimeSettings") - seqNo = getattrib(runtime_settings_node, "seqNo") + seqNo = getattrib(runtime_settings_node, "seqNo") # pylint: disable=C0103 runtime_settings_str = gettext(runtime_settings_node) try: runtime_settings = json.loads(runtime_settings_str) - except ValueError as e: + except ValueError as e: # pylint: disable=W0612,C0103 logger.error("Invalid extension settings") return depends_on_level = 0 depends_on_node = find(settings[0], "DependsOn") - if depends_on_node != None: + if depends_on_node != None: # pylint: disable=C0121 try: depends_on_level = int(getattrib(depends_on_node, "dependencyLevel")) except (ValueError, TypeError): @@ -396,7 +396,7 @@ def _parse_plugin_settings(ext_handler, plugin_settings): ext_handler.properties.extensions.append(ext) -class RemoteAccess(object): +class RemoteAccess(object): # pylint: disable=R0903 """ Object containing information about user accounts """ @@ -419,7 +419,7 @@ def __init__(self, xml_text): self.incarnation = None self.user_list = RemoteAccessUsersList() - if self.xml_text is None or len(self.xml_text) == 0: + if self.xml_text is None or len(self.xml_text) == 0: # pylint: disable=len-as-condition return xml_doc = parse_doc(self.xml_text) diff --git a/azurelinuxagent/common/protocol/healthservice.py b/azurelinuxagent/common/protocol/healthservice.py index 3abe7299b9..95d9f68b1a 100644 --- a/azurelinuxagent/common/protocol/healthservice.py +++ b/azurelinuxagent/common/protocol/healthservice.py @@ -26,7 +26,7 @@ from azurelinuxagent.common.version import AGENT_NAME, CURRENT_VERSION -class Observation(object): +class Observation(object): # pylint: disable=R0903 def __init__(self, name, is_healthy, description='', value=''): if name is None: raise ValueError("Observation name must be provided") @@ -154,7 +154,7 @@ def _report(self): try: restutil.http_post(self.endpoint, self.as_json, headers={'Content-Type': 'application/json'}) logger.verbose('HealthService: Reported observations to {0}: {1}', self.endpoint, self.as_json) - except HttpError as e: + except HttpError as e: # pylint: disable=C0103 logger.warn("HealthService: could not report observations: {0}", ustr(e)) finally: # report any failures via telemetry @@ -166,12 +166,12 @@ def _report_failures(self): try: logger.verbose("HealthService: report failures as telemetry") from azurelinuxagent.common.event import add_event, WALAEventOperation - for o in self.observations: + for o in self.observations: # pylint: disable=C0103 if not o.is_healthy: add_event(AGENT_NAME, version=CURRENT_VERSION, op=WALAEventOperation.HealthObservation, is_success=False, message=json.dumps(o.as_obj)) - except Exception as e: + except Exception as e: # pylint: disable=C0103 logger.verbose("HealthService: could not report failures: {0}".format(ustr(e))) diff --git a/azurelinuxagent/common/protocol/hostplugin.py b/azurelinuxagent/common/protocol/hostplugin.py index 5ef2987bde..b609c4dbd4 100644 --- a/azurelinuxagent/common/protocol/hostplugin.py +++ b/azurelinuxagent/common/protocol/hostplugin.py @@ -56,7 +56,7 @@ MAXIMUM_PAGEBLOB_PAGE_SIZE = 4 * 1024 * 1024 # Max page size: 4MB -class HostPluginProtocol(object): +class HostPluginProtocol(object): # pylint: disable=R0902 _is_default_channel = False FETCH_REPORTING_PERIOD = datetime.timedelta(minutes=1) @@ -141,7 +141,7 @@ def get_api_versions(self): else: return_val = ustr(remove_bom(response.read()), encoding='utf-8') is_healthy = True - except HttpError as e: + except HttpError as e: # pylint: disable=C0103 logger.error("HostGAPlugin: Exception Get API versions: {0}".format(e)) self.health_service.report_host_plugin_versions(is_healthy=is_healthy, response=error_response) @@ -232,7 +232,7 @@ def put_vm_log(self, content): data=content, headers=self._build_log_headers()) - if restutil.request_failed(response): + if restutil.request_failed(response): # pylint: disable=R1720 error_response = restutil.read_response_error(response) raise HttpError("HostGAPlugin: Upload VM logs failed: {0}".format(error_response)) else: @@ -271,7 +271,7 @@ def _put_block_blob_status(self, sas_url, status_blob): bytearray(status_blob.data, encoding='utf-8')), headers=self._build_status_headers()) - if restutil.request_failed(response): + if restutil.request_failed(response): # pylint: disable=R1720 error_response = restutil.read_response_error(response) is_healthy = not restutil.request_failed_at_hostplugin(response) self.report_status_health(is_healthy=is_healthy, response=error_response) @@ -296,7 +296,7 @@ def _put_page_blob_status(self, sas_url, status_blob): status_blob.get_page_blob_create_headers(status_size)), headers=self._build_status_headers()) - if restutil.request_failed(response): + if restutil.request_failed(response): # pylint: disable=R1720 error_response = restutil.read_response_error(response) is_healthy = not restutil.request_failed_at_hostplugin(response) self.report_status_health(is_healthy=is_healthy, response=error_response) @@ -375,7 +375,7 @@ def _build_log_headers(self): } def _base64_encode(self, data): - s = base64.b64encode(bytes(data)) + s = base64.b64encode(bytes(data)) # pylint: disable=C0103 if PY_VERSION_MAJOR > 2: return s.decode('utf-8') return s diff --git a/azurelinuxagent/common/protocol/imds.py b/azurelinuxagent/common/protocol/imds.py index a4600ad8d0..fe41f72893 100644 --- a/azurelinuxagent/common/protocol/imds.py +++ b/azurelinuxagent/common/protocol/imds.py @@ -139,7 +139,7 @@ def get_imds_client(wireserver_endpoint): }""" -class ImageInfoMatcher(object): +class ImageInfoMatcher(object): # pylint: disable=R0903 def __init__(self, doc): self.doc = json.loads(doc) @@ -169,10 +169,10 @@ def _is_match_walk(doci, keys): return _is_match_walk(self.doc, [ publisher, offer, sku, version ]) -class ComputeInfo(DataContract): +class ComputeInfo(DataContract): # pylint: disable=R0902 __matcher = ImageInfoMatcher(ENDORSED_IMAGE_INFO_MATCHER_JSON) - def __init__(self, + def __init__(self, # pylint: disable=R0913,R0914 location=None, name=None, offer=None, @@ -193,19 +193,19 @@ def __init__(self, self.location = location self.name = name self.offer = offer - self.osType = osType - self.placementGroupId = placementGroupId - self.platformFaultDomain = platformFaultDomain - self.platformUpdateDomain = placementUpdateDomain + self.osType = osType # pylint: disable=C0103 + self.placementGroupId = placementGroupId # pylint: disable=C0103 + self.platformFaultDomain = platformFaultDomain # pylint: disable=C0103 + self.platformUpdateDomain = placementUpdateDomain # pylint: disable=C0103 self.publisher = publisher - self.resourceGroupName = resourceGroupName + self.resourceGroupName = resourceGroupName # pylint: disable=C0103 self.sku = sku - self.subscriptionId = subscriptionId + self.subscriptionId = subscriptionId # pylint: disable=C0103 self.tags = tags self.version = version - self.vmId = vmId - self.vmSize = vmSize - self.vmScaleSetName = vmScaleSetName + self.vmId = vmId # pylint: disable=C0103 + self.vmSize = vmSize # pylint: disable=C0103 + self.vmScaleSetName = vmScaleSetName # pylint: disable=C0103 self.zone = zone @property @@ -227,12 +227,12 @@ def image_origin(self): if self.publisher == "": return IMDS_IMAGE_ORIGIN_CUSTOM - if ComputeInfo.__matcher.is_match(self.publisher, self.offer, self.sku, self.version): + if ComputeInfo.__matcher.is_match(self.publisher, self.offer, self.sku, self.version): # pylint: disable=R1705 return IMDS_IMAGE_ORIGIN_ENDORSED else: return IMDS_IMAGE_ORIGIN_PLATFORM - except Exception as e: + except Exception as e: # pylint: disable=C0103 logger.periodic_warn(logger.EVERY_FIFTEEN_MINUTES, "[PERIODIC] Could not determine the image origin from IMDS: {0}".format(ustr(e))) return IMDS_IMAGE_ORIGIN_UNKNOWN @@ -260,7 +260,7 @@ def _http_get(self, endpoint, resource_path, headers): url = self._get_metadata_url(endpoint, resource_path) return restutil.http_get(url, headers=headers, use_proxy=False) - def _get_metadata_from_endpoint(self, endpoint, resource_path, headers): + def _get_metadata_from_endpoint(self, endpoint, resource_path, headers): # pylint: disable=R0911 """ Get metadata from one of the IMDS endpoints. @@ -276,7 +276,7 @@ def _get_metadata_from_endpoint(self, endpoint, resource_path, headers): resp = self._http_get(endpoint=endpoint, resource_path=resource_path, headers=headers) except ResourceGoneError: return IMDS_INTERNAL_SERVER_ERROR, "IMDS error in /metadata/{0}: HTTP Failed with Status Code 410: Gone".format(resource_path) - except HttpError as e: + except HttpError as e: # pylint: disable=C0103 msg = str(e) if self._regex_throttled.match(msg): return IMDS_RESPONSE_ERROR, "IMDS error in /metadata/{0}: Throttled".format(resource_path) @@ -288,11 +288,11 @@ def _get_metadata_from_endpoint(self, endpoint, resource_path, headers): if resp.status >= 500: return IMDS_INTERNAL_SERVER_ERROR, "IMDS error in /metadata/{0}: {1}".format( - resource_path, restutil.read_response_error(resp)) + resource_path, restutil.read_response_error(resp)) # pylint: disable=C0330 if restutil.request_failed(resp): return IMDS_RESPONSE_ERROR, "IMDS error in /metadata/{0}: {1}".format( - resource_path, restutil.read_response_error(resp)) + resource_path, restutil.read_response_error(resp)) # pylint: disable=C0330 return IMDS_RESPONSE_SUCCESS, resp.read() @@ -313,7 +313,7 @@ def get_metadata(self, resource_path, is_health): endpoint = self._wireserver_endpoint status, resp = self._get_metadata_from_endpoint(endpoint, resource_path, headers) - if status == IMDS_RESPONSE_SUCCESS: + if status == IMDS_RESPONSE_SUCCESS: # pylint: disable=R1705 return MetadataResult(True, False, resp) elif status == IMDS_INTERNAL_SERVER_ERROR: return MetadataResult(False, True, resp) @@ -359,12 +359,12 @@ def validate(self): # ensure the response is valid json try: json_data = json.loads(ustr(result.response, encoding="utf-8")) - except Exception as e: + except Exception as e: # pylint: disable=C0103 return False, "JSON parsing failed: {0}".format(ustr(e)) # ensure all expected fields are present and have a value try: - # TODO: compute fields cannot be verified yet since we need to exclude rdfe vms (#1249) + # TODO: compute fields cannot be verified yet since we need to exclude rdfe vms (#1249) # pylint: disable=W0511 self.check_field(json_data, 'network') self.check_field(json_data['network'], 'interface') @@ -372,7 +372,7 @@ def validate(self): self.check_field(json_data['network']['interface'][0], 'ipv4') self.check_field(json_data['network']['interface'][0]['ipv4'], 'ipAddress') self.check_field(json_data['network']['interface'][0]['ipv4']['ipAddress'][0], 'privateIpAddress') - except ValueError as v: + except ValueError as v: # pylint: disable=C0103 return False, ustr(v) return True, '' @@ -382,5 +382,5 @@ def check_field(dict_obj, field): if field not in dict_obj or dict_obj[field] is None: raise ValueError('Missing field: [{0}]'.format(field)) - if len(dict_obj[field]) == 0: + if len(dict_obj[field]) == 0: # pylint: disable=len-as-condition raise ValueError('Empty field: [{0}]'.format(field)) diff --git a/azurelinuxagent/common/protocol/ovfenv.py b/azurelinuxagent/common/protocol/ovfenv.py index a805e845e8..d5a8f6cd66 100644 --- a/azurelinuxagent/common/protocol/ovfenv.py +++ b/azurelinuxagent/common/protocol/ovfenv.py @@ -19,14 +19,14 @@ """ Copy and parse ovf-env.xml from provisioning ISO and local cache """ -import os -import re -import shutil -import xml.dom.minidom as minidom +import os # pylint: disable=W0611 +import re # pylint: disable=W0611 +import shutil # pylint: disable=W0611 +import xml.dom.minidom as minidom # pylint: disable=W0611 import azurelinuxagent.common.logger as logger from azurelinuxagent.common.exception import ProtocolError -from azurelinuxagent.common.future import ustr -import azurelinuxagent.common.utils.fileutil as fileutil +from azurelinuxagent.common.future import ustr # pylint: disable=W0611 +import azurelinuxagent.common.utils.fileutil as fileutil # pylint: disable=W0611 from azurelinuxagent.common.utils.textutil import parse_doc, findall, find, findtext OVF_VERSION = "1.0" @@ -37,7 +37,7 @@ def _validate_ovf(val, msg): if val is None: raise ProtocolError("Failed to validate OVF: {0}".format(msg)) -class OvfEnv(object): +class OvfEnv(object): # pylint: disable=R0903,R0902 """ Read, and process provisioning info from provisioning file OvfEnv.xml """ @@ -55,7 +55,7 @@ def __init__(self, xml_text): self.provision_guest_agent = None self.parse(xml_text) - def parse(self, xml_text): + def parse(self, xml_text): # pylint: disable=R0914 """ Parse xml tree, retreiving user and ssh key information. Return self. @@ -94,7 +94,7 @@ def parse(self, xml_text): auth_option = findtext(conf_set, "DisableSshPasswordAuthentication", namespace=wans) - if auth_option is not None and auth_option.lower() == "true": + if auth_option is not None and auth_option.lower() == "true": # pylint: disable=simplifiable-if-statement self.disable_ssh_password_auth = True else: self.disable_ssh_password_auth = False diff --git a/azurelinuxagent/common/protocol/restapi.py b/azurelinuxagent/common/protocol/restapi.py index ca353e5c03..38bf24eca9 100644 --- a/azurelinuxagent/common/protocol/restapi.py +++ b/azurelinuxagent/common/protocol/restapi.py @@ -23,27 +23,27 @@ from azurelinuxagent.common.datacontract import DataContract, DataContractList -class VMInfo(DataContract): - def __init__(self, +class VMInfo(DataContract): # pylint: disable=R0903 + def __init__(self, # pylint: disable=R0913 subscriptionId=None, vmName=None, roleName=None, roleInstanceName=None, tenantName=None): - self.subscriptionId = subscriptionId - self.vmName = vmName - self.roleName = roleName - self.roleInstanceName = roleInstanceName - self.tenantName = tenantName + self.subscriptionId = subscriptionId # pylint: disable=C0103 + self.vmName = vmName # pylint: disable=C0103 + self.roleName = roleName # pylint: disable=C0103 + self.roleInstanceName = roleInstanceName # pylint: disable=C0103 + self.tenantName = tenantName # pylint: disable=C0103 -class CertificateData(DataContract): +class CertificateData(DataContract): # pylint: disable=R0903 def __init__(self, certificateData=None): - self.certificateData = certificateData + self.certificateData = certificateData # pylint: disable=C0103 -class Cert(DataContract): - def __init__(self, +class Cert(DataContract): # pylint: disable=R0903 + def __init__(self, # pylint: disable=R0913 name=None, thumbprint=None, certificateDataUri=None, @@ -51,35 +51,35 @@ def __init__(self, storeLocation=None): self.name = name self.thumbprint = thumbprint - self.certificateDataUri = certificateDataUri - self.storeLocation = storeLocation - self.storeName = storeName + self.certificateDataUri = certificateDataUri # pylint: disable=C0103 + self.storeLocation = storeLocation # pylint: disable=C0103 + self.storeName = storeName # pylint: disable=C0103 -class CertList(DataContract): +class CertList(DataContract): # pylint: disable=R0903 def __init__(self): self.certificates = DataContractList(Cert) -# TODO: confirm vmagent manifest schema -class VMAgentManifestUri(DataContract): +# TODO: confirm vmagent manifest schema # pylint: disable=W0511 +class VMAgentManifestUri(DataContract): # pylint: disable=R0903 def __init__(self, uri=None): self.uri = uri -class VMAgentManifest(DataContract): +class VMAgentManifest(DataContract): # pylint: disable=R0903 def __init__(self, family=None): self.family = family - self.versionsManifestUris = DataContractList(VMAgentManifestUri) + self.versionsManifestUris = DataContractList(VMAgentManifestUri) # pylint: disable=C0103 -class VMAgentManifestList(DataContract): +class VMAgentManifestList(DataContract): # pylint: disable=R0903 def __init__(self): - self.vmAgentManifests = DataContractList(VMAgentManifest) + self.vmAgentManifests = DataContractList(VMAgentManifest) # pylint: disable=C0103 -class Extension(DataContract): - def __init__(self, +class Extension(DataContract): # pylint: disable=R0903 + def __init__(self, # pylint: disable=R0913 name=None, sequenceNumber=None, publicSettings=None, @@ -87,34 +87,34 @@ def __init__(self, certificateThumbprint=None, dependencyLevel=0): self.name = name - self.sequenceNumber = sequenceNumber - self.publicSettings = publicSettings - self.protectedSettings = protectedSettings - self.certificateThumbprint = certificateThumbprint - self.dependencyLevel = dependencyLevel + self.sequenceNumber = sequenceNumber # pylint: disable=C0103 + self.publicSettings = publicSettings # pylint: disable=C0103 + self.protectedSettings = protectedSettings # pylint: disable=C0103 + self.certificateThumbprint = certificateThumbprint # pylint: disable=C0103 + self.dependencyLevel = dependencyLevel # pylint: disable=C0103 -class ExtHandlerProperties(DataContract): +class ExtHandlerProperties(DataContract): # pylint: disable=R0903 def __init__(self): self.version = None self.state = None self.extensions = DataContractList(Extension) -class ExtHandlerVersionUri(DataContract): +class ExtHandlerVersionUri(DataContract): # pylint: disable=R0903 def __init__(self): self.uri = None -class ExtHandler(DataContract): +class ExtHandler(DataContract): # pylint: disable=R0903 def __init__(self, name=None): self.name = name self.properties = ExtHandlerProperties() - self.versionUris = DataContractList(ExtHandlerVersionUri) + self.versionUris = DataContractList(ExtHandlerVersionUri) # pylint: disable=C0103 def sort_key(self): levels = [e.dependencyLevel for e in self.properties.extensions] - if len(levels) == 0: + if len(levels) == 0: # pylint: disable=len-as-condition level = 0 else: level = min(levels) @@ -125,45 +125,45 @@ def sort_key(self): return level -class ExtHandlerList(DataContract): +class ExtHandlerList(DataContract): # pylint: disable=R0903 def __init__(self): - self.extHandlers = DataContractList(ExtHandler) + self.extHandlers = DataContractList(ExtHandler) # pylint: disable=C0103 -class ExtHandlerPackageUri(DataContract): +class ExtHandlerPackageUri(DataContract): # pylint: disable=R0903 def __init__(self, uri=None): self.uri = uri -class ExtHandlerPackage(DataContract): +class ExtHandlerPackage(DataContract): # pylint: disable=R0903 def __init__(self, version=None): self.version = version self.uris = DataContractList(ExtHandlerPackageUri) - # TODO update the naming to align with metadata protocol + # TODO update the naming to align with metadata protocol # pylint: disable=W0511 self.isinternal = False self.disallow_major_upgrade = False -class ExtHandlerPackageList(DataContract): +class ExtHandlerPackageList(DataContract): # pylint: disable=R0903 def __init__(self): self.versions = DataContractList(ExtHandlerPackage) -class VMProperties(DataContract): +class VMProperties(DataContract): # pylint: disable=R0903 def __init__(self, certificateThumbprint=None): - # TODO need to confirm the property name - self.certificateThumbprint = certificateThumbprint + # TODO need to confirm the property name # pylint: disable=W0511 + self.certificateThumbprint = certificateThumbprint # pylint: disable=C0103 -class ProvisionStatus(DataContract): +class ProvisionStatus(DataContract): # pylint: disable=R0903 def __init__(self, status=None, subStatus=None, description=None): self.status = status - self.subStatus = subStatus + self.subStatus = subStatus # pylint: disable=C0103 self.description = description self.properties = VMProperties() -class ExtensionSubStatus(DataContract): +class ExtensionSubStatus(DataContract): # pylint: disable=R0903 def __init__(self, name=None, status=None, code=None, message=None): self.name = name self.status = status @@ -171,25 +171,25 @@ def __init__(self, name=None, status=None, code=None, message=None): self.message = message -class ExtensionStatus(DataContract): - def __init__(self, +class ExtensionStatus(DataContract): # pylint: disable=R0903 + def __init__(self, # pylint: disable=R0913 configurationAppliedTime=None, operation=None, status=None, seq_no=None, code=None, message=None): - self.configurationAppliedTime = configurationAppliedTime + self.configurationAppliedTime = configurationAppliedTime # pylint: disable=C0103 self.operation = operation self.status = status - self.sequenceNumber = seq_no + self.sequenceNumber = seq_no # pylint: disable=C0103 self.code = code self.message = message - self.substatusList = DataContractList(ExtensionSubStatus) + self.substatusList = DataContractList(ExtensionSubStatus) # pylint: disable=C0103 -class ExtHandlerStatus(DataContract): - def __init__(self, +class ExtHandlerStatus(DataContract): # pylint: disable=R0903 + def __init__(self, # pylint: disable=R0913 name=None, version=None, status=None, @@ -203,7 +203,7 @@ def __init__(self, self.extensions = DataContractList(ustr) -class VMAgentStatus(DataContract): +class VMAgentStatus(DataContract): # pylint: disable=R0903 def __init__(self, status=None, message=None): self.status = status self.message = message @@ -211,22 +211,22 @@ def __init__(self, status=None, message=None): self.version = str(CURRENT_VERSION) self.osname = DISTRO_NAME self.osversion = DISTRO_VERSION - self.extensionHandlers = DataContractList(ExtHandlerStatus) + self.extensionHandlers = DataContractList(ExtHandlerStatus) # pylint: disable=C0103 -class VMStatus(DataContract): +class VMStatus(DataContract): # pylint: disable=R0903 def __init__(self, status, message): - self.vmAgent = VMAgentStatus(status=status, message=message) + self.vmAgent = VMAgentStatus(status=status, message=message) # pylint: disable=C0103 -class RemoteAccessUser(DataContract): +class RemoteAccessUser(DataContract): # pylint: disable=R0903 def __init__(self, name, encrypted_password, expiration): self.name = name self.encrypted_password = encrypted_password self.expiration = expiration -class RemoteAccessUsersList(DataContract): +class RemoteAccessUsersList(DataContract): # pylint: disable=R0903 def __init__(self): self.users = DataContractList(RemoteAccessUser) diff --git a/azurelinuxagent/common/protocol/util.py b/azurelinuxagent/common/protocol/util.py index a758a64273..077a8eabde 100644 --- a/azurelinuxagent/common/protocol/util.py +++ b/azurelinuxagent/common/protocol/util.py @@ -79,14 +79,14 @@ def copy_ovf_env(self): try: self.osutil.mount_dvd() - except OSUtilError as e: + except OSUtilError as e: # pylint: disable=C0103 raise ProtocolError("[CopyOvfEnv] Error mounting dvd: " "{0}".format(ustr(e))) try: ovfxml = fileutil.read_file(ovf_file_path_on_dvd, remove_bom=True) ovfenv = OvfEnv(ovfxml) - except (IOError, OSError) as e: + except (IOError, OSError) as e: # pylint: disable=C0103 raise ProtocolError("[CopyOvfEnv] Error reading file " "{0}: {1}".format(ovf_file_path_on_dvd, ustr(e))) @@ -96,7 +96,7 @@ def copy_ovf_env(self): PASSWORD_REPLACEMENT, ovfxml) fileutil.write_file(ovf_file_path, ovfxml) - except (IOError, OSError) as e: + except (IOError, OSError) as e: # pylint: disable=C0103 raise ProtocolError("[CopyOvfEnv] Error writing file " "{0}: {1}".format(ovf_file_path, ustr(e))) @@ -109,7 +109,7 @@ def _cleanup_ovf_dvd(self): try: self.osutil.umount_dvd() self.osutil.eject_dvd() - except OSUtilError as e: + except OSUtilError as e: # pylint: disable=C0103 logger.warn(ustr(e)) def get_ovf_env(self): @@ -117,7 +117,7 @@ def get_ovf_env(self): Load saved ovf-env.xml """ ovf_file_path = os.path.join(conf.get_lib_dir(), OVF_FILE_NAME) - if os.path.isfile(ovf_file_path): + if os.path.isfile(ovf_file_path): # pylint: disable=R1705 xml_text = fileutil.read_file(ovf_file_path) return OvfEnv(xml_text) else: @@ -150,7 +150,7 @@ def get_wireserver_endpoint(self): return self.endpoint logger.error("[GetWireserverEndpoint] Unexpected empty file {0}", file_path) - except (IOError, OSError) as e: + except (IOError, OSError) as e: # pylint: disable=C0103 logger.error("[GetWireserverEndpoint] Error reading file {0}: {1}", file_path, str(e)) else: logger.error("[GetWireserverEndpoint] Missing file {0}", file_path) @@ -167,7 +167,7 @@ def _set_wireserver_endpoint(self, endpoint): self.endpoint = endpoint file_path = self._get_wireserver_endpoint_file_path() fileutil.write_file(file_path, endpoint) - except (IOError, OSError) as e: + except (IOError, OSError) as e: # pylint: disable=C0103 raise OSUtilError(ustr(e)) def _clear_wireserver_endpoint(self): @@ -182,7 +182,7 @@ def _clear_wireserver_endpoint(self): try: os.remove(endpoint_file_path) - except (IOError, OSError) as e: + except (IOError, OSError) as e: # pylint: disable=C0103 # Ignore file-not-found errors (since the file is being removed) if e.errno == errno.ENOENT: return @@ -198,15 +198,17 @@ def _detect_protocol(self): try: endpoint = self.dhcp_handler.endpoint if endpoint is None: + # pylint: disable=W0105 ''' Check if DHCP can be used to get the wire protocol endpoint - ''' + ''' + # pylint: enable=W0105 dhcp_available = self.osutil.is_dhcp_available() if dhcp_available: logger.info("WireServer endpoint is not found. Rerun dhcp handler") try: self.dhcp_handler.run() - except DhcpError as e: + except DhcpError as e: # pylint: disable=C0103 raise ProtocolError(ustr(e)) endpoint = self.dhcp_handler.endpoint else: @@ -219,13 +221,13 @@ def _detect_protocol(self): self._set_wireserver_endpoint(endpoint) return protocol - except ProtocolError as e: + except ProtocolError as e: # pylint: disable=C0103 logger.info("WireServer is not responding. Reset dhcp endpoint") self.dhcp_handler.endpoint = None self.dhcp_handler.skip_cache = True raise e - except ProtocolError as e: + except ProtocolError as e: # pylint: disable=C0103 logger.info("Protocol endpoint not found: {0}", e) if retry < MAX_RETRY - 1: @@ -240,7 +242,7 @@ def _save_protocol(self, protocol_name): protocol_file_path = self._get_protocol_file_path() try: fileutil.write_file(protocol_file_path, protocol_name) - except (IOError, OSError) as e: + except (IOError, OSError) as e: # pylint: disable=C0103 logger.error("Failed to save protocol endpoint: {0}", e) def clear_protocol(self): @@ -258,7 +260,7 @@ def clear_protocol(self): try: os.remove(protocol_file_path) - except (IOError, OSError) as e: + except (IOError, OSError) as e: # pylint: disable=C0103 # Ignore file-not-found errors (since the file is being removed) if e.errno == errno.ENOENT: return diff --git a/azurelinuxagent/common/protocol/wire.py b/azurelinuxagent/common/protocol/wire.py index e3345efd69..b92d8b0ae8 100644 --- a/azurelinuxagent/common/protocol/wire.py +++ b/azurelinuxagent/common/protocol/wire.py @@ -1,4 +1,4 @@ -# Microsoft Azure Linux Agent +# Microsoft Azure Linux Agent # pylint: disable=C0302 # # Copyright 2018 Microsoft Corporation # @@ -23,7 +23,7 @@ import time import traceback import xml.sax.saxutils as saxutils -from datetime import datetime +from datetime import datetime # pylint: disable=ungrouped-imports import azurelinuxagent.common.conf as conf import azurelinuxagent.common.logger as logger @@ -156,7 +156,7 @@ def _download_ext_handler_pkg_through_host(self, uri, destination): success = self.client.stream(uri, destination, headers=headers, use_proxy=False) return success - def download_ext_handler_pkg(self, uri, destination, headers=None, use_proxy=True): + def download_ext_handler_pkg(self, uri, destination, headers=None, use_proxy=True): # pylint: disable=W0613 direct_func = lambda: self.client.stream(uri, destination, headers=None, use_proxy=True) # NOTE: the host_func may be called after refreshing the goal state, be careful about any goal state data # in the lambda. @@ -185,7 +185,7 @@ def report_vm_status(self, vm_status): self.client.status_blob.set_vm_status(vm_status) self.client.upload_status_blob() - def report_ext_status(self, ext_handler_name, ext_name, ext_status): + def report_ext_status(self, ext_handler_name, ext_name, ext_status): # pylint: disable=W0613 validate_param("ext_status", ext_status, ExtensionStatus) self.client.status_blob.set_ext_status(ext_handler_name, ext_status) @@ -216,7 +216,7 @@ def _build_role_properties(container_id, role_instance_id, thumbprint): return xml -def _build_health_report(incarnation, container_id, role_instance_id, +def _build_health_report(incarnation, container_id, role_instance_id, # pylint: disable=R0913 status, substatus, description): # Escape '&', '<' and '>' description = saxutils.escape(ustr(description)) @@ -315,12 +315,12 @@ def ext_status_to_v1(ext_name, ext_status): "version": 1.0, "timestampUTC": timestamp } - if len(v1_sub_status) != 0: + if len(v1_sub_status) != 0: # pylint: disable=len-as-condition v1_ext_status['status']['substatus'] = v1_sub_status return v1_ext_status -def ext_handler_status_to_v1(handler_status, ext_statuses, timestamp): +def ext_handler_status_to_v1(handler_status, ext_statuses, timestamp): # pylint: disable=W0613 v1_handler_status = { 'handlerVersion': handler_status.version, 'handlerName': handler_status.name, @@ -334,7 +334,7 @@ def ext_handler_status_to_v1(handler_status, ext_statuses, timestamp): "message": handler_status.message } - if len(handler_status.extensions) > 0: + if len(handler_status.extensions) > 0: # pylint: disable=len-as-condition # Currently, no more than one extension per handler ext_name = handler_status.extensions[0] ext_status = ext_statuses.get(ext_name) @@ -410,7 +410,7 @@ def upload(self, url): self.put_page_blob(url, self.data) return True - except Exception as e: + except Exception as e: # pylint: disable=C0103 logger.verbose("Initial status upload failed: {0}", e) return False @@ -518,7 +518,7 @@ def event_to_v1_encoded(event, encoding='utf-8'): return event_str.encode(encoding) -class WireClient(object): +class WireClient(object): # pylint: disable=R0904 def __init__(self, endpoint): logger.info("Wire server endpoint:{0}", endpoint) @@ -548,7 +548,7 @@ def call_wireserver(self, http_req, *args, **kwargs): except ResourceGoneError: raise - except Exception as e: + except Exception as e: # pylint: disable=C0103 raise ProtocolError("[Wireserver Exception] {0}".format( ustr(e))) @@ -572,20 +572,20 @@ def fetch_cache(self, local_file): raise ProtocolError("{0} is missing.".format(local_file)) try: return fileutil.read_file(local_file) - except IOError as e: + except IOError as e: # pylint: disable=C0103 raise ProtocolError("Failed to read cache: {0}".format(e)) def save_cache(self, local_file, data): try: fileutil.write_file(local_file, data) - except IOError as e: + except IOError as e: # pylint: disable=C0103 fileutil.clean_ioerror(e, paths=[local_file]) raise ProtocolError("Failed to write cache: {0}".format(e)) @staticmethod def call_storage_service(http_req, *args, **kwargs): # Default to use the configured HTTP proxy - if not 'use_proxy' in kwargs or kwargs['use_proxy'] is None: + if not 'use_proxy' in kwargs or kwargs['use_proxy'] is None: # pylint: disable=C0113 kwargs['use_proxy'] = True return http_req(*args, **kwargs) @@ -608,10 +608,10 @@ def fetch_manifest(self, version_uris): logger.verbose('The specified manifest URL is empty, ignored.') continue - direct_func = lambda: self.fetch(version.uri) + direct_func = lambda: self.fetch(version.uri) # pylint: disable=W0640 # NOTE: the host_func may be called after refreshing the goal state, be careful about any goal state data # in the lambda. - host_func = lambda: self.fetch_manifest_through_host(version.uri) + host_func = lambda: self.fetch_manifest_through_host(version.uri) # pylint: disable=W0640 try: response = self.send_request_using_appropriate_channel(direct_func, host_func) @@ -620,7 +620,7 @@ def fetch_manifest(self, version_uris): host = self.get_host_plugin() host.update_manifest_uri(version.uri) return response - except Exception as e: + except Exception as e: # pylint: disable=C0103 logger.warn("Exception when fetching manifest. Error: {0}".format(ustr(e))) raise ExtensionDownloadError("Failed to fetch manifest from all sources") @@ -640,7 +640,7 @@ def stream(self, uri, destination, headers=None, use_proxy=None): destination_fh.write(chunk) complete = len(chunk) < chunk_size success = True - except Exception as e: + except Exception as e: # pylint: disable=C0103 logger.error('Error streaming {0} to {1}: {2}'.format(uri, destination, ustr(e))) return success @@ -665,7 +665,7 @@ def _fetch_response(self, uri, headers=None, use_proxy=None): host_plugin = self.get_host_plugin() - if restutil.request_failed(resp): + if restutil.request_failed(resp): # pylint: disable=R1720 error_response = restutil.read_response_error(resp) msg = "Fetch failed from [{0}]: {1}".format(uri, error_response) logger.warn(msg) @@ -680,14 +680,14 @@ def _fetch_response(self, uri, headers=None, use_proxy=None): if host_plugin is not None: host_plugin.report_fetch_health(uri, source='WireClient') - except (HttpError, ProtocolError, IOError) as e: + except (HttpError, ProtocolError, IOError) as e: # pylint: disable=C0103 logger.verbose("Fetch failed from [{0}]: {1}", uri, e) - if isinstance(e, ResourceGoneError) or isinstance(e, InvalidContainerError): + if isinstance(e, ResourceGoneError) or isinstance(e, InvalidContainerError): # pylint: disable=R1701 raise return resp # Type of update performed by _update_from_goal_state() - class _UpdateType(object): + class _UpdateType(object): # pylint: disable=R0903 # Update the Host GA Plugin client (Container ID and RoleConfigName) HostPlugin = 0 # Update the full goal state only if the incarnation has changed @@ -720,7 +720,7 @@ def try_update_goal_state(self): message = u"Retrieving the goal state recovered from previous errors" add_event(AGENT_NAME, op=WALAEventOperation.FetchGoalState, version=CURRENT_VERSION, is_success=True, message=message, log_event=False) logger.info(message) - except Exception as e: + except Exception as e: # pylint: disable=C0103 if not self._last_try_update_goal_state_failed: self._last_try_update_goal_state_failed = True message = u"An error occurred while retrieving the goal state: {0}".format(ustr(e)) @@ -757,16 +757,16 @@ def _update_from_goal_state(self, refresh_type): return - except IOError as e: + except IOError as e: # pylint: disable=C0103 logger.warn("IOError processing goal state (attempt {0}/{1}) [{2}]", retry, max_retry, ustr(e)) except ResourceGoneError: logger.info("Goal state is stale, re-fetching (attempt {0}/{1})", retry, max_retry) - except ProtocolError as e: + except ProtocolError as e: # pylint: disable=C0103 logger.verbose("ProtocolError processing goal state (attempt {0}/{1}) [{2}]", retry, max_retry, ustr(e)) - except Exception as e: + except Exception as e: # pylint: disable=C0103 logger.verbose("Exception processing goal state (attempt {0}/{1}) [{2}]", retry, max_retry, ustr(e)) raise ProtocolError("Exceeded max retry updating goal state") @@ -780,7 +780,7 @@ def _save_goal_state(self): try: self.goal_state_flusher.flush(datetime.utcnow()) - except Exception as e: + except Exception as e: # pylint: disable=C0103 logger.warn("Failed to save the previous goal state to the history folder: {0}", ustr(e)) try: @@ -800,7 +800,7 @@ def save_if_not_none(goal_state_property, file_name): save_if_not_none(self._goal_state.ext_conf, EXT_CONF_FILE_NAME.format(self._goal_state.incarnation)) save_if_not_none(self._goal_state.remote_access, REMOTE_ACCESS_FILE_NAME.format(self._goal_state.incarnation)) - except Exception as e: + except Exception as e: # pylint: disable=C0103 logger.warn("Failed to save the goal state to disk: {0}", ustr(e)) def _set_host_plugin(self, new_host_plugin): @@ -844,7 +844,7 @@ def get_ext_manifest(self, ext_handler): xml_text = self.fetch_manifest(ext_handler.versionUris) self.save_cache(local_file, xml_text) return ExtensionManifest(xml_text) - except Exception as e: + except Exception as e: # pylint: disable=C0103 raise ExtensionDownloadError("Failed to retrieve extension manifest. Error: {0}".format(ustr(e))) def get_remote_access(self): @@ -860,7 +860,7 @@ def get_gafamily_manifest(self, vmagent_manifest, goal_state): xml_text = self.fetch_manifest(vmagent_manifest.versionsManifestUris) fileutil.write_file(local_file, xml_text) return ExtensionManifest(xml_text) - except Exception as e: + except Exception as e: # pylint: disable=C0103 raise ProtocolError("Failed to retrieve GAFamily manifest. Error: {0}".format(ustr(e))) def check_wire_protocol_version(self): @@ -883,7 +883,7 @@ def _call_hostplugin_with_container_check(self, host_func): ret = None try: ret = host_func() - except (ResourceGoneError, InvalidContainerError) as e: + except (ResourceGoneError, InvalidContainerError) as e: # pylint: disable=C0103 host_plugin = self.get_host_plugin() old_container_id = host_plugin.container_id old_role_config_name = host_plugin.role_config_name @@ -916,7 +916,7 @@ def _call_hostplugin_with_container_check(self, host_func): message=msg, log_event=True) - except (ResourceGoneError, InvalidContainerError) as e: + except (ResourceGoneError, InvalidContainerError) as e: # pylint: disable=C0103 msg = "[PERIODIC] Request failed using the host plugin channel after goal state refresh. " \ "ContainerId changed from {0} to {1}, role config file changed from {2} to {3}. " \ "Exception type: {4}.".format(old_container_id, new_container_id, old_role_config_name, @@ -958,7 +958,7 @@ def send_request_using_appropriate_channel(self, direct_func, host_func): if not ret: logger.periodic_info(logger.EVERY_HOUR, "[PERIODIC] Request failed using the direct channel, " "switching to host plugin.") - except (ResourceGoneError, InvalidContainerError) as e: + except (ResourceGoneError, InvalidContainerError) as e: # pylint: disable=C0103 logger.periodic_info(logger.EVERY_HOUR, "[PERIODIC] Request failed using the direct channel, " "switching to host plugin. Error: {0}".format(ustr(e))) @@ -994,8 +994,8 @@ def upload_status_blob(self): try: self.status_blob.prepare(blob_type) - except Exception as e: - raise ProtocolError("Exception creating status blob: {0}", ustr(e)) + except Exception as e: # pylint: disable=C0103 + raise ProtocolError("Exception creating status blob: {0}", ustr(e)) # pylint: disable=W0715 # Swap the order of use for the HostPlugin vs. the "direct" route. # Prefer the use of HostPlugin. If HostPlugin fails fall back to the @@ -1014,7 +1014,7 @@ def upload_status_blob(self): # refresh the host plugin client and try again on the next iteration of the main loop self.update_host_plugin_from_goal_state() return - except Exception as e: + except Exception as e: # pylint: disable=C0103 # for all other errors, fall back to direct msg = "Falling back to direct upload: {0}".format(ustr(e)) self.report_status_event(msg, is_success=True) @@ -1022,7 +1022,7 @@ def upload_status_blob(self): try: if self.status_blob.upload(ext_conf.status_upload_blob): return - except Exception as e: + except Exception as e: # pylint: disable=C0103 msg = "Exception uploading status blob: {0}".format(ustr(e)) self.report_status_event(msg, is_success=False) @@ -1041,7 +1041,7 @@ def report_role_prop(self, thumbprint): role_prop_uri, role_prop, headers=headers) - except HttpError as e: + except HttpError as e: # pylint: disable=C0103 raise ProtocolError((u"Failed to send role properties: " u"{0}").format(e)) if resp.status != httpclient.ACCEPTED: @@ -1070,7 +1070,7 @@ def report_health(self, status, substatus, description): headers=headers, max_retry=30, retry_delay=15) - except HttpError as e: + except HttpError as e: # pylint: disable=C0103 raise ProtocolError((u"Failed to send provision status: " u"{0}").format(e)) if restutil.request_failed(resp): @@ -1090,7 +1090,7 @@ def send_encoded_event(self, provider_id, event_str, encoding='utf-8'): # NOTE: The call to wireserver requests utf-8 encoding in the headers, but the body should not # be encoded: some nodes in the telemetry pipeline do not support utf-8 encoding. resp = self.call_wireserver(restutil.http_post, uri, data, header) - except HttpError as e: + except HttpError as e: # pylint: disable=C0103 raise ProtocolError("Failed to send events:{0}".format(e)) if restutil.request_failed(resp): @@ -1127,11 +1127,11 @@ def report_event(self, event_list): events_per_request = 0 buf[event.providerId] = buf[event.providerId] + event_str events_per_request += 1 - except UnicodeError as e: + except UnicodeError as e: # pylint: disable=C0103 unicode_error_count += 1 if len(unicode_errors) < max_send_errors_to_report: unicode_errors.append(ustr(e)) - except Exception as e: + except Exception as e: # pylint: disable=C0103 event_report_error_count += 1 if len(event_report_errors) < max_send_errors_to_report: event_report_errors.append(ustr(e)) @@ -1144,7 +1144,7 @@ def report_event(self, event_list): # Send out all events left in buffer. for provider_id in list(buf.keys()): - if len(buf[provider_id]) > 0: + if len(buf[provider_id]) > 0: # pylint: disable=len-as-condition logger.verbose("No of events this request = {0}".format(events_per_request)) self.send_encoded_event(provider_id, buf[provider_id]) @@ -1212,7 +1212,7 @@ def get_artifacts_profile(self): try: profile = self.send_request_using_appropriate_channel(direct_func, host_func) - except Exception as e: + except Exception as e: # pylint: disable=C0103 logger.warn("Exception retrieving artifacts profile: {0}".format(ustr(e))) return None @@ -1273,7 +1273,7 @@ def get_supported(self): return self.supported -class ExtensionManifest(object): +class ExtensionManifest(object): # pylint: disable=R0903 def __init__(self, xml_text): if xml_text is None: raise ValueError("ExtensionManifest is None") @@ -1318,7 +1318,7 @@ def _handle_packages(self, packages, isinternal): # Do not extend this class -class InVMArtifactsProfile(object): +class InVMArtifactsProfile(object): # pylint: disable=R0903 """ deserialized json string of InVMArtifactsProfile. It is expected to contain the following fields: @@ -1337,5 +1337,5 @@ def __init__(self, artifacts_profile): def is_on_hold(self): # hasattr() is not available in Python 2.6 if 'onHold' in self.__dict__: - return str(self.onHold).lower() == 'true' + return str(self.onHold).lower() == 'true' # pylint: disable=E1101 return False diff --git a/azurelinuxagent/common/rdma.py b/azurelinuxagent/common/rdma.py index 72e6ea3f7d..1c13e6b51a 100644 --- a/azurelinuxagent/common/rdma.py +++ b/azurelinuxagent/common/rdma.py @@ -29,7 +29,7 @@ import azurelinuxagent.common.utils.shellutil as shellutil from azurelinuxagent.common.utils.textutil import parse_doc, find, getattrib -dapl_config_paths = [ +dapl_config_paths = [ # pylint: disable=invalid-name '/etc/dat.conf', '/etc/rdma/dat.conf', '/usr/local/etc/dat.conf' @@ -76,7 +76,7 @@ class RDMAHandler(object): driver_module_name = 'hv_network_direct' nd_version = None - def get_rdma_version(self): + def get_rdma_version(self): # pylint: disable=R1710 """Retrieve the firmware version information from the system. This depends on information provided by the Linux kernel.""" @@ -94,7 +94,7 @@ def get_rdma_version(self): logger.error(error_msg % driver_info_source) return - f = open(driver_info_source) + f = open(driver_info_source) # pylint: disable=C0103 while True : key = f.read(kvp_key_size) value = f.read(kvp_value_size) @@ -124,8 +124,8 @@ def is_kvp_daemon_running(): exitcode, ps_out = shellutil.run_get_output("ps -ef") if exitcode != 0: raise Exception('RDMA: ps -ef failed: %s' % ps_out) - for n in kvp_daemon_names: - if n in ps_out: + for n in kvp_daemon_names: # pylint: disable=C0103 + if n in ps_out: # pylint: disable=R1705 logger.info('RDMA: kvp daemon (%s) is running' % n) return True else: @@ -165,7 +165,7 @@ def install_driver(self): def is_driver_loaded(self): """Check if the network module is loaded in kernel space""" cmd = 'lsmod | grep ^%s' % self.driver_module_name - status, loaded_modules = shellutil.run_get_output(cmd) + status, loaded_modules = shellutil.run_get_output(cmd) # pylint: disable=W0612 logger.info('RDMA: Checking if the module loaded.') if loaded_modules: logger.info('RDMA: module loaded.') @@ -182,7 +182,7 @@ def reboot_system(self): logger.error('RDMA: Failed to reboot the system') -dapl_config_paths = [ +dapl_config_paths = [ # pylint: disable=invalid-name '/etc/dat.conf', '/etc/rdma/dat.conf', '/usr/local/etc/dat.conf'] class RDMADeviceHandler(object): @@ -221,7 +221,7 @@ def process(self): else : logger.info("RDMA: provisioning Network Direct RDMA device.") self.provision_network_direct_rdma() - except Exception as e: + except Exception as e: # pylint: disable=C0103 logger.error("RDMA: device processing failed: {0}".format(e)) def provision_network_direct_rdma(self) : @@ -245,10 +245,10 @@ def provision_network_direct_rdma(self) : return retcode,out = shellutil.run_get_output("modinfo %s" % module_name) if retcode == 0: - version = re.search("version:\s+(\d+)\.(\d+)\.(\d+)\D", out, re.IGNORECASE) + version = re.search("version:\s+(\d+)\.(\d+)\.(\d+)\D", out, re.IGNORECASE) # pylint: disable=W1401 if version: - v1 = int(version.groups(0)[0]) - v2 = int(version.groups(0)[1]) + v1 = int(version.groups(0)[0]) # pylint: disable=C0103 + v2 = int(version.groups(0)[1]) # pylint: disable=C0103 if v1>4 or v1==4 and v2>0: logger.info("Skip setting /dev/hvnd_rdma on 4.1 or later") skip_rdma_device = True @@ -265,7 +265,7 @@ def provision_network_direct_rdma(self) : RDMADeviceHandler.update_network_interface(self.mac_addr, self.ipv4_addr) - def provision_sriov_rdma(self) : + def provision_sriov_rdma(self) : # pylint: disable=R1711 RDMADeviceHandler.wait_any_rdma_device( self.sriov_dir, self.device_check_timeout_sec, self.device_check_interval_sec) RDMADeviceHandler.update_iboip_interface(self.ipv4_addr, self.ipoib_check_timeout_sec, self.ipoib_check_interval_sec) @@ -275,7 +275,7 @@ def provision_sriov_rdma(self) : def update_iboip_interface(ipv4_addr, timeout_sec, check_interval_sec) : logger.info("Wait for ib0 become available") total_retries = timeout_sec/check_interval_sec - n = 0 + n = 0 # pylint: disable=C0103 found_ib0 = None while not found_ib0 and n < total_retries: ret, output = shellutil.run_get_output("ifconfig -a") @@ -285,7 +285,7 @@ def update_iboip_interface(ipv4_addr, timeout_sec, check_interval_sec) : if found_ib0: break time.sleep(check_interval_sec) - n += 1 + n += 1 # pylint: disable=C0103 if not found_ib0: raise Exception("ib0 is not available") @@ -304,7 +304,7 @@ def update_dat_conf(paths, ipv4_addr): infiniband interface. """ logger.info("Updating DAPL configuration file") - for f in paths: + for f in paths: # pylint: disable=C0103 logger.info("RDMA: trying {0}".format(f)) if not os.path.isfile(f): logger.info( @@ -322,7 +322,7 @@ def update_dat_conf(paths, ipv4_addr): @staticmethod def replace_dat_conf_contents(cfg, ipv4_addr): - old = "ofa-v2-ib0 u2.0 nonthreadsafe default libdaplofa.so.2 dapl.2.0 \"\S+ 0\"" + old = "ofa-v2-ib0 u2.0 nonthreadsafe default libdaplofa.so.2 dapl.2.0 \"\S+ 0\"" # pylint: disable=W1401 new = "ofa-v2-ib0 u2.0 nonthreadsafe default libdaplofa.so.2 dapl.2.0 \"{0} 0\"".format( ipv4_addr) return re.sub(old, new, cfg) @@ -332,7 +332,7 @@ def write_rdma_config_to_device(path, ipv4_addr, mac_addr): data = RDMADeviceHandler.generate_rdma_config(ipv4_addr, mac_addr) logger.info( "RDMA: Updating device with configuration: {0}".format(data)) - with open(path, "w") as f: + with open(path, "w") as f: # pylint: disable=C0103 logger.info("RDMA: Device opened for writing") f.write(data) logger.info("RDMA: Updated device with IPv4/MAC addr successfully") @@ -345,7 +345,7 @@ def generate_rdma_config(ipv4_addr, mac_addr): def wait_rdma_device(path, timeout_sec, check_interval_sec): logger.info("RDMA: waiting for device={0} timeout={1}s".format(path, timeout_sec)) total_retries = timeout_sec/check_interval_sec - n = 0 + n = 0 # pylint: disable=C0103 while n < total_retries: if os.path.exists(path): logger.info("RDMA: device ready") @@ -353,27 +353,27 @@ def wait_rdma_device(path, timeout_sec, check_interval_sec): logger.verbose( "RDMA: device not ready, sleep {0}s".format(check_interval_sec)) time.sleep(check_interval_sec) - n += 1 + n += 1 # pylint: disable=C0103 logger.error("RDMA device wait timed out") raise Exception("The device did not show up in {0} seconds ({1} retries)".format( timeout_sec, total_retries)) @staticmethod - def wait_any_rdma_device(dir, timeout_sec, check_interval_sec): + def wait_any_rdma_device(dir, timeout_sec, check_interval_sec): # pylint: disable=W0622 logger.info( "RDMA: waiting for any Infiniband device at directory={0} timeout={1}s".format( - dir, timeout_sec)) + dir, timeout_sec)) # pylint: disable=C0330 total_retries = timeout_sec/check_interval_sec - n = 0 + n = 0 # pylint: disable=C0103 while n < total_retries: - r = os.listdir(dir) + r = os.listdir(dir) # pylint: disable=C0103 if r: logger.info("RDMA: device found in {0}".format(dir)) return logger.verbose( "RDMA: device not ready, sleep {0}s".format(check_interval_sec)) time.sleep(check_interval_sec) - n += 1 + n += 1 # pylint: disable=C0103 logger.error("RDMA device wait timed out") raise Exception("The device did not show up in {0} seconds ({1} retries)".format( timeout_sec, total_retries)) @@ -408,6 +408,6 @@ def get_interface_by_mac(mac): raise Exception("Failed to get ifname with mac: {0}".format(mac)) output = match.group(0) eths = re.findall(r"eth\d", output) - if eths is None or len(eths) == 0: + if eths is None or len(eths) == 0: # pylint: disable=len-as-condition raise Exception("ifname with mac: {0} not found".format(mac)) return eths[-1] diff --git a/azurelinuxagent/common/resourceusage.py b/azurelinuxagent/common/resourceusage.py index 841df3ea01..3ed3973e07 100644 --- a/azurelinuxagent/common/resourceusage.py +++ b/azurelinuxagent/common/resourceusage.py @@ -30,17 +30,17 @@ PROC_STATUS_FILENAME_FORMAT = "/proc/{0}/status" -class ResourceUsage(object): +class ResourceUsage(object): # pylint: disable=R0903 pass -class MemoryResourceUsage(ResourceUsage): +class MemoryResourceUsage(ResourceUsage): # pylint: disable=R0903 @staticmethod def get_memory_usage_from_proc_statm(process_id): proc_pid_rss = 0 try: proc_pid_rss = MemoryResourceUsage._get_proc_rss(process_id) - except Exception as e: + except Exception as e: # pylint: disable=C0103 if isinstance(e, (IOError, OSError)): raise logger.periodic_info(EVERY_SIX_HOURS, "[PERIODIC] Could not get the /prod/{0}/statm data due to {1}", process_id, ustr(e)) @@ -100,7 +100,7 @@ def _get_proc_cmdline(cls, process_id): cmdline_file_name = PROC_CMDLINE_FILENAME_FORMAT.format(process_id) try: pid_cmdline = fileutil.read_file(cmdline_file_name).replace("\0", " ").strip() - except Exception as e: + except Exception as e: # pylint: disable=C0103 if isinstance(e, (IOError, OSError)): raise raise ProcessInfoException("Could not get contents from {0}".format(cmdline_file_name), e) @@ -123,7 +123,7 @@ def _get_proc_comm(cls, process_id): try: pid_comm = fileutil.read_file(comm_file_name).strip() pid_comm_str = str(pid_comm) - except Exception as e: + except Exception as e: # pylint: disable=C0103 if isinstance(e, (IOError, OSError)): raise raise ProcessInfoException("Could not get contents from {0}".format(comm_file_name), e) diff --git a/azurelinuxagent/common/singletonperthread.py b/azurelinuxagent/common/singletonperthread.py index 0d9139012e..f13adb90f5 100644 --- a/azurelinuxagent/common/singletonperthread.py +++ b/azurelinuxagent/common/singletonperthread.py @@ -14,7 +14,7 @@ def __call__(cls, *args, **kwargs): return cls._instances[obj_name] -class SingletonPerThread(_SingletonPerThreadMetaClass('SingleObjectPerThreadMetaClass', (object,), {})): +class SingletonPerThread(_SingletonPerThreadMetaClass('SingleObjectPerThreadMetaClass', (object,), {})): # pylint: disable=R0903 # This base class calls the metaclass above to create the singleton per thread object. This class provides an # abstraction over how to invoke the Metaclass so just inheriting this class makes the # child class a singleton per thread (As opposed to invoking the Metaclass separately for each derived classes) diff --git a/azurelinuxagent/common/telemetryevent.py b/azurelinuxagent/common/telemetryevent.py index bbd7af2636..78117b5d46 100644 --- a/azurelinuxagent/common/telemetryevent.py +++ b/azurelinuxagent/common/telemetryevent.py @@ -20,7 +20,7 @@ from azurelinuxagent.common.datacontract import DataContract, DataContractList from azurelinuxagent.common.version import AGENT_NAME -class CommonTelemetryEventSchema(object): +class CommonTelemetryEventSchema(object): # pylint: disable=R0903 # Common schema keys for GuestAgentExtensionEvents, GuestAgentGenericLogs # and GuestAgentPerformanceCounterEvents tables in Kusto. @@ -44,7 +44,7 @@ class CommonTelemetryEventSchema(object): VMId = "VMId" ImageOrigin = "ImageOrigin" -class GuestAgentGenericLogsSchema(CommonTelemetryEventSchema): +class GuestAgentGenericLogsSchema(CommonTelemetryEventSchema): # pylint: disable=R0903 # GuestAgentGenericLogs table specific schema keys EventName = "EventName" @@ -53,7 +53,7 @@ class GuestAgentGenericLogsSchema(CommonTelemetryEventSchema): Context2 = "Context2" Context3 = "Context3" -class GuestAgentExtensionEventsSchema(CommonTelemetryEventSchema): +class GuestAgentExtensionEventsSchema(CommonTelemetryEventSchema): # pylint: disable=R0903 # GuestAgentExtensionEvents table specific schema keys ExtensionType = "ExtensionType" @@ -65,7 +65,7 @@ class GuestAgentExtensionEventsSchema(CommonTelemetryEventSchema): Message = "Message" Duration = "Duration" -class GuestAgentPerfCounterEventsSchema(CommonTelemetryEventSchema): +class GuestAgentPerfCounterEventsSchema(CommonTelemetryEventSchema): # pylint: disable=R0903 # GuestAgentPerformanceCounterEvents table specific schema keys Category = "Category" @@ -73,7 +73,7 @@ class GuestAgentPerfCounterEventsSchema(CommonTelemetryEventSchema): Instance = "Instance" Value = "Value" -class TelemetryEventParam(DataContract): +class TelemetryEventParam(DataContract): # pylint: disable=R0903 def __init__(self, name=None, value=None): self.name = name self.value = value @@ -84,8 +84,8 @@ def __eq__(self, other): class TelemetryEvent(DataContract): def __init__(self, eventId=None, providerId=None): - self.eventId = eventId - self.providerId = providerId + self.eventId = eventId # pylint: disable=C0103 + self.providerId = providerId # pylint: disable=C0103 self.parameters = DataContractList(TelemetryEventParam) self.file_type = "" @@ -109,6 +109,6 @@ def get_version(self): return None -class TelemetryEventList(DataContract): +class TelemetryEventList(DataContract): # pylint: disable=R0903 def __init__(self): self.events = DataContractList(TelemetryEvent) diff --git a/azurelinuxagent/common/utils/archive.py b/azurelinuxagent/common/utils/archive.py index 52920b2574..2fa3f12408 100644 --- a/azurelinuxagent/common/utils/archive.py +++ b/azurelinuxagent/common/utils/archive.py @@ -11,6 +11,7 @@ import azurelinuxagent.common.logger as logger +# pylint: disable=W0105 """ archive.py @@ -33,38 +34,39 @@ The timestamp is an ISO8601 formatted value. """ +# pylint: enable=W0105 ARCHIVE_DIRECTORY_NAME = 'history' MAX_ARCHIVED_STATES = 50 CACHE_PATTERNS = [ - re.compile("^(.*)\.(\d+)\.(agentsManifest)$", re.IGNORECASE), - re.compile("^(.*)\.(\d+)\.(manifest\.xml)$", re.IGNORECASE), - re.compile("^(.*)\.(\d+)\.(xml)$", re.IGNORECASE) + re.compile("^(.*)\.(\d+)\.(agentsManifest)$", re.IGNORECASE), # pylint: disable=W1401 + re.compile("^(.*)\.(\d+)\.(manifest\.xml)$", re.IGNORECASE), # pylint: disable=W1401 + re.compile("^(.*)\.(\d+)\.(xml)$", re.IGNORECASE) # pylint: disable=W1401 ] # 2018-04-06T08:21:37.142697 # 2018-04-06T08:21:37.142697.zip -ARCHIVE_PATTERNS_DIRECTORY = re.compile('^\d{4}\-\d{2}\-\d{2}T\d{2}:\d{2}:\d{2}\.\d+$') -ARCHIVE_PATTERNS_ZIP = re.compile('^\d{4}\-\d{2}\-\d{2}T\d{2}:\d{2}:\d{2}\.\d+\.zip$') +ARCHIVE_PATTERNS_DIRECTORY = re.compile('^\d{4}\-\d{2}\-\d{2}T\d{2}:\d{2}:\d{2}\.\d+$') # pylint: disable=W1401 +ARCHIVE_PATTERNS_ZIP = re.compile('^\d{4}\-\d{2}\-\d{2}T\d{2}:\d{2}:\d{2}\.\d+\.zip$') # pylint: disable=W1401 class StateFlusher(object): def __init__(self, lib_dir): self._source = lib_dir - d = os.path.join(self._source, ARCHIVE_DIRECTORY_NAME) + d = os.path.join(self._source, ARCHIVE_DIRECTORY_NAME) # pylint: disable=C0103 if not os.path.exists(d): try: fileutil.mkdir(d) - except OSError as e: + except OSError as e: # pylint: disable=C0103 if e.errno != errno.EEXIST: logger.error("{0} : {1}", self._source, e.strerror) def flush(self, timestamp): files = self._get_files_to_archive() - if len(files) == 0: + if len(files) == 0: # pylint: disable=len-as-condition return if self._mkdir(timestamp): @@ -77,10 +79,10 @@ def history_dir(self, timestamp): def _get_files_to_archive(self): files = [] - for f in os.listdir(self._source): + for f in os.listdir(self._source): # pylint: disable=C0103 full_path = os.path.join(self._source, f) for pattern in CACHE_PATTERNS: - m = pattern.match(f) + m = pattern.match(f) # pylint: disable=C0103 if m is not None: files.append(full_path) break @@ -88,26 +90,26 @@ def _get_files_to_archive(self): return files def _archive(self, files, timestamp): - for f in files: + for f in files: # pylint: disable=C0103 dst = os.path.join(self.history_dir(timestamp), os.path.basename(f)) shutil.move(f, dst) def _purge(self, files): - for f in files: + for f in files: # pylint: disable=C0103 os.remove(f) def _mkdir(self, timestamp): - d = self.history_dir(timestamp) + d = self.history_dir(timestamp) # pylint: disable=C0103 try: fileutil.mkdir(d, mode=0o700) return True - except IOError as e: + except IOError as e: # pylint: disable=C0103 logger.error("{0} : {1}".format(d, e.strerror)) return False -# TODO: use @total_ordering once RHEL/CentOS and SLES 11 are EOL. +# TODO: use @total_ordering once RHEL/CentOS and SLES 11 are EOL. # pylint: disable=W0511 # @total_ordering first appeared in Python 2.7 and 3.2 # If there are more use cases for @total_ordering, I will # consider re-implementing it. @@ -146,7 +148,7 @@ def __ge__(self, other): class StateZip(State): - def __init__(self, path, timestamp): + def __init__(self, path, timestamp): # pylint: disable=W0235 super(StateZip,self).__init__(path, timestamp) def delete(self): @@ -154,7 +156,7 @@ def delete(self): class StateDirectory(State): - def __init__(self, path, timestamp): + def __init__(self, path, timestamp): # pylint: disable=W0235 super(StateDirectory, self).__init__(path, timestamp) def delete(self): @@ -162,10 +164,10 @@ def delete(self): def archive(self): fn_tmp = "{0}.zip.tmp".format(self._path) - fn = "{0}.zip".format(self._path) + fn = "{0}.zip".format(self._path) # pylint: disable=C0103 ziph = zipfile.ZipFile(fn_tmp, 'w') - for f in os.listdir(self._path): + for f in os.listdir(self._path): # pylint: disable=C0103 full_path = os.path.join(self._path, f) ziph.write(full_path, f, zipfile.ZIP_DEFLATED) @@ -182,7 +184,7 @@ def __init__(self, lib_dir): if not os.path.isdir(self._source): try: fileutil.mkdir(self._source, mode=0o700) - except IOError as e: + except IOError as e: # pylint: disable=C0103 if e.errno != errno.EEXIST: logger.error("{0} : {1}", self._source, e.strerror) @@ -205,13 +207,13 @@ def archive(self): def _get_archive_states(self): states = [] - for f in os.listdir(self._source): + for f in os.listdir(self._source): # pylint: disable=C0103 full_path = os.path.join(self._source, f) - m = ARCHIVE_PATTERNS_DIRECTORY.match(f) + m = ARCHIVE_PATTERNS_DIRECTORY.match(f) # pylint: disable=C0103 if m is not None: states.append(StateDirectory(full_path, m.group(0))) - m = ARCHIVE_PATTERNS_ZIP.match(f) + m = ARCHIVE_PATTERNS_ZIP.match(f) # pylint: disable=C0103 if m is not None: states.append(StateZip(full_path, m.group(0))) diff --git a/azurelinuxagent/common/utils/cryptutil.py b/azurelinuxagent/common/utils/cryptutil.py index dcd31521ac..77b2520152 100644 --- a/azurelinuxagent/common/utils/cryptutil.py +++ b/azurelinuxagent/common/utils/cryptutil.py @@ -44,12 +44,12 @@ def gen_transport_cert(self, prv_file, crt_file): cmd = ("{0} req -x509 -nodes -subj /CN=LinuxTransport -days 730 " "-newkey rsa:2048 -keyout {1} " "-out {2}").format(self.openssl_cmd, prv_file, crt_file) - rc = shellutil.run(cmd) + rc = shellutil.run(cmd) # pylint: disable=C0103 if rc != 0: logger.error("Failed to create {0} and {1} certificates".format(prv_file, crt_file)) def get_pubkey_from_prv(self, file_name): - if not os.path.exists(file_name): + if not os.path.exists(file_name): # pylint: disable=R1720 raise IOError(errno.ENOENT, "File not found", file_name) else: cmd = [self.openssl_cmd, "rsa", "-in", file_name, "-pubout"] @@ -57,7 +57,7 @@ def get_pubkey_from_prv(self, file_name): return pub def get_pubkey_from_crt(self, file_name): - if not os.path.exists(file_name): + if not os.path.exists(file_name): # pylint: disable=R1720 raise IOError(errno.ENOENT, "File not found", file_name) else: cmd = [self.openssl_cmd, "x509", "-in", file_name, "-pubkey", "-noout"] @@ -65,7 +65,7 @@ def get_pubkey_from_crt(self, file_name): return pub def get_thumbprint_from_crt(self, file_name): - if not os.path.exists(file_name): + if not os.path.exists(file_name): # pylint: disable=R1720 raise IOError(errno.ENOENT, "File not found", file_name) else: cmd = [self.openssl_cmd, "x509", "-in", file_name, "-fingerprint", "-noout"] @@ -74,7 +74,7 @@ def get_thumbprint_from_crt(self, file_name): return thumbprint def decrypt_p7m(self, p7m_file, trans_prv_file, trans_cert_file, pem_file): - if not os.path.exists(p7m_file): + if not os.path.exists(p7m_file): # pylint: disable=R1720 raise IOError(errno.ENOENT, "File not found", p7m_file) elif not os.path.exists(trans_prv_file): raise IOError(errno.ENOENT, "File not found", trans_prv_file) @@ -84,7 +84,7 @@ def decrypt_p7m(self, p7m_file, trans_prv_file, trans_cert_file, pem_file): "").format(self.openssl_cmd, p7m_file, trans_prv_file, trans_cert_file, self.openssl_cmd, pem_file) shellutil.run(cmd) - rc = shellutil.run(cmd) + rc = shellutil.run(cmd) # pylint: disable=C0103 if rc != 0: logger.error("Failed to decrypt {0}".format(p7m_file)) @@ -97,13 +97,13 @@ def asn1_to_ssh(self, pubkey): lines = [x for x in lines if not x.startswith("----")] base64_encoded = "".join(lines) try: - #TODO remove pyasn1 dependency + #TODO remove pyasn1 dependency # pylint: disable=W0511 from pyasn1.codec.der import decoder as der_decoder der_encoded = base64.b64decode(base64_encoded) - der_encoded = der_decoder.decode(der_encoded)[0][1] + der_encoded = der_decoder.decode(der_encoded)[0][1] # pylint: disable=unsubscriptable-object key = der_decoder.decode(self.bits_to_bytes(der_encoded))[0] - n=key[0] - e=key[1] + n=key[0] # pylint: disable=C0103,unsubscriptable-object + e=key[1] # pylint: disable=C0103,unsubscriptable-object keydata = bytearray() keydata.extend(struct.pack('>I', len("ssh-rsa"))) keydata.extend(b"ssh-rsa") @@ -115,7 +115,7 @@ def asn1_to_ssh(self, pubkey): keydata_base64 = base64.b64encode(bytebuffer(keydata)) return ustr(b"ssh-rsa " + keydata_base64 + b"\n", encoding='utf-8') - except ImportError as e: + except ImportError as e: # pylint: disable=C0103 raise CryptError("Failed to load pyasn1.codec.der") def num_to_bytes(self, num): @@ -149,12 +149,12 @@ def decrypt_secret(self, encrypted_password, private_key): try: decoded = base64.b64decode(encrypted_password) args = DECRYPT_SECRET_CMD.format(self.openssl_cmd, private_key).split(' ') - p = subprocess.Popen(args, stdout=subprocess.PIPE, stdin=subprocess.PIPE, stderr=subprocess.STDOUT) + p = subprocess.Popen(args, stdout=subprocess.PIPE, stdin=subprocess.PIPE, stderr=subprocess.STDOUT) # pylint: disable=C0103 p.stdin.write(decoded) output = p.communicate()[0] retcode = p.poll() if retcode: raise subprocess.CalledProcessError(retcode, "openssl cms -decrypt", output=output) return output.decode('utf-16') - except Exception as e: + except Exception as e: # pylint: disable=C0103 raise CryptError("Error decoding secret", e) diff --git a/azurelinuxagent/common/utils/extensionprocessutil.py b/azurelinuxagent/common/utils/extensionprocessutil.py index 7c3f5b62d5..6aaffe7ba9 100644 --- a/azurelinuxagent/common/utils/extensionprocessutil.py +++ b/azurelinuxagent/common/utils/extensionprocessutil.py @@ -19,9 +19,9 @@ from azurelinuxagent.common.exception import ExtensionErrorCodes, ExtensionOperationError, ExtensionError from azurelinuxagent.common.future import ustr -import os -import signal -import time +import os # pylint: disable=C0411 +import signal # pylint: disable=C0411 +import time # pylint: disable=C0411 TELEMETRY_MESSAGE_MAX_LEN = 3200 @@ -50,7 +50,7 @@ def wait_for_process_completion_or_timeout(process, timeout): return timeout == 0, return_code -def handle_process_completion(process, command, timeout, stdout, stderr, error_code): +def handle_process_completion(process, command, timeout, stdout, stderr, error_code): # pylint: disable=R0913 """ Utility function that waits for process completion and retrieves its output (stdout and stderr) if it completed before the timeout period. Otherwise, the process will get killed and an ExtensionError will be raised. @@ -95,7 +95,7 @@ def read_output(stdout, stderr): errors='backslashreplace') return format_stdout_stderr(stdout, stderr) - except Exception as e: + except Exception as e: # pylint: disable=C0103 return format_stdout_stderr("", "Cannot read stdout/stderr: {0}".format(ustr(e))) @@ -125,10 +125,10 @@ def format_stdout_stderr(stdout, stderr): return '' def to_s(captured_stdout, stdout_offset, captured_stderr, stderr_offset): - s = template.format(captured_stdout[stdout_offset:], captured_stderr[stderr_offset:]) + s = template.format(captured_stdout[stdout_offset:], captured_stderr[stderr_offset:]) # pylint: disable=C0103 return s - if len(stdout) + len(stderr) < max_len: + if len(stdout) + len(stderr) < max_len: # pylint: disable=R1705 return to_s(stdout, 0, stderr, 0) elif len(stdout) < max_len_each: bonus = max_len_each - len(stdout) diff --git a/azurelinuxagent/common/utils/fileutil.py b/azurelinuxagent/common/utils/fileutil.py index 85146c2011..a734c396e2 100644 --- a/azurelinuxagent/common/utils/fileutil.py +++ b/azurelinuxagent/common/utils/fileutil.py @@ -21,7 +21,7 @@ File operation util functions """ -import errno as errno +import errno as errno # pylint: disable=C0414 import glob import os import pwd @@ -85,7 +85,7 @@ def append_file(filepath, contents, asbin=False, encoding='utf-8'): def base_name(path): - head, tail = os.path.split(path) + head, tail = os.path.split(path) # pylint: disable=W0612 return tail @@ -135,11 +135,11 @@ def rm_dirs(*args): """ Remove the contents of each directory """ - for p in args: + for p in args: # pylint: disable=C0103 if not os.path.isdir(p): continue - for pp in os.listdir(p): + for pp in os.listdir(p): # pylint: disable=C0103 path = os.path.join(p, pp) if os.path.isfile(path): os.remove(path) @@ -167,7 +167,7 @@ def update_conf_file(path, line_start, val, chk_err=False): def search_file(target_dir_name, target_file_name): - for root, dirs, files in os.walk(target_dir_name): + for root, dirs, files in os.walk(target_dir_name): # pylint: disable=W0612 for file_name in files: if file_name == target_file_name: return os.path.join(root, file_name) @@ -175,7 +175,7 @@ def search_file(target_dir_name, target_file_name): def chmod_tree(path, mode): - for root, dirs, files in os.walk(path): + for root, dirs, files in os.walk(path): # pylint: disable=W0612 for file_name in files: os.chmod(os.path.join(root, file_name), mode) @@ -186,7 +186,7 @@ def findstr_in_file(file_path, line_str): (Trailing whitespace is ignored.) """ try: - with open(file_path, 'r') as fh: + with open(file_path, 'r') as fh: # pylint: disable=C0103 for line in fh.readlines(): if line_str == line.rstrip(): return True @@ -201,13 +201,13 @@ def findre_in_file(file_path, line_re): Return match object if found in file. """ try: - with open(file_path, 'r') as fh: + with open(file_path, 'r') as fh: # pylint: disable=C0103 pattern = re.compile(line_re) for line in fh.readlines(): match = re.search(pattern, line) if match: return match - except: + except: # pylint: disable=W0702 pass return None @@ -218,13 +218,13 @@ def get_all_files(root_path): Find all files under the given root path """ result = [] - for root, dirs, files in os.walk(root_path): - result.extend([os.path.join(root, file) for file in files]) + for root, dirs, files in os.walk(root_path): # pylint: disable=W0612 + result.extend([os.path.join(root, file) for file in files]) # pylint: disable=redefined-builtin return result -def clean_ioerror(e, paths=[]): +def clean_ioerror(e, paths=[]): # pylint: disable=W0102,C0103 """ Clean-up possibly bad files and directories after an IO error. The code ignores *all* errors since disk state may be unhealthy. diff --git a/azurelinuxagent/common/utils/flexible_version.py b/azurelinuxagent/common/utils/flexible_version.py index 0cbf2aec4b..b058de7e9a 100644 --- a/azurelinuxagent/common/utils/flexible_version.py +++ b/azurelinuxagent/common/utils/flexible_version.py @@ -17,11 +17,11 @@ # Requires Python 2.6+ and Openssl 1.0+ # -from distutils import version +from distutils import version # pylint: disable=no-name-in-module import re -class FlexibleVersion(version.Version): +class FlexibleVersion(version.Version): # pylint: disable=R0902 """ A more flexible implementation of distutils.version.StrictVersion @@ -40,7 +40,7 @@ class FlexibleVersion(version.Version): http://stackoverflow.com/questions/12255554/sort-versions-in-python """ - def __init__(self, vstring=None, sep='.', prerel_tags=('alpha', 'beta', 'rc')): + def __init__(self, vstring=None, sep='.', prerel_tags=('alpha', 'beta', 'rc')): # pylint: disable=R1711 version.Version.__init__(self) if sep is None: @@ -71,7 +71,7 @@ def __init__(self, vstring=None, sep='.', prerel_tags=('alpha', 'beta', 'rc')): @property def major(self): - return self.version[0] if len(self.version) > 0 else 0 + return self.version[0] if len(self.version) > 0 else 0 # pylint: disable=len-as-condition @property def minor(self): @@ -81,8 +81,8 @@ def minor(self): def patch(self): return self.version[2] if len(self.version) > 2 else 0 - def _parse(self, vstring): - m = self.version_re.match(vstring) + def _parse(self, vstring): # pylint: disable=R1711 + m = self.version_re.match(vstring) # pylint: disable=C0103 if not m: raise ValueError("Invalid version number '{0}'".format(vstring)) @@ -94,19 +94,19 @@ def _parse(self, vstring): tag_num = m.group(self._nn_prerel_num) if tag is not None and tag_num is not None: - self.prerelease = (tag, int(tag_num) if len(tag_num) else None) + self.prerelease = (tag, int(tag_num) if len(tag_num) else None) # pylint: disable=C1801 self.version = tuple(map(int, self.sep_re.split(m.group(self._nn_version)))) return def __add__(self, increment): - version = list(self.version) + version = list(self.version) # pylint: disable=W0621 version[-1] += increment vstring = self._assemble(version, self.sep, self.prerel_sep, self.prerelease) return FlexibleVersion(vstring=vstring, sep=self.sep, prerel_tags=self.prerel_tags) def __sub__(self, decrement): - version = list(self.version) + version = list(self.version) # pylint: disable=W0621 if version[-1] <= 0: raise ArithmeticError("Cannot decrement final numeric component of {0} below zero" \ .format(self)) @@ -179,27 +179,27 @@ def matches(self, that): return True - def _assemble(self, version, sep, prerel_sep, prerelease): - s = sep.join(map(str, version)) + def _assemble(self, version, sep, prerel_sep, prerelease): # pylint: disable=W0621 + s = sep.join(map(str, version)) # pylint: disable=C0103 if prerelease is not None: if prerel_sep is not None: - s += prerel_sep - s += prerelease[0] + s += prerel_sep # pylint: disable=C0103 + s += prerelease[0] # pylint: disable=C0103 if prerelease[1] is not None: - s += str(prerelease[1]) + s += str(prerelease[1]) # pylint: disable=C0103 return s - def _compile_pattern(self): + def _compile_pattern(self): # pylint: disable=R1711 sep, self.sep_re = self._compile_separator(self.sep) if self.prerel_tags: tags = '|'.join(re.escape(tag) for tag in self.prerel_tags) self.prerel_tags_set = dict(zip(self.prerel_tags, range(len(self.prerel_tags)))) - release_re = '(?:{prerel_sep}(?P<{tn}>{tags})(?P<{nn}>\d*))?'.format( - prerel_sep=self._re_prerel_sep, - tags=tags, - tn=self._nn_prerel_tag, - nn=self._nn_prerel_num) + release_re = '(?:{prerel_sep}(?P<{tn}>{tags})(?P<{nn}>\d*))?'.format( # pylint: disable=W1401 + prerel_sep=self._re_prerel_sep, # pylint: disable=C0330 + tags=tags, # pylint: disable=C0330 + tn=self._nn_prerel_tag, # pylint: disable=C0330 + nn=self._nn_prerel_num) # pylint: disable=C0330 else: release_re = '' diff --git a/azurelinuxagent/common/utils/networkutil.py b/azurelinuxagent/common/utils/networkutil.py index 68ba08ac51..df4a05f050 100644 --- a/azurelinuxagent/common/utils/networkutil.py +++ b/azurelinuxagent/common/utils/networkutil.py @@ -22,7 +22,7 @@ class RouteEntry(object): Represents a single route. The destination, gateway, and mask members are hex representations of the IPv4 address in network byte order. """ - def __init__(self, interface, destination, gateway, mask, flags, metric): + def __init__(self, interface, destination, gateway, mask, flags, metric): # pylint: disable=R0913 self.interface = interface self.destination = destination self.gateway = gateway @@ -49,12 +49,12 @@ def mask_quad(self): return self._net_hex_to_dotted_quad(self.mask) def to_json(self): - f = '{{"Iface": "{0}", "Destination": "{1}", "Gateway": "{2}", "Mask": "{3}", "Flags": "{4:#06x}", "Metric": "{5}"}}' + f = '{{"Iface": "{0}", "Destination": "{1}", "Gateway": "{2}", "Mask": "{3}", "Flags": "{4:#06x}", "Metric": "{5}"}}' # pylint: disable=C0103 return f.format(self.interface, self.destination_quad(), self.gateway_quad(), self.mask_quad(), - self.flags, self.metric) + self.flags, self.metric) # pylint: disable=C0330 def __str__(self): - f = "Iface: {0}\tDestination: {1}\tGateway: {2}\tMask: {3}\tFlags: {4:#06x}\tMetric: {5}" + f = "Iface: {0}\tDestination: {1}\tGateway: {2}\tMask: {3}\tFlags: {4:#06x}\tMetric: {5}" # pylint: disable=C0103 return f.format(self.interface, self.destination_quad(), self.gateway_quad(), self.mask_quad(), self.flags, self.metric) @@ -88,8 +88,8 @@ def _json_array(items): def __str__(self): entries = ['"name": "{0}"'.format(self.name), '"link": "{0}"'.format(self.link)] - if len(self.ipv4) > 0: + if len(self.ipv4) > 0: # pylint: disable=len-as-condition entries.append('"ipv4": {0}'.format(self._json_array(self.ipv4))) - if len(self.ipv6) > 0: + if len(self.ipv6) > 0: # pylint: disable=len-as-condition entries.append('"ipv6": {0}'.format(self._json_array(self.ipv6))) return "{{ {0} }}".format(", ".join(entries)) diff --git a/azurelinuxagent/common/utils/restutil.py b/azurelinuxagent/common/utils/restutil.py index 29b5b5fbeb..7ffffc28a5 100644 --- a/azurelinuxagent/common/utils/restutil.py +++ b/azurelinuxagent/common/utils/restutil.py @@ -135,11 +135,11 @@ def set_protocol_endpoint(endpoint=KNOWN_WIRESERVER_IP): IOErrorCounter._protocol_endpoint = endpoint -def _is_retry_status(status, retry_codes=RETRY_CODES): +def _is_retry_status(status, retry_codes=RETRY_CODES): # pylint: disable=W0102 return status in retry_codes -def _is_retry_exception(e): +def _is_retry_exception(e): # pylint: disable=C0103 return len([x for x in RETRY_EXCEPTIONS if isinstance(e, x)]) > 0 @@ -152,7 +152,7 @@ def _parse_url(url): Parse URL to get the components of the URL broken down to host, port :rtype: string, int, bool, string """ - o = urlparse(url) + o = urlparse(url) # pylint: disable=C0103 rel_uri = o.path if o.fragment: rel_uri = "{0}#{1}".format(rel_uri, o.fragment) @@ -196,7 +196,7 @@ def dotted_netmask(mask): return socket.inet_ntoa(struct.pack('>I', bits)) -def address_in_network(ip, net): +def address_in_network(ip, net): # pylint: disable=C0103 """This function allows you to check if an IP belongs to a network subnet Example: returns True if ip = 192.168.1.1 and net = 192.168.1.0/24 returns False if ip = 192.168.1.1 and net = 192.168.100.0/24 @@ -264,7 +264,7 @@ def _get_http_proxy(secure=False): else: http_proxy_env = HTTPS_PROXY_ENV if secure else HTTP_PROXY_ENV http_proxy_url = None - for v in [http_proxy_env, http_proxy_env.upper()]: + for v in [http_proxy_env, http_proxy_env.upper()]: # pylint: disable=C0103 if v in os.environ: http_proxy_url = os.environ[v] break @@ -279,7 +279,7 @@ def redact_sas_tokens_in_urls(url): return SAS_TOKEN_RETRIEVAL_REGEX.sub(r"\1" + REDACTED_TEXT + r"\3", url) -def _http_request(method, host, rel_uri, port=None, data=None, secure=False, +def _http_request(method, host, rel_uri, port=None, data=None, secure=False, # pylint: disable=R0913 headers=None, proxy_host=None, proxy_port=None): headers = {} if headers is None else headers @@ -323,14 +323,14 @@ def _http_request(method, host, rel_uri, port=None, data=None, secure=False, return conn.getresponse() -def http_request(method, - url, data, headers=None, - use_proxy=False, - max_retry=DEFAULT_RETRIES, - retry_codes=RETRY_CODES, - retry_delay=DELAY_IN_SECONDS): +def http_request(method, # pylint: disable=R0913,R0912,R0914,W0102 + url, data, headers=None, # pylint: disable=C0330 + use_proxy=False, # pylint: disable=C0330 + max_retry=DEFAULT_RETRIES, # pylint: disable=C0330 + retry_codes=RETRY_CODES, # pylint: disable=C0330 + retry_delay=DELAY_IN_SECONDS): # pylint: disable=C0330 - global SECURE_WARNING_EMITTED + global SECURE_WARNING_EMITTED # pylint: disable=W0603 host, port, secure, rel_uri = _parse_url(url) @@ -382,11 +382,11 @@ def http_request(method, delay = THROTTLE_DELAY_IN_SECONDS if was_throttled else retry_delay logger.verbose("[HTTP Retry] " - "Attempt {0} of {1} will delay {2} seconds: {3}", - attempt+1, - max_retry, - delay, - msg) + "Attempt {0} of {1} will delay {2} seconds: {3}", # pylint: disable=C0330 + attempt+1, # pylint: disable=C0330 + max_retry, # pylint: disable=C0330 + delay, # pylint: disable=C0330 + msg) # pylint: disable=C0330 time.sleep(delay) @@ -429,14 +429,14 @@ def http_request(method, return resp - except httpclient.HTTPException as e: + except httpclient.HTTPException as e: # pylint: disable=C0103 clean_url = redact_sas_tokens_in_urls(url) msg = '[HTTP Failed] {0} {1} -- HttpException {2}'.format(method, clean_url, e) if _is_retry_exception(e): continue break - except IOError as e: + except IOError as e: # pylint: disable=C0103 IOErrorCounter.increment(host=host, port=port) clean_url = redact_sas_tokens_in_urls(url) msg = '[HTTP Failed] {0} {1} -- IOError {2}'.format(method, clean_url, e) @@ -445,7 +445,7 @@ def http_request(method, raise HttpError("{0} -- {1} attempts made".format(msg, attempt)) -def http_get(url, +def http_get(url, # pylint: disable=R0913,W0102 headers=None, use_proxy=False, max_retry=DEFAULT_RETRIES, @@ -460,7 +460,7 @@ def http_get(url, retry_delay=retry_delay) -def http_head(url, +def http_head(url, # pylint: disable=R0913,W0102 headers=None, use_proxy=False, max_retry=DEFAULT_RETRIES, @@ -475,7 +475,7 @@ def http_head(url, retry_delay=retry_delay) -def http_post(url, +def http_post(url, # pylint: disable=R0913,W0102 data, headers=None, use_proxy=False, @@ -491,7 +491,7 @@ def http_post(url, retry_delay=retry_delay) -def http_put(url, +def http_put(url, # pylint: disable=R0913,W0102 data, headers=None, use_proxy=False, @@ -507,7 +507,7 @@ def http_put(url, retry_delay=retry_delay) -def http_delete(url, +def http_delete(url, # pylint: disable=R0913,W0102 headers=None, use_proxy=False, max_retry=DEFAULT_RETRIES, @@ -522,11 +522,11 @@ def http_delete(url, retry_delay=retry_delay) -def request_failed(resp, ok_codes=OK_CODES): +def request_failed(resp, ok_codes=OK_CODES): # pylint: disable=W0102 return not request_succeeded(resp, ok_codes=ok_codes) -def request_succeeded(resp, ok_codes=OK_CODES): +def request_succeeded(resp, ok_codes=OK_CODES): # pylint: disable=W0102 return resp is not None and resp.status in ok_codes @@ -534,7 +534,7 @@ def request_not_modified(resp): return resp is not None and resp.status in NOT_MODIFIED_CODES -def request_failed_at_hostplugin(resp, upstream_failure_codes=HOSTPLUGIN_UPSTREAM_FAILURE_CODES): +def request_failed_at_hostplugin(resp, upstream_failure_codes=HOSTPLUGIN_UPSTREAM_FAILURE_CODES): # pylint: disable=W0102 """ Host plugin will return 502 for any upstream issue, so a failure is any 5xx except 502 """ @@ -546,9 +546,9 @@ def read_response_error(resp): if resp is not None: try: result = "[HTTP Failed] [{0}: {1}] {2}".format( - resp.status, - resp.reason, - resp.read()) + resp.status, # pylint: disable=C0330 + resp.reason, # pylint: disable=C0330 + resp.read()) # pylint: disable=C0330 # this result string is passed upstream to several methods # which do a raise HttpError() or a format() of some kind; diff --git a/azurelinuxagent/common/utils/shellutil.py b/azurelinuxagent/common/utils/shellutil.py index 66a79396ac..6649d02760 100644 --- a/azurelinuxagent/common/utils/shellutil.py +++ b/azurelinuxagent/common/utils/shellutil.py @@ -40,7 +40,7 @@ def check_output(*popenargs, **kwargs): # Exception classes used by this module. class CalledProcessError(Exception): - def __init__(self, returncode, cmd, output=None): + def __init__(self, returncode, cmd, output=None): # pylint: disable=W0231 self.returncode = returncode self.cmd = cmd self.output = output @@ -53,9 +53,11 @@ def __str__(self): subprocess.check_output = check_output subprocess.CalledProcessError = CalledProcessError +# pylint: disable=W0105 """ Shell command util functions -""" +""" +# pylint: enable=W0105 def has_command(cmd): """ @@ -63,18 +65,18 @@ def has_command(cmd): """ return not run(cmd, False) -def run(cmd, chk_err=True, expected_errors=[]): +def run(cmd, chk_err=True, expected_errors=[]): # pylint: disable=W0102 """ Note: Deprecating in favour of `azurelinuxagent.common.utils.shellutil.run_command` function. Calls run_get_output on 'cmd', returning only the return code. If chk_err=True then errors will be reported in the log. If chk_err=False then errors will be suppressed from the log. """ - retcode, out = run_get_output(cmd, chk_err=chk_err, expected_errors=expected_errors) + retcode, out = run_get_output(cmd, chk_err=chk_err, expected_errors=expected_errors) # pylint: disable=W0612 return retcode -def run_get_output(cmd, chk_err=True, log_cmd=True, expected_errors=[]): +def run_get_output(cmd, chk_err=True, log_cmd=True, expected_errors=[]): # pylint: disable=W0102 """ Wrapper for subprocess.check_output. Execute 'cmd'. Returns return code and STDOUT, trapping expected @@ -91,7 +93,7 @@ def run_get_output(cmd, chk_err=True, log_cmd=True, expected_errors=[]): stderr=subprocess.STDOUT, shell=True) output = _encode_command_output(output) - except subprocess.CalledProcessError as e: + except subprocess.CalledProcessError as e: # pylint: disable=C0103 output = _encode_command_output(e.output) if chk_err: @@ -103,7 +105,7 @@ def run_get_output(cmd, chk_err=True, log_cmd=True, expected_errors=[]): else: logger.error(msg) return e.returncode, output - except Exception as e: + except Exception as e: # pylint: disable=C0103 if chk_err: logger.error(u"Command [{0}] raised unexpected exception: [{1}]" .format(cmd, ustr(e))) @@ -121,11 +123,11 @@ class CommandError(Exception): """ @staticmethod def _get_message(command, return_code, stderr): - command_name = command[0] if isinstance(command, list) and len(command) > 0 else command + command_name = command[0] if isinstance(command, list) and len(command) > 0 else command # pylint: disable=len-as-condition return "'{0}' failed: {1} ({2})".format(command_name, return_code, stderr.rstrip()) def __init__(self, command, return_code, stdout, stderr): - super(Exception, self).__init__(CommandError._get_message(command, return_code, stderr)) + super(Exception, self).__init__(CommandError._get_message(command, return_code, stderr)) # pylint: disable=E1003 self.command = command self.returncode = return_code self.stdout = stdout @@ -154,7 +156,7 @@ def format_command(cmd): process = subprocess.Popen(command, stdin=stdin, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=False) stdout, stderr = process.communicate(input=process_input) returncode = process.returncode - except Exception as e: + except Exception as e: # pylint: disable=C0103 if log_error: logger.error(u"Command [{0}] raised unexpected exception: [{1}]", format_command(command), ustr(e)) raise diff --git a/azurelinuxagent/common/utils/textutil.py b/azurelinuxagent/common/utils/textutil.py index f7171c2df3..ae54973531 100644 --- a/azurelinuxagent/common/utils/textutil.py +++ b/azurelinuxagent/common/utils/textutil.py @@ -47,7 +47,7 @@ def findall(root, tag, namespace=None): if root is None: return [] - if namespace is None: + if namespace is None: # pylint: disable=R1705 return root.getElementsByTagName(tag) else: return root.getElementsByTagNameNS(namespace, tag) @@ -58,7 +58,7 @@ def find(root, tag, namespace=None): Get first node by tag and namespace under Node root. """ nodes = findall(root, tag, namespace=namespace) - if nodes is not None and len(nodes) >= 1: + if nodes is not None and len(nodes) >= 1: # pylint: disable=R1705 return nodes[0] else: return None @@ -89,13 +89,13 @@ def getattrib(node, attr_name): """ Get attribute of xml node """ - if node is not None: + if node is not None: # pylint: disable=R1705 return node.getAttribute(attr_name) else: return None -def unpack(buf, offset, range): +def unpack(buf, offset, range): # pylint: disable=W0622 """ Unpack bytes into python values. """ @@ -133,14 +133,14 @@ def hex_dump2(buf): return hex_dump3(buf, 0, len(buf)) -def is_in_range(a, low, high): +def is_in_range(a, low, high): # pylint: disable=C0103 """ Return True if 'a' in 'low' <= a >= 'high' """ - return (a >= low and a <= high) + return (a >= low and a <= high) # pylint: disable=R1716 -def is_printable(ch): +def is_printable(ch): # pylint: disable=C0103 """ Return True if character is displayable. """ @@ -149,7 +149,7 @@ def is_printable(ch): or is_in_range(ch, str_to_ord('0'), str_to_ord('9'))) -def hex_dump(buffer, size): +def hex_dump(buffer, size): # pylint: disable=redefined-builtin """ Return Hex formated dump of a 'buffer' of 'size'. """ @@ -160,7 +160,7 @@ def hex_dump(buffer, size): if (i % 16) == 0: result += "%06X: " % i byte = buffer[i] - if type(byte) == str: + if type(byte) == str: # pylint: disable=C0123 byte = ord(byte.decode('latin1')) result += "%02X " % byte if (i & 15) == 7: @@ -175,7 +175,7 @@ def hex_dump(buffer, size): result += " " for j in range(i - (i % 16), i + 1): byte = buffer[j] - if type(byte) == str: + if type(byte) == str: # pylint: disable=C0123 byte = str_to_ord(byte.decode('latin1')) k = '.' if is_printable(byte): @@ -186,24 +186,24 @@ def hex_dump(buffer, size): return result -def str_to_ord(a): +def str_to_ord(a): # pylint: disable=C0103 """ Allows indexing into a string or an array of integers transparently. Generic utility function. """ - if type(a) == type(b'') or type(a) == type(u''): + if type(a) == type(b'') or type(a) == type(u''): # pylint: disable=C0123 a = ord(a) return a -def compare_bytes(a, b, start, length): +def compare_bytes(a, b, start, length): # pylint: disable=C0103 for offset in range(start, start + length): if str_to_ord(a[offset]) != str_to_ord(b[offset]): return False return True -def int_to_ip4_addr(a): +def int_to_ip4_addr(a): # pylint: disable=C0103 """ Build DHCP request string. """ @@ -213,13 +213,13 @@ def int_to_ip4_addr(a): (a) & 0xFF) -def hexstr_to_bytearray(a): +def hexstr_to_bytearray(a): # pylint: disable=C0103 """ Return hex string packed into a binary struct. """ - b = b"" - for c in range(0, len(a) // 2): - b += struct.pack("B", int(a[c * 2:c * 2 + 2], 16)) + b = b"" # pylint: disable=C0103 + for c in range(0, len(a) // 2): # pylint: disable=C0103 + b += struct.pack("B", int(a[c * 2:c * 2 + 2], 16)) # pylint: disable=C0103 return b @@ -228,7 +228,7 @@ def set_ssh_config(config, name, val): no_match = -1 match_start = no_match - for i in range(0, len(config)): + for i in range(0, len(config)): # pylint: disable=C0200 if config[i].startswith(name) and match_start == no_match: config[i] = "{0} {1}".format(name, val) found = True @@ -248,7 +248,7 @@ def set_ssh_config(config, name, val): def set_ini_config(config, name, val): notfound = True - nameEqual = name + '=' + nameEqual = name + '=' # pylint: disable=C0103 length = len(config) text = "{0}=\"{1}\"".format(name, val) @@ -265,7 +265,7 @@ def set_ini_config(config, name, val): def replace_non_ascii(incoming, replace_char=''): outgoing = '' if incoming is not None: - for c in incoming: + for c in incoming: # pylint: disable=C0103 if str_to_ord(c) > 128: outgoing += replace_char else: @@ -273,7 +273,7 @@ def replace_non_ascii(incoming, replace_char=''): return outgoing -def remove_bom(c): +def remove_bom(c): # pylint: disable=C0103 """ bom is comprised of a sequence of three chars,0xef, 0xbb, 0xbf, in case of utf-8. """ @@ -304,7 +304,7 @@ def get_bytes_from_pem(pem_str): return base64_bytes -def compress(s): +def compress(s): # pylint: disable=C0103 """ Compress a string, and return the base64 encoded result of the compression. @@ -319,21 +319,21 @@ def compress(s): return base64.b64encode(zlib.compress(s)) -def b64encode(s): +def b64encode(s): # pylint: disable=C0103 from azurelinuxagent.common.version import PY_VERSION_MAJOR if PY_VERSION_MAJOR > 2: return base64.b64encode(bytes(s, 'utf-8')).decode('utf-8') return base64.b64encode(s) -def b64decode(s): +def b64decode(s): # pylint: disable=C0103 from azurelinuxagent.common.version import PY_VERSION_MAJOR if PY_VERSION_MAJOR > 2: return base64.b64decode(s).decode('utf-8') return base64.b64decode(s) -def safe_shlex_split(s): +def safe_shlex_split(s): # pylint: disable=C0103 import shlex from azurelinuxagent.common.version import PY_VERSION if PY_VERSION[:2] == (2, 6): @@ -341,16 +341,16 @@ def safe_shlex_split(s): return shlex.split(s) -def swap_hexstring(s, width=2): - r = len(s) % width +def swap_hexstring(s, width=2): # pylint: disable=C0103 + r = len(s) % width # pylint: disable=C0103 if r != 0: s = ('0' * (width - (len(s) % width))) + s return ''.join(reversed( - re.findall( - r'[a-f0-9]{{{0}}}'.format(width), - s, - re.IGNORECASE))) + re.findall( # pylint: disable=C0330 + r'[a-f0-9]{{{0}}}'.format(width), # pylint: disable=C0330 + s, # pylint: disable=C0330 + re.IGNORECASE))) # pylint: disable=C0330 def parse_json(json_str): @@ -366,11 +366,11 @@ def parse_json(json_str): return result -def is_str_none_or_whitespace(s): +def is_str_none_or_whitespace(s): # pylint: disable=C0103 return s is None or len(s) == 0 or s.isspace() -def is_str_empty(s): +def is_str_empty(s): # pylint: disable=C0103 return is_str_none_or_whitespace(s) or is_str_none_or_whitespace(s.rstrip(' \t\r\n\0')) @@ -399,7 +399,7 @@ def format_memory_value(unit, value): return int(value * units[unit]) -def str_to_encoded_ustr(s, encoding='utf-8'): +def str_to_encoded_ustr(s, encoding='utf-8'): # pylint: disable=C0103 """ This function takes the string and converts it into the corresponding encoded ustr if its not already a ustr. The encoding is utf-8 by default if not specified. @@ -409,16 +409,16 @@ def str_to_encoded_ustr(s, encoding='utf-8'): :return: Returns the corresponding ustr string. Returns None if input is None. """ - # TODO: Import at the top of the file instead of a local import (using local import here to avoid cyclic dependency) + # TODO: Import at the top of the file instead of a local import (using local import here to avoid cyclic dependency) # pylint: disable=W0511 from azurelinuxagent.common.version import PY_VERSION_MAJOR - if s is None or type(s) is ustr: + if s is None or type(s) is ustr: # pylint: disable=C0123 # If its already a ustr/None then return as is return s if PY_VERSION_MAJOR > 2: try: # For py3+, str() is unicode by default - if isinstance(s, bytes): + if isinstance(s, bytes): # pylint: disable=R1705 # str.encode() returns bytes which should be decoded to get the str. return s.decode(encoding) else: diff --git a/azurelinuxagent/common/version.py b/azurelinuxagent/common/version.py index 0423cfadaf..9c46d4aed7 100644 --- a/azurelinuxagent/common/version.py +++ b/azurelinuxagent/common/version.py @@ -21,7 +21,6 @@ import sys import azurelinuxagent.common.conf as conf -import azurelinuxagent.common.utils.fileutil as fileutil from azurelinuxagent.common.utils.flexible_version import FlexibleVersion from azurelinuxagent.common.future import ustr, get_linux_distribution @@ -33,10 +32,10 @@ def get_f5_platform(): the version and product information is contained in the /VERSION file. """ result = [None, None, None, None] - f5_version = re.compile("^Version: (\d+\.\d+\.\d+)") - f5_product = re.compile("^Product: ([\w-]+)") + f5_version = re.compile("^Version: (\d+\.\d+\.\d+)") # pylint: disable=W1401 + f5_product = re.compile("^Product: ([\w-]+)") # pylint: disable=W1401 - with open('/VERSION', 'r') as fh: + with open('/VERSION', 'r') as fh: # pylint: disable=C0103 content = fh.readlines() for line in content: version_matches = f5_version.match(line) @@ -60,10 +59,10 @@ def get_f5_platform(): def get_checkpoint_platform(): take = build = release = "" full_name = open("/etc/cp-release").read().strip() - with open("/etc/cloud-version") as f: + with open("/etc/cloud-version") as f: # pylint: disable=C0103 for line in f: - k, _, v = line.partition(": ") - v = v.strip() + k, _, v = line.partition(": ") # pylint: disable=C0103 + v = v.strip() # pylint: disable=C0103 if k == "release": release = v elif k == "take": @@ -75,21 +74,21 @@ def get_checkpoint_platform(): def get_distro(): if 'FreeBSD' in platform.system(): - release = re.sub('\-.*\Z', '', ustr(platform.release())) + release = re.sub('\-.*\Z', '', ustr(platform.release())) # pylint: disable=W1401 osinfo = ['freebsd', release, '', 'freebsd'] elif 'OpenBSD' in platform.system(): - release = re.sub('\-.*\Z', '', ustr(platform.release())) + release = re.sub('\-.*\Z', '', ustr(platform.release())) # pylint: disable=W1401 osinfo = ['openbsd', release, '', 'openbsd'] elif 'Linux' in platform.system(): osinfo = get_linux_distribution(0, 'alpine') elif 'NS-BSD' in platform.system(): - release = re.sub('\-.*\Z', '', ustr(platform.release())) + release = re.sub('\-.*\Z', '', ustr(platform.release())) # pylint: disable=W1401 osinfo = ['nsbsd', release, '', 'nsbsd'] else: try: # dist() removed in Python 3.8 - osinfo = list(platform.dist()) + [''] - except: + osinfo = list(platform.dist()) + [''] # pylint: disable=W1505,E1101 + except: # pylint: disable=W0702 osinfo = ['UNKNOWN', 'FFFF', '', ''] # The platform.py lib has issue with detecting oracle linux distribution. @@ -132,13 +131,13 @@ def get_distro(): AGENT_PATTERN = "{0}-(.*)".format(AGENT_NAME) AGENT_NAME_PATTERN = re.compile(AGENT_PATTERN) -AGENT_PKG_PATTERN = re.compile(AGENT_PATTERN+"\.zip") +AGENT_PKG_PATTERN = re.compile(AGENT_PATTERN+"\.zip") # pylint: disable=W1401 AGENT_DIR_PATTERN = re.compile(".*/{0}".format(AGENT_PATTERN)) # The execution mode of the VM - IAAS or PAAS. Linux VMs are only executed in IAAS mode. AGENT_EXECUTION_MODE = "IAAS" -EXT_HANDLER_PATTERN = b".*/WALinuxAgent-(\d+.\d+.\d+[.\d+]*).*-run-exthandlers" +EXT_HANDLER_PATTERN = b".*/WALinuxAgent-(\d+.\d+.\d+[.\d+]*).*-run-exthandlers" # pylint: disable=W1401 EXT_HANDLER_REGEX = re.compile(EXT_HANDLER_PATTERN) __distro__ = get_distro()