From 013a365a26063eed75f5b0f0af607d6218bcb961 Mon Sep 17 00:00:00 2001 From: keclar Date: Fri, 4 Sep 2020 02:08:19 +0000 Subject: [PATCH 1/7] Fixed linter errors for undefined variables. --- azurelinuxagent/common/future.py | 18 +++++++++++++++++- azurelinuxagent/common/osutil/clearlinux.py | 11 ++++------- azurelinuxagent/common/osutil/gaia.py | 6 +++--- azurelinuxagent/common/osutil/iosxe.py | 17 +++++++++-------- azurelinuxagent/common/osutil/openwrt.py | 2 +- azurelinuxagent/daemon/resourcedisk/openwrt.py | 4 ++-- azurelinuxagent/pa/deprovision/default.py | 3 +++ tests/protocol/test_hostplugin.py | 9 +-------- tests/tools.py | 6 ++---- 9 files changed, 42 insertions(+), 34 deletions(-) diff --git a/azurelinuxagent/common/future.py b/azurelinuxagent/common/future.py index 79e6e3ba12..9b57398eae 100644 --- a/azurelinuxagent/common/future.py +++ b/azurelinuxagent/common/future.py @@ -31,11 +31,27 @@ import httplib as httpclient # pylint: disable=E0401,W0611 from urlparse import urlparse # pylint: disable=E0401 - """Rename Python2 unicode to ustr""" # pylint: disable=W0105 + # Rename Python2 unicode to ustr + # This is not defined in python3, and the linter will thus + # throw an undefined-variable error on this line. + # Suppress it here. ustr = unicode # pylint: disable=E0602,invalid-name + # This is not defined in python3, and the linter will thus + # throw an undefined-variable error on this line. + # Suppress it here. bytebuffer = buffer # pylint: disable=E0602,invalid-name + # This is not defined in python3, and the linter will thus + # throw an undefined-variable error on this line. + # Suppress it here. + range = xrange # pylint: disable=E0602,invalid-name,redefined-builtin + + # This is not defined in python3, and the linter will thus + # throw an undefined-variable error on this line. + # Suppress it here. + int = long # pylint: disable=E0602,invalid-name,redefined-builtin + if sys.version_info[1] >= 7: from collections import OrderedDict # For Py 2.7+ # pylint: disable=C0412 else: diff --git a/azurelinuxagent/common/osutil/clearlinux.py b/azurelinuxagent/common/osutil/clearlinux.py index dd2decffed..c3d2bb3ba0 100644 --- a/azurelinuxagent/common/osutil/clearlinux.py +++ b/azurelinuxagent/common/osutil/clearlinux.py @@ -32,6 +32,7 @@ import azurelinuxagent.common.utils.shellutil as shellutil import azurelinuxagent.common.utils.textutil as textutil # pylint: disable=W0611 from azurelinuxagent.common.osutil.default import DefaultOSUtil +from azurelinuxagent.common.exception import OSUtilError class ClearLinuxUtil(DefaultOSUtil): @@ -75,12 +76,8 @@ def conf_sshd(self, disable_password): def del_root_password(self): try: passwd_file_path = conf.get_passwd_file_path() - try: - passwd_content = fileutil.read_file(passwd_file_path) - if not passwd_content: - # Empty file is no better than no file - raise FileNotFoundError # pylint: disable=undefined-variable - except FileNotFoundError: # pylint: disable=undefined-variable + passwd_content = fileutil.read_file(passwd_file_path) + if not passwd_content: new_passwd = ["root:*LOCK*:14600::::::"] else: passwd = passwd_content.split('\n') @@ -88,5 +85,5 @@ def del_root_password(self): new_passwd.insert(0, "root:*LOCK*:14600::::::") fileutil.write_file(passwd_file_path, "\n".join(new_passwd)) except IOError as e: # pylint: disable=C0103 - raise OSUtilError("Failed to delete root password:{0}".format(e)) # pylint: disable=E0602 + raise OSUtilError("Failed to delete root password:{0}".format(e)) pass # pylint: disable=W0107 diff --git a/azurelinuxagent/common/osutil/gaia.py b/azurelinuxagent/common/osutil/gaia.py index 20d80a911c..afae08861b 100644 --- a/azurelinuxagent/common/osutil/gaia.py +++ b/azurelinuxagent/common/osutil/gaia.py @@ -23,7 +23,7 @@ import azurelinuxagent.common.conf as conf from azurelinuxagent.common.exception import OSUtilError -from azurelinuxagent.common.future import ustr, bytebuffer +from azurelinuxagent.common.future import ustr, bytebuffer, range, int # pylint: disable=redefined-builtin import azurelinuxagent.common.logger as logger from azurelinuxagent.common.osutil.default import DefaultOSUtil from azurelinuxagent.common.utils.cryptutil import CryptUtil @@ -40,7 +40,7 @@ def __init__(self): # pylint: disable=W0235 def _run_clish(self, cmd): ret = 0 out = "" - for i in xrange(10): # pylint: disable=E0602,W0612 + for i in range(10): # pylint: disable=W0612 try: final_command = ["/bin/clish", "-s", "-c", "'{0}'".format(cmd)] out = shellutil.run_command(final_command, log_error=True) @@ -122,7 +122,7 @@ def openssl_to_openssh(self, input_file, output_file): def text_to_num(buf): if len(buf) == 1: return int(buf[0].split()[1]) - return long(''.join(buf[1:]), 16) # pylint: disable=E0602 + return int(''.join(buf[1:]), 16) n = text_to_num(modulus) # pylint: disable=C0103 e = text_to_num(exponent) # pylint: disable=C0103 diff --git a/azurelinuxagent/common/osutil/iosxe.py b/azurelinuxagent/common/osutil/iosxe.py index 0c49484072..a1dd93db55 100644 --- a/azurelinuxagent/common/osutil/iosxe.py +++ b/azurelinuxagent/common/osutil/iosxe.py @@ -17,12 +17,13 @@ # Requires Python 2.6+ and Openssl 1.0+ # +import os + import azurelinuxagent.common.logger as logger import azurelinuxagent.common.utils.shellutil as shellutil from azurelinuxagent.common.future import ustr -from azurelinuxagent.common.osutil.default import DefaultOSUtil -from azurelinuxagent.common.osutil.redhat import Redhat6xOSUtil # pylint: disable=W0611 -from azurelinuxagent.common.utils import textutil # pylint: disable=W0611 +from azurelinuxagent.common.osutil.default import DefaultOSUtil, PRODUCT_ID_FILE, DMIDECODE_CMD, UUID_PATTERN +from azurelinuxagent.common.utils import textutil, fileutil # pylint: disable=W0611 # pylint: disable=W0105 ''' @@ -58,7 +59,7 @@ def publish_hostname(self, hostname): Restart NetworkManager first before publishing hostname """ shellutil.run("service NetworkManager restart") - super(RedhatOSUtil, self).publish_hostname(hostname) # pylint: disable=E0602 + super(IosxeOSUtil, self).publish_hostname(hostname) def register_agent_service(self): return shellutil.run("systemctl enable waagent", chk_err=False) @@ -79,13 +80,13 @@ def get_instance_id(self): If that is missing, then extracts from dmidecode If nothing works (for old VMs), return the empty string ''' - if os.path.isfile(PRODUCT_ID_FILE): # pylint: disable=E0602 + if os.path.isfile(PRODUCT_ID_FILE): try: - s = fileutil.read_file(PRODUCT_ID_FILE).strip() # pylint: disable=E0602,C0103 + s = fileutil.read_file(PRODUCT_ID_FILE).strip() # pylint: disable=C0103 return self._correct_instance_id(s.strip()) except IOError: pass - rc, s = shellutil.run_get_output(DMIDECODE_CMD) # pylint: disable=E0602,C0103 - if rc != 0 or UUID_PATTERN.match(s) is None: # pylint: disable=E0602 + rc, s = shellutil.run_get_output(DMIDECODE_CMD) # pylint: disable=C0103 + if rc != 0 or UUID_PATTERN.match(s) is None: return "" return self._correct_instance_id(s.strip()) diff --git a/azurelinuxagent/common/osutil/openwrt.py b/azurelinuxagent/common/osutil/openwrt.py index d070878aad..0a52e78dca 100644 --- a/azurelinuxagent/common/osutil/openwrt.py +++ b/azurelinuxagent/common/osutil/openwrt.py @@ -126,7 +126,7 @@ def restart_ssh_service(self): # pylint: disable=R1710 if os.path.exists("/etc/init.d/sshd"): # pylint: disable=R1705 return shellutil.run("/etc/init.d/sshd restart", chk_err=True) else: - logger.warn("sshd service does not exists", username) # pylint: disable=E0602 + logger.warn("sshd service does not exists") def stop_agent_service(self): return shellutil.run("/etc/init.d/{0} stop".format(self.service_name), chk_err=True) diff --git a/azurelinuxagent/daemon/resourcedisk/openwrt.py b/azurelinuxagent/daemon/resourcedisk/openwrt.py index 6bc568adf5..308a4fe1d6 100644 --- a/azurelinuxagent/daemon/resourcedisk/openwrt.py +++ b/azurelinuxagent/daemon/resourcedisk/openwrt.py @@ -18,7 +18,7 @@ # Requires Python 2.6+ and Openssl 1.0+ # import os -import errno as errno # pylint: disable=C0414,W0611 +from time import sleep import azurelinuxagent.common.logger as logger import azurelinuxagent.common.utils.fileutil as fileutil @@ -94,7 +94,7 @@ def mount_resource_disk(self, mount_point): # pylint: disable=R0914 logger.info("Waiting for partition [{0}], {1} attempts remaining", partition, attempts) - sleep(5) # pylint: disable=E0602 + sleep(5) attempts -= 1 if not os.path.exists(partition): diff --git a/azurelinuxagent/pa/deprovision/default.py b/azurelinuxagent/pa/deprovision/default.py index 6cda1a5394..d3cf845e13 100644 --- a/azurelinuxagent/pa/deprovision/default.py +++ b/azurelinuxagent/pa/deprovision/default.py @@ -38,6 +38,9 @@ def read_input(message): if sys.version_info[0] >= 3: # pylint: disable=R1705 return input(message) else: + # This is not defined in python3, and the linter will thus + # throw an undefined-variable error on this line. + # Suppress it here. return raw_input(message) # pylint: disable=E0602 class DeprovisionAction(object): # pylint: disable=R0903 diff --git a/tests/protocol/test_hostplugin.py b/tests/protocol/test_hostplugin.py index b51f7daa31..baf515786d 100644 --- a/tests/protocol/test_hostplugin.py +++ b/tests/protocol/test_hostplugin.py @@ -27,7 +27,7 @@ import azurelinuxagent.common.protocol.wire as wire from azurelinuxagent.common.errorstate import ErrorState from azurelinuxagent.common.exception import HttpError, ResourceGoneError -from azurelinuxagent.common.future import ustr +from azurelinuxagent.common.future import ustr, httpclient from azurelinuxagent.common.osutil.default import UUID_PATTERN from azurelinuxagent.common.protocol.hostplugin import API_VERSION from azurelinuxagent.common.utils import restutil @@ -37,13 +37,6 @@ from tests.protocol.test_wire import MockResponse as TestWireMockResponse from tests.tools import AgentTestCase, PY_VERSION_MAJOR, Mock, patch -if sys.version_info[0] == 3: - import http.client as httpclient # pylint: disable=import-error - bytebuffer = memoryview # pylint: disable=invalid-name -elif sys.version_info[0] == 2: - import httplib as httpclient # pylint: disable=import-error - bytebuffer = buffer # pylint: disable=undefined-variable,invalid-name - hostplugin_status_url = "http://168.63.129.16:32526/status" # pylint: disable=invalid-name hostplugin_versions_url = "http://168.63.129.16:32526/versions" # pylint: disable=invalid-name diff --git a/tests/tools.py b/tests/tools.py index aaad50de4f..128b658dfa 100644 --- a/tests/tools.py +++ b/tests/tools.py @@ -34,9 +34,7 @@ import azurelinuxagent.common.conf as conf import azurelinuxagent.common.event as event import azurelinuxagent.common.logger as logger -from azurelinuxagent.common.cgroupconfigurator import CGroupConfigurator # pylint: disable=unused-import -from azurelinuxagent.common.osutil.factory import _get_osutil # pylint: disable=unused-import -from azurelinuxagent.common.osutil.ubuntu import Ubuntu14OSUtil, Ubuntu16OSUtil # pylint: disable=unused-import +from azurelinuxagent.common.future import range # pylint: disable=redefined-builtin from azurelinuxagent.common.utils import fileutil from azurelinuxagent.common.version import PY_VERSION_MAJOR @@ -358,7 +356,7 @@ def emulate_assertListEqual(self, seq1, seq2, msg=None, seq_type=None): # pylint elements = (seq_type_name.capitalize(), seq1_repr, seq2_repr) differing = '%ss differ: %s != %s\n' % elements - for i in xrange(min(len1, len2)): # pylint: disable=undefined-variable + for i in range(min(len1, len2)): try: item1 = seq1[i] except (TypeError, IndexError, NotImplementedError): From e67ff5fbe56eac00006606d20843347281d73511 Mon Sep 17 00:00:00 2001 From: keclar Date: Fri, 4 Sep 2020 02:24:11 +0000 Subject: [PATCH 2/7] Fixed python3 import errors. --- azurelinuxagent/common/future.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/azurelinuxagent/common/future.py b/azurelinuxagent/common/future.py index 9b57398eae..ed125fb03e 100644 --- a/azurelinuxagent/common/future.py +++ b/azurelinuxagent/common/future.py @@ -25,6 +25,10 @@ bytebuffer = memoryview # pylint: disable=C0103 + range = range + + int = int + from collections import OrderedDict # pylint: disable=W0611 elif sys.version_info[0] == 2: From e9a3ab8e9e8997e77f7d38acbec92636e22cb052 Mon Sep 17 00:00:00 2001 From: keclar Date: Fri, 4 Sep 2020 02:33:03 +0000 Subject: [PATCH 3/7] Fixed linter errors. --- azurelinuxagent/common/future.py | 48 +++++++++++++++++--------------- 1 file changed, 25 insertions(+), 23 deletions(-) diff --git a/azurelinuxagent/common/future.py b/azurelinuxagent/common/future.py index ed125fb03e..adfa5f0a38 100644 --- a/azurelinuxagent/common/future.py +++ b/azurelinuxagent/common/future.py @@ -25,9 +25,11 @@ bytebuffer = memoryview # pylint: disable=C0103 - range = range - - int = int + # We aren't using these imports in this file, but we want them to be available + # to import from this module in others. + # Additionally, python2 doesn't have this, so we need to disable import-error + # as well. + from builtins import int, range # pylint: disable=unused-import,import-error from collections import OrderedDict # pylint: disable=W0611 @@ -35,26 +37,26 @@ import httplib as httpclient # pylint: disable=E0401,W0611 from urlparse import urlparse # pylint: disable=E0401 - # Rename Python2 unicode to ustr - # This is not defined in python3, and the linter will thus - # throw an undefined-variable error on this line. - # Suppress it here. - ustr = unicode # pylint: disable=E0602,invalid-name - - # This is not defined in python3, and the linter will thus - # throw an undefined-variable error on this line. - # Suppress it here. - bytebuffer = buffer # pylint: disable=E0602,invalid-name - - # This is not defined in python3, and the linter will thus - # throw an undefined-variable error on this line. - # Suppress it here. - range = xrange # pylint: disable=E0602,invalid-name,redefined-builtin - - # This is not defined in python3, and the linter will thus - # throw an undefined-variable error on this line. - # Suppress it here. - int = long # pylint: disable=E0602,invalid-name,redefined-builtin + + # We want to suppress the following: + # - undefined-variable: + # These builtins are not defined in python3 + # - invalid-name: + # The defined variables are constants, but don't use UPPER_SNAKE_CASE + # as we're redefining some builtins that also do not use that format. + # - redefined-builtin: + # This is intentional, so that code that wants to use builtins we're + # assigning new names to doesn't need to check python versions before + # doing so. + + # pylint: disable=undefined-variable,invalid-name,redefined-builtin + + ustr = unicode # Rename Python2 unicode to ustr + bytebuffer = buffer + range = xrange + int = long + + # pylint: enable=undefined-variable,invalid-name,redefined-builtin if sys.version_info[1] >= 7: from collections import OrderedDict # For Py 2.7+ # pylint: disable=C0412 From fce88f1f59825430447ffdd4c6a60350ea527f8c Mon Sep 17 00:00:00 2001 From: keclar Date: Fri, 4 Sep 2020 03:23:50 +0000 Subject: [PATCH 4/7] removed line from pylintrcs. --- ci/2.7.pylintrc | 1 - ci/3.6.pylintrc | 1 - 2 files changed, 2 deletions(-) diff --git a/ci/2.7.pylintrc b/ci/2.7.pylintrc index 168e991578..374394fa15 100644 --- a/ci/2.7.pylintrc +++ b/ci/2.7.pylintrc @@ -50,7 +50,6 @@ # too-many-nested-blocks: Used when a function or a method has too many nested blocks. # too-many-public-methods: Used when class has too many public methods, try to reduce this to get a simpler (and so easier to use) class. # too-many-return-statements: Used when a function or method has too many return statement, making it hard to follow. -# undefined-variable: (hi-pri) Used when an undefined variable is accessed. # ungrouped-imports: (needs review) Used when imports are not grouped by packages # unidiomatic-typecheck: (hi-pri) The idiomatic way to perform an explicit typecheck in Python is to use isinstance(x, Y) rather than type(x) == Y, type(x) is Y. # unnecessary-lambda: Used when the body of a lambda expression is a function call on the same argument list as the lambda itself diff --git a/ci/3.6.pylintrc b/ci/3.6.pylintrc index 13d25c21f6..b103dc97bc 100644 --- a/ci/3.6.pylintrc +++ b/ci/3.6.pylintrc @@ -62,7 +62,6 @@ # too-many-nested-blocks: Used when a function or a method has too many nested blocks. # too-many-public-methods: Used when class has too many public methods, try to reduce this to get a simpler (and so easier to use) class. # too-many-return-statements: Used when a function or method has too many return statement, making it hard to follow. -# undefined-variable: (hi-pri) Used when an undefined variable is accessed. # ungrouped-imports: (needs review) Used when imports are not grouped by packages # unidiomatic-typecheck: (hi-pri) The idiomatic way to perform an explicit typecheck in Python is to use isinstance(x, Y) rather than type(x) == Y, type(x) is Y. # unnecessary-comprehension: (hi-pri) Instead of using an identity comprehension, consider using the list, dict or set constructor. From 93462123e0256ec233f3bd6892f37c99fe98fd53 Mon Sep 17 00:00:00 2001 From: keclar Date: Fri, 4 Sep 2020 17:28:27 +0000 Subject: [PATCH 5/7] Fixed exception handling. --- azurelinuxagent/common/osutil/clearlinux.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/azurelinuxagent/common/osutil/clearlinux.py b/azurelinuxagent/common/osutil/clearlinux.py index c3d2bb3ba0..ad90373e87 100644 --- a/azurelinuxagent/common/osutil/clearlinux.py +++ b/azurelinuxagent/common/osutil/clearlinux.py @@ -76,8 +76,12 @@ def conf_sshd(self, disable_password): def del_root_password(self): try: passwd_file_path = conf.get_passwd_file_path() - passwd_content = fileutil.read_file(passwd_file_path) - if not passwd_content: + try: + passwd_content = fileutil.read_file(passwd_file_path) + if not passwd_content: + # Empty file is no better than no file + raise IOError + except (IOError, OSError): new_passwd = ["root:*LOCK*:14600::::::"] else: passwd = passwd_content.split('\n') From 8672bcee995c842aaee1648367cdfeba50573376 Mon Sep 17 00:00:00 2001 From: keclar Date: Tue, 15 Sep 2020 18:24:16 +0000 Subject: [PATCH 6/7] fixed change such that only FileNotFoundErrors are actually swallowed. --- azurelinuxagent/common/osutil/clearlinux.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/azurelinuxagent/common/osutil/clearlinux.py b/azurelinuxagent/common/osutil/clearlinux.py index ad90373e87..84fd67e6df 100644 --- a/azurelinuxagent/common/osutil/clearlinux.py +++ b/azurelinuxagent/common/osutil/clearlinux.py @@ -80,8 +80,10 @@ def del_root_password(self): passwd_content = fileutil.read_file(passwd_file_path) if not passwd_content: # Empty file is no better than no file - raise IOError - except (IOError, OSError): + raise IOError(os.errno.ENOENT, "Empty File", passwd_file_path) + except (IOError, OSError) as file_read_err: + if file_read_err.errno != os.errno.ENOENT: + raise new_passwd = ["root:*LOCK*:14600::::::"] else: passwd = passwd_content.split('\n') From 7f0707c5428892d2874a0a660706e4de37264481 Mon Sep 17 00:00:00 2001 From: keclar Date: Wed, 16 Sep 2020 17:31:04 +0000 Subject: [PATCH 7/7] Imported errno instead of using os.errno --- azurelinuxagent/common/osutil/clearlinux.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/azurelinuxagent/common/osutil/clearlinux.py b/azurelinuxagent/common/osutil/clearlinux.py index 471d448de1..27685696e1 100644 --- a/azurelinuxagent/common/osutil/clearlinux.py +++ b/azurelinuxagent/common/osutil/clearlinux.py @@ -26,6 +26,7 @@ import fcntl # pylint: disable=W0611 import time # pylint: disable=W0611 import base64 # pylint: disable=W0611 +import errno import azurelinuxagent.common.conf as conf import azurelinuxagent.common.logger as logger # pylint: disable=W0611 import azurelinuxagent.common.utils.fileutil as fileutil @@ -80,9 +81,9 @@ def del_root_password(self): passwd_content = fileutil.read_file(passwd_file_path) if not passwd_content: # Empty file is no better than no file - raise IOError(os.errno.ENOENT, "Empty File", passwd_file_path) + raise IOError(errno.ENOENT, "Empty File", passwd_file_path) except (IOError, OSError) as file_read_err: - if file_read_err.errno != os.errno.ENOENT: + if file_read_err.errno != errno.ENOENT: raise new_passwd = ["root:*LOCK*:14600::::::"] else: