Run pylint on Python 3.11 (#3067)

* Run pylint on Python 3.11

---------

Co-authored-by: narrieta <narrieta>

.github/workflows/ci_pr.yml

@@ -9,7 +9,7 @@ on:
 
 jobs:
   test-python-2_6-and-3_4-versions:
-    
+
     strategy:
       fail-fast: false
       matrix:
@@ -87,32 +87,19 @@ jobs:
       matrix:
         include:
           - python-version: "3.5"
-            PYLINTOPTS: "--rcfile=ci/3.6.pylintrc --ignore=tests_e2e,makepkg.py"
-
           - python-version: "3.6"
-            PYLINTOPTS: "--rcfile=ci/3.6.pylintrc --ignore=tests_e2e"
-
           - python-version: "3.7"
-            PYLINTOPTS: "--rcfile=ci/3.6.pylintrc --ignore=tests_e2e"
-
           - python-version: "3.8"
-            PYLINTOPTS: "--rcfile=ci/3.6.pylintrc --ignore=tests_e2e"
-
           - python-version: "3.9"
-            PYLINTOPTS: "--rcfile=ci/3.6.pylintrc"
             additional-nose-opts: "--with-coverage --cover-erase --cover-inclusive --cover-branches --cover-package=azurelinuxagent"
-
           - python-version: "3.10"
-            PYLINTOPTS: "--rcfile=ci/3.10.pylintrc --ignore=tests"
+          - python-version: "3.11"
 
     name: "Python ${{ matrix.python-version }} Unit Tests"
     runs-on: ubuntu-20.04
 
     env:
-      PYLINTOPTS: ${{ matrix.PYLINTOPTS }}
-      PYLINTFILES: "azurelinuxagent setup.py makepkg.py tests tests_e2e"
       NOSEOPTS: "--with-timer ${{ matrix.additional-nose-opts }}"
-      PYTHON_VERSION: ${{ matrix.python-version }}
 
     steps:
 
@@ -130,13 +117,46 @@ jobs:
         sudo env "PATH=$PATH" python -m pip install --upgrade pip
         sudo env "PATH=$PATH" pip install -r requirements.txt
         sudo env "PATH=$PATH" pip install -r test-requirements.txt
+        sudo env "PATH=$PATH" pip install --upgrade pylint
 
     - name: Run pylint
       run: |
-        pylint $PYLINTOPTS --jobs=0 $PYLINTFILES
+        #
+        # List of files/directories to be checked by pylint.
+        # The end-to-end tests run only on Python 3.9 and we lint them only on that version.
+        #
+        PYLINT_FILES="azurelinuxagent setup.py makepkg.py tests"
+        if [[ "${{ matrix.python-version }}" == "3.9" ]]; then
+          PYLINT_FILES="$PYLINT_FILES tests_e2e"
+        fi
+
+        #
+        # Command-line options for pylint.
+        # * "unused-private-member" is not implemented on 3.5 and will produce "E0012: Bad option value 'unused-private-member' (bad-option-value)"
+        #   so we suppress "bad-option-value".
+        # * 3.9 will produce "no-member" for several properties/methods that are added to the mocks used by the unit tests (e.g
+        #   "E1101: Instance of 'WireProtocol' has no 'aggregate_status' member") so we suppress that warning.
+        # * 'no-self-use' ("R0201: Method could be a function") was moved to an optional extension on 3.9 and is no longer used by default. It needs
+        #    to be suppressed for previous versions (3.0-3.8), though.
+        #
+        PYLINT_OPTIONS="--rcfile=ci/pylintrc --jobs=0"
+        if [[ "${{ matrix.python-version }}" == "3.5" ]]; then
+          PYLINT_OPTIONS="$PYLINT_OPTIONS --disable=bad-option-value"
+        fi
+        if [[ "${{ matrix.python-version }}" == "3.9" ]]; then
+          PYLINT_OPTIONS="$PYLINT_OPTIONS --disable=no-member"
+        fi
+        if [[ "${{ matrix.python-version }}" =~ ^3\.[0-8]$ ]]; then
+          PYLINT_OPTIONS="$PYLINT_OPTIONS --disable=no-self-use"
+        fi
+
+        echo "PYLINT_OPTIONS: $PYLINT_OPTIONS"
+        echo "PYLINT_FILES: $PYLINT_FILES"
+
+        pylint $PYLINT_OPTIONS $PYLINT_FILES
 
     - name: Test with nosetests
-      if: matrix.python-version != '3.10' && (success() || (failure() && steps.install-dependencies.outcome == 'success'))
+      if: contains(fromJSON('["3.10", "3.11"]'), matrix.python-version) == false && (success() || (failure() && steps.install-dependencies.outcome == 'success'))
       run: |
         ./ci/nosetests.sh
         exit $?

azurelinuxagent/common/utils/textutil.py

@@ -17,7 +17,8 @@
 # Requires Python 2.6+ and Openssl 1.0+
 
 import base64
-import crypt
+# W4901: Deprecated module 'crypt' (deprecated-module)
+import crypt  # pylint: disable=deprecated-module
 import hashlib
 import random
 import re

azurelinuxagent/ga/cgroupconfigurator.py

@@ -350,17 +350,19 @@ def __reload_systemd_config():
             except Exception as exception:
                 _log_cgroup_warning("daemon-reload failed (create azure slice): {0}", ustr(exception))
 
+        # W0238: Unused private member `_Impl.__create_unit_file(path, contents)` (unused-private-member)
         @staticmethod
-        def __create_unit_file(path, contents):
+        def __create_unit_file(path, contents):  # pylint: disable=unused-private-member
             parent, _ = os.path.split(path)
             if not os.path.exists(parent):
                 fileutil.mkdir(parent, mode=0o755)
             exists = os.path.exists(path)
             fileutil.write_file(path, contents)
             _log_cgroup_info("{0} {1}", "Updated" if exists else "Created", path)
 
+        # W0238: Unused private member `_Impl.__cleanup_unit_file(path)` (unused-private-member)
         @staticmethod
-        def __cleanup_unit_file(path):
+        def __cleanup_unit_file(path):  # pylint: disable=unused-private-member
             if os.path.exists(path):
                 try:
                     os.remove(path)
@@ -522,8 +524,9 @@ def __reset_agent_cpu_quota():
                 _log_cgroup_info('CPUQuota: {0}',
                                  systemd.get_unit_property(systemd.get_agent_unit_name(), "CPUQuotaPerSecUSec"))
 
+        # W0238: Unused private member `_Impl.__try_set_cpu_quota(quota)` (unused-private-member)
         @staticmethod
-        def __try_set_cpu_quota(quota):
+        def __try_set_cpu_quota(quota):  # pylint: disable=unused-private-member
             try:
                 drop_in_file = os.path.join(systemd.get_agent_drop_in_path(), _DROP_IN_FILE_CPU_QUOTA)
                 contents = _DROP_IN_FILE_CPU_QUOTA_CONTENTS_FORMAT.format(quota)

ci/3.10.pylintrc → ci/pylintrc

@@ -16,7 +16,6 @@ disable=C, # (C) convention, for programming standard violation
     no-else-continue, # R1724: *Unnecessary "%s" after "continue"*
     no-else-raise, # R1720: *Unnecessary "%s" after "raise"*
     no-else-return, # R1705: *Unnecessary "%s" after "return"*
-    no-self-use, # R0201: Method could be a function
     protected-access, # W0212: Access to a protected member of a client class
     raise-missing-from, # W0707: *Consider explicitly re-raising using the 'from' keyword*
     redundant-u-string-prefix, # The u prefix for strings is no longer necessary in Python >=3.0

tests/common/test_event.py

@@ -60,7 +60,7 @@ def setUp(self):
 
         self.event_dir = os.path.join(self.tmp_dir, EVENTS_DIRECTORY)
         EventLoggerTools.initialize_event_logger(self.event_dir)
-        threading.current_thread().setName("TestEventThread")
+        threading.current_thread().name = "TestEventThread"
         osutil = get_osutil()
 
         self.expected_common_parameters = {
@@ -70,7 +70,7 @@ def setUp(self):
             CommonTelemetryEventSchema.ContainerId: AgentGlobals.get_container_id(),
             CommonTelemetryEventSchema.EventTid: threading.current_thread().ident,
             CommonTelemetryEventSchema.EventPid: os.getpid(),
-            CommonTelemetryEventSchema.TaskName: threading.current_thread().getName(),
+            CommonTelemetryEventSchema.TaskName: threading.current_thread().name,
             CommonTelemetryEventSchema.KeywordName: json.dumps({"CpuArchitecture": platform.machine()}),
             # common parameters computed from the OS platform
             CommonTelemetryEventSchema.OSVersion: EventLoggerTools.get_expected_os_version(),

tests/common/test_singletonperthread.py

@@ -1,6 +1,6 @@
 import uuid
 from multiprocessing import Queue
-from threading import Thread, currentThread
+from threading import Thread, current_thread
 
 from azurelinuxagent.common.singletonperthread import SingletonPerThread
 from tests.lib.tools import AgentTestCase, clear_singleton_instances
@@ -32,7 +32,7 @@ class TestClassToTestSingletonPerThread(SingletonPerThread):
 
     def __init__(self):
         # Set the name of the object to the current thread name
-        self.name = currentThread().getName()
+        self.name = current_thread().name
         # Unique identifier for a class object
         self.uuid = str(uuid.uuid4())
 
@@ -53,8 +53,8 @@ def _setup_multithread_and_execute(self, func1, args1, func2, args2, t1_name=Non
 
         t1 = Thread(target=func1, args=args1)
         t2 = Thread(target=func2, args=args2)
-        t1.setName(t1_name if t1_name else self.THREAD_NAME_1)
-        t2.setName(t2_name if t2_name else self.THREAD_NAME_2)
+        t1.name = t1_name if t1_name else self.THREAD_NAME_1
+        t2.name = t2_name if t2_name else self.THREAD_NAME_2
         t1.start()
         t2.start()
         t1.join()

tests/ga/test_multi_config_extension.py

@@ -41,7 +41,7 @@ def __init__(self, name, version, state="enabled"):
             self.version = version
             self.state = state
             self.is_invalid_setting = False
-            self.settings = dict()
+            self.settings = {}
 
     class _TestExtensionObject:
         def __init__(self, name, seq_no, dependency_level="0", state="enabled"):
@@ -94,12 +94,11 @@ def _get_mock_expected_handler_data(self, rc_extensions, vmaccess_extensions, ge
     def test_it_should_parse_multi_config_settings_properly(self):
         self.test_data['ext_conf'] = os.path.join(self._MULTI_CONFIG_TEST_DATA, "ext_conf_with_multi_config.xml")
 
-        rc_extensions = dict()
-        rc_extensions["firstRunCommand"] = self._TestExtensionObject(name="firstRunCommand", seq_no=2)
-        rc_extensions["secondRunCommand"] = self._TestExtensionObject(name="secondRunCommand", seq_no=2,
-                                                                      dependency_level="3")
-        rc_extensions["thirdRunCommand"] = self._TestExtensionObject(name="thirdRunCommand", seq_no=1,
-                                                                     dependency_level="4")
+        rc_extensions = {
+            "firstRunCommand": self._TestExtensionObject(name="firstRunCommand", seq_no=2),
+            "secondRunCommand": self._TestExtensionObject(name="secondRunCommand", seq_no=2, dependency_level="3"),
+            "thirdRunCommand": self._TestExtensionObject(name="thirdRunCommand", seq_no=1, dependency_level="4")
+        }
 
         vmaccess_extensions = {
             "Microsoft.Compute.VMAccessAgent": self._TestExtensionObject(name="Microsoft.Compute.VMAccessAgent",
@@ -115,12 +114,11 @@ def test_it_should_parse_multi_config_with_disable_state_properly(self):
         self.test_data['ext_conf'] = os.path.join(self._MULTI_CONFIG_TEST_DATA,
                                                   "ext_conf_with_disabled_multi_config.xml")
 
-        rc_extensions = dict()
-        rc_extensions["firstRunCommand"] = self._TestExtensionObject(name="firstRunCommand", seq_no=3)
-        rc_extensions["secondRunCommand"] = self._TestExtensionObject(name="secondRunCommand", seq_no=3,
-                                                                      dependency_level="1")
-        rc_extensions["thirdRunCommand"] = self._TestExtensionObject(name="thirdRunCommand", seq_no=1,
-                                                                     dependency_level="4", state="disabled")
+        rc_extensions = {
+            "firstRunCommand": self._TestExtensionObject(name="firstRunCommand", seq_no=3),
+            "secondRunCommand": self._TestExtensionObject(name="secondRunCommand", seq_no=3, dependency_level="1"),
+            "thirdRunCommand": self._TestExtensionObject(name="thirdRunCommand", seq_no=1, dependency_level="4", state="disabled")
+        }
 
         vmaccess_extensions = {
             "Microsoft.Compute.VMAccessAgent": self._TestExtensionObject(name="Microsoft.Compute.VMAccessAgent",

tests/ga/test_remoteaccess_handler.py

@@ -75,15 +75,14 @@ def mock_add_event(name, op, is_success, version, message):
 
 
 class TestRemoteAccessHandler(AgentTestCase):
-    eventing_data = [()]
+    eventing_data = ()
 
     def setUp(self):
         super(TestRemoteAccessHandler, self).setUp()
         # Since ProtocolUtil is a singleton per thread, we need to clear it to ensure that the test cases do not
         # reuse a previous state
         clear_singleton_instances(ProtocolUtil)
-        for data in TestRemoteAccessHandler.eventing_data:
-            del data
+        TestRemoteAccessHandler.eventing_data = ()
 
     # add_user tests
     @patch('azurelinuxagent.common.utils.cryptutil.CryptUtil.decrypt_secret', return_value="]aPPEv}uNg1FPnl?")

tests/lib/tools.py

@@ -29,7 +29,7 @@
 import time
 import unittest
 from functools import wraps
-from threading import currentThread
+from threading import current_thread
 
 import azurelinuxagent.common.conf as conf
 import azurelinuxagent.common.event as event
@@ -543,6 +543,6 @@ def wrapper(self, *args, **kwargs):
 def clear_singleton_instances(cls):
     # Adding this lock to avoid any race conditions
     with cls._lock:
-        obj_name = "%s__%s" % (cls.__name__, currentThread().getName())  # Object Name = className__threadName
+        obj_name = "%s__%s" % (cls.__name__, current_thread().name)  # Object Name = className__threadName
         if obj_name in cls._instances:
             del cls._instances[obj_name]