From 5558a78798961e3953f5d9332296771d51197f26 Mon Sep 17 00:00:00 2001 From: nnandigam Date: Wed, 27 Mar 2024 09:47:25 -0700 Subject: [PATCH 1/6] check for unexpected process in cgroup before enable --- azurelinuxagent/ga/cgroupconfigurator.py | 81 ++++++++++++---- tests/ga/test_cgroupconfigurator.py | 18 ++++ tests_e2e/test_suites/agent_cgroups.yml | 4 +- .../agent_cgroups_process_check.py | 77 ++++++++++++++++ tests_e2e/tests/lib/cgroup_helpers.py | 14 ++- ...roups_process_check-cgroups_not_enabled.py | 60 ++++++++++++ ...ups_process_check-unknown_process_check.py | 92 +++++++++++++++++++ .../agent_cpu_quota-check_agent_cpu_quota.py | 46 +++------- 8 files changed, 336 insertions(+), 56 deletions(-) create mode 100644 tests_e2e/tests/agent_cgroups/agent_cgroups_process_check.py create mode 100755 tests_e2e/tests/scripts/agent_cgroups_process_check-cgroups_not_enabled.py create mode 100755 tests_e2e/tests/scripts/agent_cgroups_process_check-unknown_process_check.py diff --git a/azurelinuxagent/ga/cgroupconfigurator.py b/azurelinuxagent/ga/cgroupconfigurator.py index 09eb8b55ad..22e754d183 100644 --- a/azurelinuxagent/ga/cgroupconfigurator.py +++ b/azurelinuxagent/ga/cgroupconfigurator.py @@ -152,23 +152,9 @@ def initialize(self): return # This check is to reset the quotas if agent goes from cgroup supported to unsupported distros later in time. if not CGroupsApi.cgroups_supported(): - agent_drop_in_path = systemd.get_agent_drop_in_path() - try: - if os.path.exists(agent_drop_in_path) and os.path.isdir(agent_drop_in_path): - files_to_cleanup = [] - agent_drop_in_file_slice = os.path.join(agent_drop_in_path, _AGENT_DROP_IN_FILE_SLICE) - agent_drop_in_file_cpu_accounting = os.path.join(agent_drop_in_path, - _DROP_IN_FILE_CPU_ACCOUNTING) - agent_drop_in_file_memory_accounting = os.path.join(agent_drop_in_path, - _DROP_IN_FILE_MEMORY_ACCOUNTING) - agent_drop_in_file_cpu_quota = os.path.join(agent_drop_in_path, _DROP_IN_FILE_CPU_QUOTA) - files_to_cleanup.extend([agent_drop_in_file_slice, agent_drop_in_file_cpu_accounting, - agent_drop_in_file_memory_accounting, agent_drop_in_file_cpu_quota]) - self.__cleanup_all_files(files_to_cleanup) - self.__reload_systemd_config() - logger.info("Agent reset the quotas if distro: {0} goes from supported to unsupported list", get_distro()) - except Exception as err: - logger.warn("Unable to delete Agent drop-in files while resetting the quotas: {0}".format(err)) + logger.info("Agent reset the quotas if distro: {0} goes from supported to unsupported list", + get_distro()) + self._reset_agent_cgroup_setup() # check whether cgroup monitoring is supported on the current distro self._cgroups_supported = CGroupsApi.cgroups_supported() @@ -187,15 +173,21 @@ def initialize(self): if not self.__check_no_legacy_cgroups(): return + cpu_controller_root, memory_controller_root = self.__get_cgroup_controllers() agent_unit_name = systemd.get_agent_unit_name() agent_slice = systemd.get_unit_property(agent_unit_name, "Slice") + + if conf.get_cgroup_disable_on_process_check_failure() and self._check_processes_in_agent_cgroup_before_enable(agent_slice, cpu_controller_root, memory_controller_root): + reason = "Found unexpected processes in the agent cgroup before agent enable cgroups." + self.disable(reason, DisableCgroups.ALL) + return + if agent_slice not in (AZURE_SLICE, "system.slice"): _log_cgroup_warning("The agent is within an unexpected slice: {0}", agent_slice) return self.__setup_azure_slice() - cpu_controller_root, memory_controller_root = self.__get_cgroup_controllers() self._agent_cpu_cgroup_path, self._agent_memory_cgroup_path = self.__get_agent_cgroups(agent_slice, cpu_controller_root, memory_controller_root) @@ -341,6 +333,25 @@ def __setup_azure_slice(): CGroupConfigurator._Impl.__reload_systemd_config() + def _reset_agent_cgroup_setup(self): + try: + agent_drop_in_path = systemd.get_agent_drop_in_path() + if os.path.exists(agent_drop_in_path) and os.path.isdir(agent_drop_in_path): + files_to_cleanup = [] + agent_drop_in_file_slice = os.path.join(agent_drop_in_path, _AGENT_DROP_IN_FILE_SLICE) + agent_drop_in_file_cpu_accounting = os.path.join(agent_drop_in_path, + _DROP_IN_FILE_CPU_ACCOUNTING) + agent_drop_in_file_memory_accounting = os.path.join(agent_drop_in_path, + _DROP_IN_FILE_MEMORY_ACCOUNTING) + agent_drop_in_file_cpu_quota = os.path.join(agent_drop_in_path, _DROP_IN_FILE_CPU_QUOTA) + files_to_cleanup.extend([agent_drop_in_file_slice, agent_drop_in_file_cpu_accounting, + agent_drop_in_file_memory_accounting, agent_drop_in_file_cpu_quota]) + self.__cleanup_all_files(files_to_cleanup) + self.__reload_systemd_config() + except Exception as err: + logger.warn("Unable to delete Agent drop-in files while resetting the quotas: {0}".format(err)) + + @staticmethod def __reload_systemd_config(): # reload the systemd configuration; the new slices will be used once the agent's service restarts @@ -546,6 +557,35 @@ def __try_set_cpu_quota(quota): # pylint: disable=unused-private-member return False return True + def _check_processes_in_agent_cgroup_before_enable(self, agent_slice, cpu_root_cgroup_path, memory_root_cgroup_path): + """ + This check ensures that before we enable the agent's cgroups, there are no unexpected processes in the agent's cgroup already. + + The issue we observed that long running extension processes may be in agent cgroups if agent goes this cycle enabled(1)->disabled(2)->enabled(3). + 1. Agent cgroups enabled in some version + 2. Disabled agent cgroups due to check_cgroups regular check, now extension runs will be in agent cgroups. + 3. When ext_hanlder restart and enable the cgroups again, already running processes from step 2 still be in agent cgroups. This may cause the extensions run with agent limit. + """ + if agent_slice not in AZURE_SLICE: + return False + cpu_cgroup_relative_path, memory_cgroup_relative_path = self._cgroups_api.get_process_cgroup_paths("self") + agent_cpu_cgroup_path = os.path.join(cpu_root_cgroup_path, cpu_cgroup_relative_path) + agent_memory_cgroup_path = os.path.join(memory_root_cgroup_path, memory_cgroup_relative_path) + + check_cgroup_path = agent_cpu_cgroup_path if agent_cpu_cgroup_path is not None else agent_memory_cgroup_path + + if check_cgroup_path is None: + return False + + try: + _log_cgroup_info("Checking for unexpected processes in the agent's cgroup before enabling cgroups") + self._check_processes_in_agent_cgroup(check_cgroup_path) + except CGroupsException as exception: + _log_cgroup_warning(ustr(exception)) + return True + + return False + def check_cgroups(self, cgroup_metrics): self._check_cgroups_lock.acquire() try: @@ -579,7 +619,7 @@ def check_cgroups(self, cgroup_metrics): finally: self._check_cgroups_lock.release() - def _check_processes_in_agent_cgroup(self): + def _check_processes_in_agent_cgroup(self, cgroup_path=None): """ Verifies that the agent's cgroup includes only the current process, its parent, commands started using shellutil and instances of systemd-run (those processes correspond, respectively, to the extension handler, the daemon, commands started by the extension handler, and the systemd-run @@ -591,6 +631,7 @@ def _check_processes_in_agent_cgroup(self): """ unexpected = [] agent_cgroup_proc_names = [] + check_cgroup_path = cgroup_path if cgroup_path is not None else self._agent_cpu_cgroup_path try: daemon = os.getppid() extension_handler = os.getpid() @@ -598,7 +639,7 @@ def _check_processes_in_agent_cgroup(self): agent_commands.update(shellutil.get_running_commands()) systemd_run_commands = set() systemd_run_commands.update(self._cgroups_api.get_systemd_run_commands()) - agent_cgroup = CGroupsApi.get_processes_in_cgroup(self._agent_cpu_cgroup_path) + agent_cgroup = CGroupsApi.get_processes_in_cgroup(check_cgroup_path) # get the running commands again in case new commands started or completed while we were fetching the processes in the cgroup; agent_commands.update(shellutil.get_running_commands()) systemd_run_commands.update(self._cgroups_api.get_systemd_run_commands()) diff --git a/tests/ga/test_cgroupconfigurator.py b/tests/ga/test_cgroupconfigurator.py index 82c86c956f..251734ce80 100644 --- a/tests/ga/test_cgroupconfigurator.py +++ b/tests/ga/test_cgroupconfigurator.py @@ -905,6 +905,24 @@ def test_check_cgroups_should_disable_cgroups_when_a_check_fails(self): for p in patchers: p.stop() + @patch('azurelinuxagent.ga.cgroupconfigurator.CGroupConfigurator._Impl._check_processes_in_agent_cgroup', side_effect=CGroupsException("Test")) + @patch('azurelinuxagent.ga.cgroupconfigurator.add_event') + def test_agent_not_enable_cgroups_if_unexpected_process_already_in_agent_cgroups(self, add_event, _): + command_mocks = [MockCommand(r"^systemctl show walinuxagent\.service --property Slice", +'''Slice=azure.slice +''')] + with self._get_cgroup_configurator(mock_commands=command_mocks) as configurator: + self.assertFalse(configurator.enabled(), "Cgroups should not be enabled") + + disable_events = [kwargs for _, kwargs in add_event.call_args_list if kwargs["op"] == WALAEventOperation.CGroupsDisabled] + self.assertTrue( + len(disable_events) == 1, + "Exactly 1 event should have been emitted. Got: {0}".format(disable_events)) + self.assertIn( + "Found unexpected processes in the agent cgroup before agent enable cgroups", + disable_events[0]["message"], + "The error message is not correct when process check failed") + def test_check_agent_memory_usage_should_raise_a_cgroups_exception_when_the_limit_is_exceeded(self): metrics = [MetricValue(MetricsCategory.MEMORY_CATEGORY, MetricsCounter.TOTAL_MEM_USAGE, AGENT_NAME_TELEMETRY, conf.get_agent_memory_quota() + 1), MetricValue(MetricsCategory.MEMORY_CATEGORY, MetricsCounter.SWAP_MEM_USAGE, AGENT_NAME_TELEMETRY, conf.get_agent_memory_quota() + 1)] diff --git a/tests_e2e/test_suites/agent_cgroups.yml b/tests_e2e/test_suites/agent_cgroups.yml index d6d1fc0f17..7844e606f4 100644 --- a/tests_e2e/test_suites/agent_cgroups.yml +++ b/tests_e2e/test_suites/agent_cgroups.yml @@ -1,9 +1,11 @@ # -# The test suite verify the agent running in expected cgroups and also, checks agent tracking the cgroups for polling resource metrics. Also, it verifies the agent cpu quota is set as expected. +# The test suite verify the agent running in expected cgroups and also, checks agent tracking the cgroups for polling resource metrics, +# checks unexpected processes in the agent cgroups, and it verifies the agent cpu quota is set as expected. # name: "AgentCgroups" tests: - "agent_cgroups/agent_cgroups.py" - "agent_cgroups/agent_cpu_quota.py" + - "agent_cgroups/agent_cgroups_process_check.py" images: "cgroups-endorsed" owns_vm: true \ No newline at end of file diff --git a/tests_e2e/tests/agent_cgroups/agent_cgroups_process_check.py b/tests_e2e/tests/agent_cgroups/agent_cgroups_process_check.py new file mode 100644 index 0000000000..d0996caec8 --- /dev/null +++ b/tests_e2e/tests/agent_cgroups/agent_cgroups_process_check.py @@ -0,0 +1,77 @@ +#!/usr/bin/env python3 + +# Microsoft Azure Linux Agent +# +# Copyright 2018 Microsoft Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from typing import List, Dict, Any + +from tests_e2e.tests.lib.agent_test import AgentVmTest +from tests_e2e.tests.lib.agent_test_context import AgentVmTestContext +from tests_e2e.tests.lib.logging import log +from tests_e2e.tests.lib.virtual_machine_extension_client import VirtualMachineExtensionClient +from tests_e2e.tests.lib.vm_extension_identifier import VmExtensionIds + + +class AgentCgroupsProcessCheck(AgentVmTest): + """ + Tests the agent's ability to detect processes that do not belong to the agent's cgroup + """ + def __init__(self, context: AgentVmTestContext): + super().__init__(context) + self._ssh_client = self._context.create_ssh_client() + + def run(self): + """ + Steps: + 1. Verify that agent detects processes that do not belong to the agent's cgroup and disable the cgroups + 2. Run the extension, so that they are run in the agent's cgroup + 3. Restart the ext_handler process to re-initialize the cgroups setup + 4. Verify that agent detects extension processes and will not enable the cgroups + """ + + log.info("=====Validating agent cgroups process check") + self._run_remote_test(self._ssh_client, "agent_cgroups_process_check-unknown_process_check.py", use_sudo=True) + + self._install_ama_extension() + + log.info("=====Validating agent cgroups not enabled") + self._run_remote_test(self._ssh_client, "agent_cgroups_process_check-cgroups_not_enabled.py", use_sudo=True) + + def _install_ama_extension(self): + ama_extension = VirtualMachineExtensionClient( + self._context.vm, VmExtensionIds.AzureMonitorLinuxAgent, + resource_name="AMAAgent") + log.info("Installing %s", ama_extension) + ama_extension.enable() + ama_extension.assert_instance_view() + + def get_ignore_error_rules(self) -> List[Dict[str, Any]]: + + ignore_rules = [ + # This is produced by the test, so it is expected + # Examples: + # 2024-04-01T19:16:11.929000Z INFO MonitorHandler ExtHandler [CGW] Disabling resource usage monitoring. Reason: Check on cgroups failed: + # [CGroupsException] The agent's cgroup includes unexpected processes: ['[PID: 2957] dd\x00if=/dev/zero\x00of=/dev/null\x00 '] + # 2024-04-01T19:17:04.995276Z WARNING ExtHandler ExtHandler [CGroupsException] The agent's cgroup includes unexpected processes: ['[PID: 3285] /usr/bin/python3\x00/var/lib/waagent/Microsoft.Azure.Monitor.AzureM', '[PID: 3286] /usr/bin/python3\x00/var/lib/waagent/Microsoft.Azure.Monitor.AzureM'] + {'message': r"The agent's cgroup includes unexpected processes"}, + {'message': r"Found unexpected processes in the agent cgroup before agent enable cgroups"} + ] + return ignore_rules + + +if __name__ == "__main__": + AgentCgroupsProcessCheck.run_from_command_line() diff --git a/tests_e2e/tests/lib/cgroup_helpers.py b/tests_e2e/tests/lib/cgroup_helpers.py index 5c552ef19e..9e55790641 100644 --- a/tests_e2e/tests/lib/cgroup_helpers.py +++ b/tests_e2e/tests/lib/cgroup_helpers.py @@ -1,3 +1,4 @@ +import datetime import os import re @@ -146,10 +147,17 @@ def check_cgroup_disabled_with_unknown_process(): """ Returns True if the cgroup is disabled with unknown process """ + return check_log_message("Disabling resource usage monitoring. Reason: Check on cgroups failed:.+UNKNOWN") + + +def check_log_message(message, after_timestamp=datetime.datetime.min): + """ + Check if the log message is present after the given timestamp(if provided) in the agent log + """ + log.info("Checking log message: {0}".format(message)) for record in AgentLog().read(): - match = re.search("Disabling resource usage monitoring. Reason: Check on cgroups failed:.+UNKNOWN", - record.message, flags=re.DOTALL) - if match is not None: + match = re.search(message, record.message, flags=re.DOTALL) + if match is not None and record.timestamp > after_timestamp: log.info("Found message:\n\t%s", record.text.replace("\n", "\n\t")) return True return False diff --git a/tests_e2e/tests/scripts/agent_cgroups_process_check-cgroups_not_enabled.py b/tests_e2e/tests/scripts/agent_cgroups_process_check-cgroups_not_enabled.py new file mode 100755 index 0000000000..a8db751e61 --- /dev/null +++ b/tests_e2e/tests/scripts/agent_cgroups_process_check-cgroups_not_enabled.py @@ -0,0 +1,60 @@ +#!/usr/bin/env pypy3 + +# Microsoft Azure Linux Agent +# +# Copyright 2018 Microsoft Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# This script verifies agent detected unexpected processes in the agent cgroup before cgroup initialization + +from assertpy import fail + +from azurelinuxagent.common.utils import shellutil +from tests_e2e.tests.lib.cgroup_helpers import check_agent_quota_disabled, check_log_message +from tests_e2e.tests.lib.logging import log +from tests_e2e.tests.lib.retry import retry_if_false + + +def restart_ext_handler(): + log.info("Restarting the extension handler") + shellutil.run_command(["pkill", "-f", "WALinuxAgent.*run-exthandler"]) + + +def verify_agent_cgroups_not_enabled(): + """ + Verifies that the agent cgroups not enabled when ama extension(unexpected) processes are found in the agent cgroup + """ + log.info("Verifying agent cgroups are not enabled") + + ama_process_found: bool = retry_if_false(lambda: check_log_message("The agent's cgroup includes unexpected processes:.+/var/lib/waagent/Microsoft.Azure.Monitor")) + if not ama_process_found: + fail("Agent failed to found ama extension processes in the agent cgroup") + + found: bool = retry_if_false(lambda: check_log_message("Found unexpected processes in the agent cgroup before agent enable cgroups")) + if not found: + fail("Agent failed to found unknown processes in the agent cgroup") + + disabled: bool = retry_if_false(check_agent_quota_disabled) + if not disabled: + fail("The agent failed to disable its CPUQuota when cgroups were not enabled") + + +def main(): + restart_ext_handler() + verify_agent_cgroups_not_enabled() + + +if __name__ == "__main__": + main() diff --git a/tests_e2e/tests/scripts/agent_cgroups_process_check-unknown_process_check.py b/tests_e2e/tests/scripts/agent_cgroups_process_check-unknown_process_check.py new file mode 100755 index 0000000000..4445a517f8 --- /dev/null +++ b/tests_e2e/tests/scripts/agent_cgroups_process_check-unknown_process_check.py @@ -0,0 +1,92 @@ +#!/usr/bin/env pypy3 +# Microsoft Azure Linux Agent +# +# Copyright 2018 Microsoft Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# This script forces the process check by putting unknown process in the agent's cgroup + +import os +import subprocess +import datetime + +from assertpy import fail + +from azurelinuxagent.common.utils import shellutil +from tests_e2e.tests.lib.cgroup_helpers import get_agent_cgroup_mount_path, \ + BASE_CGROUP, check_agent_quota_disabled, check_log_message +from tests_e2e.tests.lib.logging import log +from tests_e2e.tests.lib.retry import retry_if_false + + +def prepare_agent(): + check_time = datetime.datetime.utcnow() + log.info("Executing script update-waagent-conf to enable agent cgroups config flag") + result = shellutil.run_command(["update-waagent-conf", "Debug.CgroupCheckPeriod=20", "Debug.CgroupLogMetrics=y", + "Debug.CgroupDisableOnProcessCheckFailure=y", + "Debug.CgroupDisableOnQuotaCheckFailure=n"]) + log.info("Successfully enabled agent cgroups config flag: {0}".format(result)) + + found: bool = retry_if_false(lambda: check_log_message(" Agent cgroups enabled: True", after_timestamp=check_time)) + if not found: + fail("Agent cgroups not enabled") + + +def creating_dummy_process(): + log.info("Creating dummy process to add to agent's cgroup") + dd_command = ["dd", "if=/dev/zero", "of=/dev/null"] + proc = subprocess.Popen(dd_command) + return proc.pid + + +def remove_dummy_process(pid): + log.info("Removing dummy process from agent's cgroup") + shellutil.run_command(["kill", "-9", str(pid)]) + + +def disable_agent_cgroups_with_unknown_process(pid): + """ + Adding dummy process to the agent's cgroup and verifying that the agent detects the unknown process and disables cgroups + + Note: System may kick the added process out of the cgroups, keeps adding until agent detect that process + """ + def unknown_process_found(): + cgroup_procs_path = os.path.join(BASE_CGROUP, "cpu,cpuacct" + get_agent_cgroup_mount_path(), "cgroup.procs") + log.info("Adding dummy process %s to cgroup.procs file %s", pid, cgroup_procs_path) + process = subprocess.Popen(f"echo {pid} > {cgroup_procs_path}", shell=True) + process.communicate() + + # The log message indicating the check failed is similar to + # 2021-03-29T23:33:15.603530Z INFO MonitorHandler ExtHandler Disabling resource usage monitoring. Reason: Check on cgroups failed: + # [CGroupsException] The agent's cgroup includes unexpected processes: ['[PID: 25826] python3\x00/home/nam/Compute-Runtime-Tux-Pipeline/dungeon_crawler/s'] + found: bool = retry_if_false(lambda: check_log_message( + "Disabling resource usage monitoring. Reason: Check on cgroups failed:.+The agent's cgroup includes unexpected processes:.+{0}".format( + pid)), attempts=3) + return found and retry_if_false(check_agent_quota_disabled, attempts=3) + + found: bool = retry_if_false(unknown_process_found, attempts=3) + if not found: + fail("The agent did not detect unknown process: {0}".format(pid)) + + +def main(): + prepare_agent() + pid = creating_dummy_process() + disable_agent_cgroups_with_unknown_process(pid) + remove_dummy_process(pid) + + +if __name__ == "__main__": + main() diff --git a/tests_e2e/tests/scripts/agent_cpu_quota-check_agent_cpu_quota.py b/tests_e2e/tests/scripts/agent_cpu_quota-check_agent_cpu_quota.py index c8aad49f59..94781356b9 100755 --- a/tests_e2e/tests/scripts/agent_cpu_quota-check_agent_cpu_quota.py +++ b/tests_e2e/tests/scripts/agent_cpu_quota-check_agent_cpu_quota.py @@ -146,45 +146,18 @@ def wait_for_log_message(message, timeout=datetime.timedelta(minutes=5)): fail("The agent did not find [{0}] in its log within the allowed timeout".format(message)) -def verify_process_check_on_agent_cgroups(): - """ - This method checks agent detect unexpected processes in its cgroup and disables the CPUQuota - """ - log.info("***Verifying process check on agent cgroups") - log.info("Ensuring agent CPUQuota is enabled and backup the drop-in file to restore later in further tests") - if check_agent_quota_disabled(): - fail("The agent's CPUQuota is not enabled: {0}".format(get_agent_cpu_quota())) - quota_drop_in = os.path.join(systemd.get_agent_drop_in_path(), _DROP_IN_FILE_CPU_QUOTA) - quota_drop_in_backup = quota_drop_in + ".bk" - log.info("Backing up %s to %s...", quota_drop_in, quota_drop_in_backup) - shutil.copy(quota_drop_in, quota_drop_in_backup) - # - # Re-enable Process checks on cgroups and verify that the agent detects unexpected processes in its cgroup and disables the CPUQuota wehen - # that happens - # - shellutil.run_command(["update-waagent-conf", "Debug.CgroupDisableOnProcessCheckFailure=y"]) - - # The log message indicating the check failed is similar to - # 2021-03-29T23:33:15.603530Z INFO MonitorHandler ExtHandler Disabling resource usage monitoring. Reason: Check on cgroups failed: - # [CGroupsException] The agent's cgroup includes unexpected processes: ['[PID: 25826] python3\x00/home/nam/Compute-Runtime-Tux-Pipeline/dungeon_crawler/s'] - wait_for_log_message( - "Disabling resource usage monitoring. Reason: Check on cgroups failed:.+The agent's cgroup includes unexpected processes") - disabled: bool = retry_if_false(check_agent_quota_disabled) - if not disabled: - fail("The agent did not disable its CPUQuota: {0}".format(get_agent_cpu_quota())) - - def verify_throttling_time_check_on_agent_cgroups(): """ This method checks agent disables its CPUQuota when it exceeds its throttling limit """ log.info("***Verifying CPU throttling check on agent cgroups") # Now disable the check on unexpected processes and enable the check on throttledtime and verify that the agent disables its CPUQuota when it exceeds its throttling limit - log.info("Re-enabling CPUQuota...") + if check_agent_quota_disabled(): + fail("The agent's CPUQuota is not enabled: {0}".format(get_agent_cpu_quota())) quota_drop_in = os.path.join(systemd.get_agent_drop_in_path(), _DROP_IN_FILE_CPU_QUOTA) quota_drop_in_backup = quota_drop_in + ".bk" - log.info("Restoring %s from %s...", quota_drop_in, quota_drop_in_backup) - shutil.copy(quota_drop_in_backup, quota_drop_in) + log.info("Backing up %s to %s...", quota_drop_in, quota_drop_in_backup) + shutil.copy(quota_drop_in, quota_drop_in_backup) shellutil.run_command(["systemctl", "daemon-reload"]) shellutil.run_command(["update-waagent-conf", "Debug.CgroupDisableOnProcessCheckFailure=n", "Debug.CgroupDisableOnQuotaCheckFailure=y", "Debug.AgentCpuThrottledTimeThreshold=5"]) @@ -205,11 +178,20 @@ def verify_throttling_time_check_on_agent_cgroups(): fail("The agent did not disable its CPUQuota: {0}".format(get_agent_cpu_quota())) +def cleanup_test_setup(): + log.info("Cleaning up test setup") + drop_in_file = os.path.join(systemd.get_agent_drop_in_path(), "99-ExecStart.conf") + if os.path.exists(drop_in_file): + log.info("Removing %s...", drop_in_file) + os.remove(drop_in_file) + shellutil.run_command(["systemctl", "daemon-reload"]) + + def main(): prepare_agent() verify_agent_reported_metrics() - verify_process_check_on_agent_cgroups() verify_throttling_time_check_on_agent_cgroups() + cleanup_test_setup() run_remote_test(main) From ae32cf3f4030660f95812bb232e73e0803554ac1 Mon Sep 17 00:00:00 2001 From: nnandigam Date: Mon, 1 Apr 2024 16:54:59 -0700 Subject: [PATCH 2/6] agent restart --- tests_e2e/tests/scripts/agent_cpu_quota-check_agent_cpu_quota.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests_e2e/tests/scripts/agent_cpu_quota-check_agent_cpu_quota.py b/tests_e2e/tests/scripts/agent_cpu_quota-check_agent_cpu_quota.py index 94781356b9..8c91edb179 100755 --- a/tests_e2e/tests/scripts/agent_cpu_quota-check_agent_cpu_quota.py +++ b/tests_e2e/tests/scripts/agent_cpu_quota-check_agent_cpu_quota.py @@ -185,6 +185,7 @@ def cleanup_test_setup(): log.info("Removing %s...", drop_in_file) os.remove(drop_in_file) shellutil.run_command(["systemctl", "daemon-reload"]) + shellutil.run_command(["agent-service", "restart"]) def main(): From e83786879c9082a5b5aa31ac0c7f091a8e6f64d8 Mon Sep 17 00:00:00 2001 From: Nageswara Nandigam Date: Mon, 1 Apr 2024 20:47:34 -0700 Subject: [PATCH 3/6] move the process check --- azurelinuxagent/ga/cgroupconfigurator.py | 29 +++++++++--------------- 1 file changed, 11 insertions(+), 18 deletions(-) diff --git a/azurelinuxagent/ga/cgroupconfigurator.py b/azurelinuxagent/ga/cgroupconfigurator.py index 22e754d183..73fb7df643 100644 --- a/azurelinuxagent/ga/cgroupconfigurator.py +++ b/azurelinuxagent/ga/cgroupconfigurator.py @@ -173,25 +173,24 @@ def initialize(self): if not self.__check_no_legacy_cgroups(): return - cpu_controller_root, memory_controller_root = self.__get_cgroup_controllers() agent_unit_name = systemd.get_agent_unit_name() agent_slice = systemd.get_unit_property(agent_unit_name, "Slice") - - if conf.get_cgroup_disable_on_process_check_failure() and self._check_processes_in_agent_cgroup_before_enable(agent_slice, cpu_controller_root, memory_controller_root): - reason = "Found unexpected processes in the agent cgroup before agent enable cgroups." - self.disable(reason, DisableCgroups.ALL) - return - if agent_slice not in (AZURE_SLICE, "system.slice"): _log_cgroup_warning("The agent is within an unexpected slice: {0}", agent_slice) return self.__setup_azure_slice() + cpu_controller_root, memory_controller_root = self.__get_cgroup_controllers() self._agent_cpu_cgroup_path, self._agent_memory_cgroup_path = self.__get_agent_cgroups(agent_slice, cpu_controller_root, memory_controller_root) + if conf.get_cgroup_disable_on_process_check_failure() and self._check_processes_in_agent_cgroup_before_enable(agent_slice): + reason = "Found unexpected processes in the agent cgroup before agent enable cgroups." + self.disable(reason, DisableCgroups.ALL) + return + if self._agent_cpu_cgroup_path is not None or self._agent_memory_cgroup_path is not None: self.enable() @@ -557,7 +556,7 @@ def __try_set_cpu_quota(quota): # pylint: disable=unused-private-member return False return True - def _check_processes_in_agent_cgroup_before_enable(self, agent_slice, cpu_root_cgroup_path, memory_root_cgroup_path): + def _check_processes_in_agent_cgroup_before_enable(self, agent_slice): """ This check ensures that before we enable the agent's cgroups, there are no unexpected processes in the agent's cgroup already. @@ -568,18 +567,13 @@ def _check_processes_in_agent_cgroup_before_enable(self, agent_slice, cpu_root_c """ if agent_slice not in AZURE_SLICE: return False - cpu_cgroup_relative_path, memory_cgroup_relative_path = self._cgroups_api.get_process_cgroup_paths("self") - agent_cpu_cgroup_path = os.path.join(cpu_root_cgroup_path, cpu_cgroup_relative_path) - agent_memory_cgroup_path = os.path.join(memory_root_cgroup_path, memory_cgroup_relative_path) - - check_cgroup_path = agent_cpu_cgroup_path if agent_cpu_cgroup_path is not None else agent_memory_cgroup_path - if check_cgroup_path is None: + if self._agent_cpu_cgroup_path is None: return False try: _log_cgroup_info("Checking for unexpected processes in the agent's cgroup before enabling cgroups") - self._check_processes_in_agent_cgroup(check_cgroup_path) + self._check_processes_in_agent_cgroup() except CGroupsException as exception: _log_cgroup_warning(ustr(exception)) return True @@ -619,7 +613,7 @@ def check_cgroups(self, cgroup_metrics): finally: self._check_cgroups_lock.release() - def _check_processes_in_agent_cgroup(self, cgroup_path=None): + def _check_processes_in_agent_cgroup(self): """ Verifies that the agent's cgroup includes only the current process, its parent, commands started using shellutil and instances of systemd-run (those processes correspond, respectively, to the extension handler, the daemon, commands started by the extension handler, and the systemd-run @@ -631,7 +625,6 @@ def _check_processes_in_agent_cgroup(self, cgroup_path=None): """ unexpected = [] agent_cgroup_proc_names = [] - check_cgroup_path = cgroup_path if cgroup_path is not None else self._agent_cpu_cgroup_path try: daemon = os.getppid() extension_handler = os.getpid() @@ -639,7 +632,7 @@ def _check_processes_in_agent_cgroup(self, cgroup_path=None): agent_commands.update(shellutil.get_running_commands()) systemd_run_commands = set() systemd_run_commands.update(self._cgroups_api.get_systemd_run_commands()) - agent_cgroup = CGroupsApi.get_processes_in_cgroup(check_cgroup_path) + agent_cgroup = CGroupsApi.get_processes_in_cgroup(self._agent_cpu_cgroup_path) # get the running commands again in case new commands started or completed while we were fetching the processes in the cgroup; agent_commands.update(shellutil.get_running_commands()) systemd_run_commands.update(self._cgroups_api.get_systemd_run_commands()) From a1dbe214b5b33d75ee2d4b7713297069a86c786a Mon Sep 17 00:00:00 2001 From: nnandigam Date: Mon, 1 Apr 2024 22:10:09 -0700 Subject: [PATCH 4/6] fix unit tests --- .../data/cgroups/proc_self_cgroup_azure_slice | 13 ++++++++ tests/ga/test_cgroupconfigurator.py | 33 +++++++++++++------ 2 files changed, 36 insertions(+), 10 deletions(-) create mode 100644 tests/data/cgroups/proc_self_cgroup_azure_slice diff --git a/tests/data/cgroups/proc_self_cgroup_azure_slice b/tests/data/cgroups/proc_self_cgroup_azure_slice new file mode 100644 index 0000000000..58df643b24 --- /dev/null +++ b/tests/data/cgroups/proc_self_cgroup_azure_slice @@ -0,0 +1,13 @@ +12:blkio:/azure.slice/walinuxagent.service +11:cpu,cpuacct:/azure.slice/walinuxagent.service +10:devices:/azure.slice/walinuxagent.service +9:pids:/azure.slice/walinuxagent.service +8:memory:/azure.slice/walinuxagent.service +7:freezer:/ +6:hugetlb:/ +5:perf_event:/ +4:net_cls,net_prio:/ +3:cpuset:/ +2:rdma:/ +1:name=systemd:/azure.slice/walinuxagent.service +0::/azure.slice/walinuxagent.service diff --git a/tests/ga/test_cgroupconfigurator.py b/tests/ga/test_cgroupconfigurator.py index 251734ce80..b04a0a3105 100644 --- a/tests/ga/test_cgroupconfigurator.py +++ b/tests/ga/test_cgroupconfigurator.py @@ -46,6 +46,10 @@ def tearDownClass(cls): CGroupConfigurator._instance = None AgentTestCase.tearDownClass() + def tearDown(self): + CGroupConfigurator._instance = None + AgentTestCase.tearDown(self) + @contextlib.contextmanager def _get_cgroup_configurator(self, initialize=True, enable=True, mock_commands=None): CGroupConfigurator._instance = None @@ -911,17 +915,26 @@ def test_agent_not_enable_cgroups_if_unexpected_process_already_in_agent_cgroups command_mocks = [MockCommand(r"^systemctl show walinuxagent\.service --property Slice", '''Slice=azure.slice ''')] - with self._get_cgroup_configurator(mock_commands=command_mocks) as configurator: - self.assertFalse(configurator.enabled(), "Cgroups should not be enabled") + original_read_file = fileutil.read_file - disable_events = [kwargs for _, kwargs in add_event.call_args_list if kwargs["op"] == WALAEventOperation.CGroupsDisabled] - self.assertTrue( - len(disable_events) == 1, - "Exactly 1 event should have been emitted. Got: {0}".format(disable_events)) - self.assertIn( - "Found unexpected processes in the agent cgroup before agent enable cgroups", - disable_events[0]["message"], - "The error message is not correct when process check failed") + def mock_read_file(filepath, **args): + if filepath == "/proc/self/cgroup": + filepath = os.path.join(data_dir, "cgroups", "proc_self_cgroup_azure_slice") + return original_read_file(filepath, **args) + + with self._get_cgroup_configurator(initialize=False, mock_commands=command_mocks) as configurator: + with patch("azurelinuxagent.common.utils.fileutil.read_file", side_effect=mock_read_file): + configurator.initialize() + + self.assertFalse(configurator.enabled(), "Cgroups should not be enabled") + disable_events = [kwargs for _, kwargs in add_event.call_args_list if kwargs["op"] == WALAEventOperation.CGroupsDisabled] + self.assertTrue( + len(disable_events) == 1, + "Exactly 1 event should have been emitted. Got: {0}".format(disable_events)) + self.assertIn( + "Found unexpected processes in the agent cgroup before agent enable cgroups", + disable_events[0]["message"], + "The error message is not correct when process check failed") def test_check_agent_memory_usage_should_raise_a_cgroups_exception_when_the_limit_is_exceeded(self): metrics = [MetricValue(MetricsCategory.MEMORY_CATEGORY, MetricsCounter.TOTAL_MEM_USAGE, AGENT_NAME_TELEMETRY, conf.get_agent_memory_quota() + 1), From fd137292268bdde5acb8dc015ca7dca51f457bc8 Mon Sep 17 00:00:00 2001 From: nnandigam Date: Wed, 3 Apr 2024 18:02:44 -0700 Subject: [PATCH 5/6] address comments --- azurelinuxagent/ga/cgroupconfigurator.py | 19 ++++++++------- tests/ga/test_cgroupconfigurator.py | 2 +- tests_e2e/tests/lib/cgroup_helpers.py | 9 +++++++ ...ups_process_check-unknown_process_check.py | 24 +++++++++++++------ .../agent_cpu_quota-check_agent_cpu_quota.py | 8 ++++++- 5 files changed, 44 insertions(+), 18 deletions(-) diff --git a/azurelinuxagent/ga/cgroupconfigurator.py b/azurelinuxagent/ga/cgroupconfigurator.py index 73fb7df643..150d5c79d5 100644 --- a/azurelinuxagent/ga/cgroupconfigurator.py +++ b/azurelinuxagent/ga/cgroupconfigurator.py @@ -186,7 +186,7 @@ def initialize(self): cpu_controller_root, memory_controller_root) - if conf.get_cgroup_disable_on_process_check_failure() and self._check_processes_in_agent_cgroup_before_enable(agent_slice): + if conf.get_cgroup_disable_on_process_check_failure() and self._check_fails_if_processes_found_in_agent_cgroup_before_enable(agent_slice): reason = "Found unexpected processes in the agent cgroup before agent enable cgroups." self.disable(reason, DisableCgroups.ALL) return @@ -556,21 +556,17 @@ def __try_set_cpu_quota(quota): # pylint: disable=unused-private-member return False return True - def _check_processes_in_agent_cgroup_before_enable(self, agent_slice): + def _check_fails_if_processes_found_in_agent_cgroup_before_enable(self, agent_slice): """ This check ensures that before we enable the agent's cgroups, there are no unexpected processes in the agent's cgroup already. The issue we observed that long running extension processes may be in agent cgroups if agent goes this cycle enabled(1)->disabled(2)->enabled(3). 1. Agent cgroups enabled in some version - 2. Disabled agent cgroups due to check_cgroups regular check, now extension runs will be in agent cgroups. + 2. Disabled agent cgroups due to check_cgroups regular check. Once we disable the cgroups we don't run the extensions in it's own slice, so they will be in agent cgroups. 3. When ext_hanlder restart and enable the cgroups again, already running processes from step 2 still be in agent cgroups. This may cause the extensions run with agent limit. """ - if agent_slice not in AZURE_SLICE: + if agent_slice != AZURE_SLICE: return False - - if self._agent_cpu_cgroup_path is None: - return False - try: _log_cgroup_info("Checking for unexpected processes in the agent's cgroup before enabling cgroups") self._check_processes_in_agent_cgroup() @@ -625,6 +621,11 @@ def _check_processes_in_agent_cgroup(self): """ unexpected = [] agent_cgroup_proc_names = [] + # Now _check_processes_in_agent_cgroup called before we enable the cgroups, So agent cgroup paths can be none and also, + # It's possible that any one of the controller is not mounted, so we need to check both. + cgroup_path = self._agent_cpu_cgroup_path if self._agent_cpu_cgroup_path is not None else self._agent_memory_cgroup_path + if cgroup_path is None: + return try: daemon = os.getppid() extension_handler = os.getpid() @@ -632,7 +633,7 @@ def _check_processes_in_agent_cgroup(self): agent_commands.update(shellutil.get_running_commands()) systemd_run_commands = set() systemd_run_commands.update(self._cgroups_api.get_systemd_run_commands()) - agent_cgroup = CGroupsApi.get_processes_in_cgroup(self._agent_cpu_cgroup_path) + agent_cgroup = CGroupsApi.get_processes_in_cgroup(cgroup_path) # get the running commands again in case new commands started or completed while we were fetching the processes in the cgroup; agent_commands.update(shellutil.get_running_commands()) systemd_run_commands.update(self._cgroups_api.get_systemd_run_commands()) diff --git a/tests/ga/test_cgroupconfigurator.py b/tests/ga/test_cgroupconfigurator.py index b04a0a3105..841a4b72d0 100644 --- a/tests/ga/test_cgroupconfigurator.py +++ b/tests/ga/test_cgroupconfigurator.py @@ -911,7 +911,7 @@ def test_check_cgroups_should_disable_cgroups_when_a_check_fails(self): @patch('azurelinuxagent.ga.cgroupconfigurator.CGroupConfigurator._Impl._check_processes_in_agent_cgroup', side_effect=CGroupsException("Test")) @patch('azurelinuxagent.ga.cgroupconfigurator.add_event') - def test_agent_not_enable_cgroups_if_unexpected_process_already_in_agent_cgroups(self, add_event, _): + def test_agent_should_not_enable_cgroups_if_unexpected_process_already_in_agent_cgroups(self, add_event, _): command_mocks = [MockCommand(r"^systemctl show walinuxagent\.service --property Slice", '''Slice=azure.slice ''')] diff --git a/tests_e2e/tests/lib/cgroup_helpers.py b/tests_e2e/tests/lib/cgroup_helpers.py index 9e55790641..ef49284e15 100644 --- a/tests_e2e/tests/lib/cgroup_helpers.py +++ b/tests_e2e/tests/lib/cgroup_helpers.py @@ -7,6 +7,7 @@ from azurelinuxagent.common.osutil import systemd from azurelinuxagent.common.utils import shellutil from azurelinuxagent.common.version import DISTRO_NAME, DISTRO_VERSION +from azurelinuxagent.ga.cgroupapi import SystemdCgroupsApi from tests_e2e.tests.lib.agent_log import AgentLog from tests_e2e.tests.lib.logging import log from tests_e2e.tests.lib.retry import retry_if_false @@ -161,3 +162,11 @@ def check_log_message(message, after_timestamp=datetime.datetime.min): log.info("Found message:\n\t%s", record.text.replace("\n", "\n\t")) return True return False + + +def get_unit_cgroup_paths(unit_name): + """ + Returns the cgroup paths for the given unit + """ + cgroups_api = SystemdCgroupsApi() + return cgroups_api.get_unit_cgroup_paths(unit_name) diff --git a/tests_e2e/tests/scripts/agent_cgroups_process_check-unknown_process_check.py b/tests_e2e/tests/scripts/agent_cgroups_process_check-unknown_process_check.py index 4445a517f8..4d2c887959 100755 --- a/tests_e2e/tests/scripts/agent_cgroups_process_check-unknown_process_check.py +++ b/tests_e2e/tests/scripts/agent_cgroups_process_check-unknown_process_check.py @@ -25,8 +25,9 @@ from assertpy import fail from azurelinuxagent.common.utils import shellutil +from azurelinuxagent.ga.cgroupapi import SystemdCgroupsApi from tests_e2e.tests.lib.cgroup_helpers import get_agent_cgroup_mount_path, \ - BASE_CGROUP, check_agent_quota_disabled, check_log_message + BASE_CGROUP, check_agent_quota_disabled, check_log_message, get_unit_cgroup_paths, AGENT_SERVICE_NAME from tests_e2e.tests.lib.logging import log from tests_e2e.tests.lib.retry import retry_if_false @@ -46,7 +47,7 @@ def prepare_agent(): def creating_dummy_process(): log.info("Creating dummy process to add to agent's cgroup") - dd_command = ["dd", "if=/dev/zero", "of=/dev/null"] + dd_command = ["sleep", "60m"] proc = subprocess.Popen(dd_command) return proc.pid @@ -62,11 +63,18 @@ def disable_agent_cgroups_with_unknown_process(pid): Note: System may kick the added process out of the cgroups, keeps adding until agent detect that process """ - def unknown_process_found(): - cgroup_procs_path = os.path.join(BASE_CGROUP, "cpu,cpuacct" + get_agent_cgroup_mount_path(), "cgroup.procs") + + def unknown_process_found(cpu_cgroup): + cgroup_procs_path = os.path.join(cpu_cgroup, "cgroup.procs") log.info("Adding dummy process %s to cgroup.procs file %s", pid, cgroup_procs_path) - process = subprocess.Popen(f"echo {pid} > {cgroup_procs_path}", shell=True) - process.communicate() + try: + with open(cgroup_procs_path, 'a') as f: + f.write("\n") + f.write(str(pid)) + f.close() + except Exception as e: + log.warning("Error while adding process to cgroup.procs file: {0}".format(e)) + return False # The log message indicating the check failed is similar to # 2021-03-29T23:33:15.603530Z INFO MonitorHandler ExtHandler Disabling resource usage monitoring. Reason: Check on cgroups failed: @@ -76,7 +84,9 @@ def unknown_process_found(): pid)), attempts=3) return found and retry_if_false(check_agent_quota_disabled, attempts=3) - found: bool = retry_if_false(unknown_process_found, attempts=3) + cpu_cgroup, _ = get_unit_cgroup_paths(AGENT_SERVICE_NAME) + + found: bool = retry_if_false(lambda: unknown_process_found(cpu_cgroup), attempts=3) if not found: fail("The agent did not detect unknown process: {0}".format(pid)) diff --git a/tests_e2e/tests/scripts/agent_cpu_quota-check_agent_cpu_quota.py b/tests_e2e/tests/scripts/agent_cpu_quota-check_agent_cpu_quota.py index 8c91edb179..5dfc55be82 100755 --- a/tests_e2e/tests/scripts/agent_cpu_quota-check_agent_cpu_quota.py +++ b/tests_e2e/tests/scripts/agent_cpu_quota-check_agent_cpu_quota.py @@ -30,7 +30,7 @@ from azurelinuxagent.ga.cgroupconfigurator import _DROP_IN_FILE_CPU_QUOTA from tests_e2e.tests.lib.agent_log import AgentLog from tests_e2e.tests.lib.cgroup_helpers import check_agent_quota_disabled, \ - get_agent_cpu_quota + get_agent_cpu_quota, check_log_message from tests_e2e.tests.lib.logging import log from tests_e2e.tests.lib.remote_test import run_remote_test from tests_e2e.tests.lib.retry import retry_if_false @@ -185,8 +185,14 @@ def cleanup_test_setup(): log.info("Removing %s...", drop_in_file) os.remove(drop_in_file) shellutil.run_command(["systemctl", "daemon-reload"]) + + check_time = datetime.datetime.utcnow() shellutil.run_command(["agent-service", "restart"]) + found: bool = retry_if_false(lambda: check_log_message(" Agent cgroups enabled: True", after_timestamp=check_time)) + if not found: + fail("Agent cgroups not enabled yet") + def main(): prepare_agent() From 27941cca4ec9b7f71ad6c95d26562f5195e16acb Mon Sep 17 00:00:00 2001 From: nnandigam Date: Wed, 3 Apr 2024 18:12:11 -0700 Subject: [PATCH 6/6] pylint --- azurelinuxagent/ga/cgroupconfigurator.py | 4 ++-- .../agent_cgroups_process_check-unknown_process_check.py | 5 +---- 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/azurelinuxagent/ga/cgroupconfigurator.py b/azurelinuxagent/ga/cgroupconfigurator.py index 150d5c79d5..ce86101e07 100644 --- a/azurelinuxagent/ga/cgroupconfigurator.py +++ b/azurelinuxagent/ga/cgroupconfigurator.py @@ -621,8 +621,8 @@ def _check_processes_in_agent_cgroup(self): """ unexpected = [] agent_cgroup_proc_names = [] - # Now _check_processes_in_agent_cgroup called before we enable the cgroups, So agent cgroup paths can be none and also, - # It's possible that any one of the controller is not mounted, so we need to check both. + # Now we call _check_processes_in_agent_cgroup before we enable the cgroups or any one of the controller is not mounted, agent cgroup paths can be None. + # so we need to check both. cgroup_path = self._agent_cpu_cgroup_path if self._agent_cpu_cgroup_path is not None else self._agent_memory_cgroup_path if cgroup_path is None: return diff --git a/tests_e2e/tests/scripts/agent_cgroups_process_check-unknown_process_check.py b/tests_e2e/tests/scripts/agent_cgroups_process_check-unknown_process_check.py index 4d2c887959..d1b3014a03 100755 --- a/tests_e2e/tests/scripts/agent_cgroups_process_check-unknown_process_check.py +++ b/tests_e2e/tests/scripts/agent_cgroups_process_check-unknown_process_check.py @@ -25,9 +25,7 @@ from assertpy import fail from azurelinuxagent.common.utils import shellutil -from azurelinuxagent.ga.cgroupapi import SystemdCgroupsApi -from tests_e2e.tests.lib.cgroup_helpers import get_agent_cgroup_mount_path, \ - BASE_CGROUP, check_agent_quota_disabled, check_log_message, get_unit_cgroup_paths, AGENT_SERVICE_NAME +from tests_e2e.tests.lib.cgroup_helpers import check_agent_quota_disabled, check_log_message, get_unit_cgroup_paths, AGENT_SERVICE_NAME from tests_e2e.tests.lib.logging import log from tests_e2e.tests.lib.retry import retry_if_false @@ -71,7 +69,6 @@ def unknown_process_found(cpu_cgroup): with open(cgroup_procs_path, 'a') as f: f.write("\n") f.write(str(pid)) - f.close() except Exception as e: log.warning("Error while adding process to cgroup.procs file: {0}".format(e)) return False