Start sending additional memory information per process

azurelinuxagent/common/cgroup.py

@@ -111,6 +111,30 @@ def is_active(self):
 
         return False
 
+    def get_tracked_processes(self):
+        """
+
+        :return: List of Str (Pids)
+        """
+        try:
+            procs = self._get_parameters("cgroup.procs")
+            if procs:
+                return procs
+        except (IOError, OSError) as e:
+            if e.errno == errno.ENOENT:
+                # only suppressing file not found exceptions.
+                pass
+            else:
+                logger.periodic_warn(logger.EVERY_HALF_HOUR,
+                                     'Could not get list of procs from "cgroup.procs" file in the cgroup: {0}.'
+                                     ' Internal error: {1}'.format(self.path, ustr(e)))
+        except CGroupsException as e:
+            logger.periodic_warn(logger.EVERY_HALF_HOUR,
+                                 'Could not get list of tasks from "cgroup.procs" file in the cgroup: {0}.'
+                                 ' Internal error: {1}'.format(self.path, ustr(e)))
+            return None
+        return None
+
 
 class CpuCgroup(CGroup):
     def __init__(self, name, cgroup_path):
@@ -213,15 +237,11 @@ def get_memory_usage(self):
 
         try:
             usage = self._get_parameters('memory.usage_in_bytes', first_line_only=True)
-        except (IOError, OSError) as e:
-            if e.errno == errno.ENOENT:
-                # only suppressing file not found exceptions.
-                pass
-            else:
+        except Exception as e:
+            if isinstance(e, (IOError, OSError)) and e.errno == errno.ENOENT:
                 raise e
+            raise CGroupsException("Exception while attempting to read {0}".format("memory.usage_in_bytes"), e)
 
-        if not usage:
-            usage = "0"
         return int(usage)
 
     def get_max_memory_usage(self):
@@ -234,12 +254,9 @@ def get_max_memory_usage(self):
         usage = None
         try:
             usage = self._get_parameters('memory.max_usage_in_bytes', first_line_only=True)
-        except (IOError, OSError) as e:
-            if e.errno == errno.ENOENT:
-                # only suppressing file not found exceptions.
-                pass
-            else:
+        except Exception as e:
+            if isinstance(e, (IOError, OSError)) and e.errno == errno.ENOENT:
                 raise e
-        if not usage:
-            usage = "0"
+            raise CGroupsException("Exception while attempting to read {0}".format("memory.usage_in_bytes"), e)
+
         return int(usage)

azurelinuxagent/common/cgroupstelemetry.py

@@ -14,6 +14,7 @@
 #
 # Requires Python 2.6+ and Openssl 1.0+
 import errno
+import os
 import threading
 from collections import namedtuple
 from datetime import datetime as dt
@@ -22,9 +23,14 @@
 from azurelinuxagent.common.cgroup import CpuCgroup
 from azurelinuxagent.common.exception import CGroupsException
 from azurelinuxagent.common.future import ustr
-
+from azurelinuxagent.common.resourceusage import MemoryResourceUsage, ProcessInfo
 
 MetricValue = namedtuple('Metric', ['category', 'counter', 'instance', 'value'])
+StatmMetricValue = namedtuple('StatmMetricValue', ['pid', 'resource_metric'])
+
+DELIM = " | "
+DEFAULT_PROCESS_NAME = "NO_PROCESS_FOUND"
+DEFAULT_PROCESS_COMMANDLINE = "NO_CMDLINE_FOUND"
 
 
 class CGroupsTelemetry(object):
@@ -34,16 +40,28 @@ class CGroupsTelemetry(object):
     _cgroup_metrics = {}
     _rlock = threading.RLock()
 
+    @staticmethod
+    def get_process_info_summary(process_id):
+        process_cmdline = ProcessInfo.get_proc_cmdline(process_id) if not None else DEFAULT_PROCESS_COMMANDLINE
+        process_name = ProcessInfo.get_proc_name(process_id) if not None else DEFAULT_PROCESS_NAME
+
+        # The ProcessName and ProcessCommandLine can be None if the file /proc/<pid>/{comm,cmdline} cease to exist;
+        # eg: the process can die, or finish. Which is why we need Default Names, in case we fail to fetch the details
+        # from those files.
+
+        return process_id + DELIM + process_name + DELIM + process_cmdline
+
     @staticmethod
     def _get_metrics_list(metric):
         return [metric.average(), metric.min(), metric.max(), metric.median(), metric.count(),
                 metric.first_poll_time(), metric.last_poll_time()]
 
     @staticmethod
     def _process_cgroup_metric(cgroup_metrics):
-        memory_usage = cgroup_metrics.get_memory_usage()
-        max_memory_usage = cgroup_metrics.get_max_memory_usage()
-        cpu_usage = cgroup_metrics.get_cpu_usage()
+        memory_usage = cgroup_metrics.get_memory_metrics()
+        max_memory_usage = cgroup_metrics.get_max_memory_metrics()
+        cpu_usage = cgroup_metrics.get_cpu_metrics()
+        memory_usage_per_process = cgroup_metrics.get_proc_statm_memory_metrics()
 
         processed_extension = {}
 
@@ -62,6 +80,14 @@ def _process_cgroup_metric(cgroup_metrics):
             else:
                 processed_extension["memory"] = {"max_mem": CGroupsTelemetry._get_metrics_list(max_memory_usage)}
 
+        if memory_usage_per_process:
+            for pid_process_memory in memory_usage_per_process:
+                if "proc_statm_memory" in processed_extension:
+                    processed_extension["proc_statm_memory"][pid_process_memory.pid] = \
+                        CGroupsTelemetry._get_metrics_list(pid_process_memory.resource_metric)
+                else:
+                    processed_extension["proc_statm_memory"] = {pid_process_memory.pid:
+                        CGroupsTelemetry._get_metrics_list(pid_process_memory.resource_metric)}
         return processed_extension
 
     @staticmethod
@@ -151,9 +177,20 @@ def poll_all_tracked():
                         max_memory_usage = cgroup.get_max_memory_usage()
                         CGroupsTelemetry._cgroup_metrics[cgroup.name].add_max_memory_usage(max_memory_usage)
                         metrics.append(MetricValue("Memory", "Max Memory Usage", cgroup.name, max_memory_usage))
+
+                        pids = cgroup.get_tracked_processes()
+
+                        if pids:
+                            for pid in pids:
+                                mem_usage_from_procstatm = MemoryResourceUsage.get_memory_usage_from_proc_statm(pid)
+                                metrics.append(MetricValue("Memory", "Memory Used by Process",
+                                                           CGroupsTelemetry.get_process_info_summary(pid),
+                                                           mem_usage_from_procstatm))
+                                CGroupsTelemetry._cgroup_metrics[cgroup.name].add_proc_statm_memory(
+                                    CGroupsTelemetry.get_process_info_summary(pid), mem_usage_from_procstatm)
                     else:
                         raise CGroupsException('CGroup controller {0} is not supported for cgroup {1}'.format(
-                                               cgroup.controller, cgroup.name))
+                            cgroup.controller, cgroup.name))
                 except Exception as e:
                     # There can be scenarios when the CGroup has been deleted by the time we are fetching the values
                     # from it. This would raise IOError with file entry not found (ERRNO: 2). We do not want to log
@@ -187,30 +224,45 @@ def __init__(self):
         self._memory_usage = Metric()
         self._max_memory_usage = Metric()
         self._cpu_usage = Metric()
+        self._proc_statm_mem = {}
+
         self.marked_for_delete = False
 
     def add_memory_usage(self, usage):
-        self._memory_usage.append(usage)
+        if not self.marked_for_delete:
+            self._memory_usage.append(usage)
 
     def add_max_memory_usage(self, usage):
-        self._max_memory_usage.append(usage)
+        if not self.marked_for_delete:
+            self._max_memory_usage.append(usage)
 
     def add_cpu_usage(self, usage):
-        self._cpu_usage.append(usage)
+        if not self.marked_for_delete:
+            self._cpu_usage.append(usage)
 
-    def get_memory_usage(self):
+    def add_proc_statm_memory(self, pid, usage):
+        if not self.marked_for_delete:
+            if pid not in self._proc_statm_mem:
+                self._proc_statm_mem[pid] = Metric()
+            self._proc_statm_mem[pid].append(usage)
+
+    def get_memory_metrics(self):
         return self._memory_usage
 
-    def get_max_memory_usage(self):
+    def get_max_memory_metrics(self):
         return self._max_memory_usage
 
-    def get_cpu_usage(self):
+    def get_cpu_metrics(self):
         return self._cpu_usage
 
+    def get_proc_statm_memory_metrics(self):
+        return [StatmMetricValue(pid, metric) for pid, metric in self._proc_statm_mem.items()]
+
     def clear(self):
         self._memory_usage.clear()
         self._max_memory_usage.clear()
         self._cpu_usage.clear()
+        self._proc_statm_mem.clear()
 
 
 class Metric(object):

azurelinuxagent/common/exception.py

@@ -16,6 +16,7 @@
 #
 # Requires Python 2.6+ and Openssl 1.0+
 #
+
 """
 Defines all exceptions
 """
@@ -44,14 +45,18 @@ def __init__(self, msg=None, inner=None):
 
 class AgentNetworkError(AgentError):
     """
-    When network is not available\.
+    When network is not available.
     """
 
     def __init__(self, msg=None, inner=None):
         super(AgentNetworkError, self).__init__(msg, inner)
 
 
 class CGroupsException(AgentError):
+    """
+    When cgroups are not available or could not fetch data from CGroup.
+    """
+
     def __init__(self, msg=None, inner=None):
         super(CGroupsException, self).__init__(msg, inner)
 

azurelinuxagent/common/resourceusage.py

@@ -0,0 +1,147 @@
+# Copyright 2019 Microsoft Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# Requires Python 2.6+ and Openssl 1.0+
+
+import os
+
+from azurelinuxagent.common import logger
+from azurelinuxagent.common.exception import AgentError
+from azurelinuxagent.common.future import ustr
+from azurelinuxagent.common.logger import EVERY_SIX_HOURS
+from azurelinuxagent.common.utils import fileutil
+
+
+PAGE_SIZE = os.sysconf('SC_PAGE_SIZE')
+PROC_STATM_FILENAME_FORMAT = "/proc/{0}/statm"
+PROC_CMDLINE_FILENAME_FORMAT = "/proc/{0}/cmdline"
+PROC_COMM_FILENAME_FORMAT = "/proc/{0}/comm"
+PROC_STATUS_FILENAME_FORMAT = "/proc/{0}/status"
+
+
+class ResourceUsage(object):
+    pass
+
+
+class MemoryResourceUsage(ResourceUsage):
+    @staticmethod
+    def get_memory_usage_from_proc_statm(process_id):
+        proc_pid_rss = 0
+        try:
+            proc_pid_rss = MemoryResourceUsage._get_proc_rss(process_id)
+        except Exception as e:
+            logger.periodic_info(EVERY_SIX_HOURS, "[PERIODIC] Could not get the /prod/{0}/statm data due to {1}",
+                                 process_id, ustr(e))
+
+        return proc_pid_rss
+
+    @staticmethod
+    def _get_proc_rss(process_id):
+        """
+        /proc/<pid>/statm fields: columns are (in pages):
+
+        total program size|
+        resident set size|
+        shared pages|
+        text (code) |
+        data/stack |
+        library |
+        dirty pages |
+
+        Here an example:
+        root@vm:/# cat /proc/1392/statm
+        17637 5316 2125 938 0 3332 0
+
+        :return: resident set size in bytes.
+        """
+        try:
+            pid_statm = fileutil.read_file(PROC_STATM_FILENAME_FORMAT.format(process_id)).split()
+            pid_rss = int(pid_statm[1])  # Index 1 is RSS.
+        except Exception:
+            raise
+
+        return pid_rss * PAGE_SIZE
+
+
+class ProcessInfo(object):
+    @staticmethod
+    def get_proc_name(process_id):
+        proc_pid_rss = None
+        try:
+            proc_pid_rss = ProcessInfo._get_proc_comm(process_id)
+        except ProcessInfoException as e:
+            logger.periodic_info(EVERY_SIX_HOURS, "[PERIODIC] {0}", ustr(e))
+        return proc_pid_rss
+
+    @staticmethod
+    def get_proc_cmdline(process_id):
+        proc_pid_rss = None
+        try:
+            proc_pid_rss = ProcessInfo._get_proc_cmdline(process_id)
+        except ProcessInfoException as e:
+            logger.periodic_info(EVERY_SIX_HOURS, "[PERIODIC] {0}", ustr(e))
+        return proc_pid_rss
+
+    @classmethod
+    def _get_proc_cmdline(cls, process_id):
+        """
+        /proc/<pid>/cmdline returns cmdline arguments passed to the Linux kernel. The returned string is delimited with
+        the \0 character and needs to be replaced with some other character to make it readable.
+
+        Here an example:
+        root@vm:/# cat /proc/1392/cmdline
+        python--targettest_resourceusage.py
+        root@vm:/# cat /proc/1392/cmdline | tr "\0" " "
+        python --target test_resourceusage.py
+
+        :return: command line passed to the process string.
+        """
+        cmdline_file_name = PROC_CMDLINE_FILENAME_FORMAT.format(process_id)
+        try:
+            pid_cmdline = fileutil.read_file(cmdline_file_name).replace("\0", " ").strip()
+            pid_cmdline_str = str(pid_cmdline)
+        except Exception as e:
+            raise ProcessInfoException("Could not get contents from {0}".format(cmdline_file_name), e)
+
+        return pid_cmdline_str
+
+    @classmethod
+    def _get_proc_comm(cls, process_id):
+        """
+        /proc/<pid>/comm This file exposes the process's comm value-that is, the command name associated with the
+        process. Strings longer than TASK_COMM_LEN (16) characters are silently truncated.
+
+        Here an example:
+        root@vm:/# cat /proc/1392/comm
+        python
+
+        :return: process name
+        """
+        comm_file_name = PROC_COMM_FILENAME_FORMAT.format(process_id)
+        try:
+            pid_comm = fileutil.read_file(comm_file_name).strip()
+            pid_comm_str = str(pid_comm)
+        except Exception as e:
+            raise ProcessInfoException("Could not get contents from {0}".format(comm_file_name), e)
+
+        return pid_comm_str
+
+
+class ProcessInfoException(AgentError):
+    """
+    When we
+    """
+
+    def __init__(self, msg=None, inner=None):
+        super(ProcessInfoException, self).__init__(msg, inner)