Adding common params into all the events if not present & other changes

1. Added "in" support for TelemetryEvent for easier code check. Also added complementary tests for the same.
2. Updated the tests in test_monitor to reflect newer sysinfo fields added.
3. Added comments to describe the decisions behind the field decisions.

azurelinuxagent/common/event.py

@@ -344,9 +344,22 @@ def add_metric(self, category, counter, instance, value, log_event=False):
 
     @staticmethod
     def _add_default_parameters_to_event(event):
+        # We write the GAVersion here rather than add it in azurelinuxagent.ga.monitor.MonitorHandler.add_sysinfo
+        # as there could be a possibility of events being sent with newer version of the agent, rather than the agent
+        # version generating the event.
+        # Old behavior example: V1 writes the event on the disk and finds an update immediately, and updates. Now the
+        # new monitor thread would pick up the events from the disk and send it with the CURRENT_AGENT, which would have
+        # newer version of the agent. This causes confusion.
         event.parameters.append(TelemetryEventParam("GAVersion", CURRENT_AGENT))
+
+        # ContainerId can change due to live migration and we want to preserve the container Id of the container writing
+        # the event, rather than sending the event.
         event.parameters.append(TelemetryEventParam('ContainerId', get_container_id_from_env()))
+
+        # This is used as the actual time of event generation.
         event.parameters.append(TelemetryEventParam('OpcodeName', datetime.utcnow().__str__()))
+
+        # Other commong parameters added to the events
         event.parameters.append(TelemetryEventParam('EventTid', threading.current_thread().ident))
         event.parameters.append(TelemetryEventParam('EventPid', os.getpid()))
         event.parameters.append(TelemetryEventParam("TaskName", threading.current_thread().getName()))

azurelinuxagent/common/telemetryevent.py

@@ -35,6 +35,10 @@ def __init__(self, eventId=None, providerId=None):
         self.providerId = providerId
         self.parameters = DataContractList(TelemetryEventParam)
 
+    # Checking if the particular param name is in the TelemetryEvent.
+    def __contains__(self, param_name):
+        return param_name in [param.name for param in self.parameters]
+
 
 class TelemetryEventList(DataContract):
     def __init__(self):

azurelinuxagent/ga/monitor.py

@@ -29,7 +29,8 @@
 
 from azurelinuxagent.common.cgroupstelemetry import CGroupsTelemetry
 from azurelinuxagent.common.errorstate import ErrorState
-from azurelinuxagent.common.event import add_event, WALAEventOperation, CONTAINER_ID_ENV_VARIABLE
+from azurelinuxagent.common.event import add_event, WALAEventOperation, CONTAINER_ID_ENV_VARIABLE, \
+    get_container_id_from_env
 from azurelinuxagent.common.exception import EventError, ProtocolError, OSUtilError, HttpError
 from azurelinuxagent.common.future import ustr
 from azurelinuxagent.common.osutil import get_osutil
@@ -329,10 +330,26 @@ def add_sysinfo(self, event):
         # Add sys_info params populated by the agent
         final_parameters.extend(self.sysinfo)
 
-        # Container id is populated for agent events on the fly, but not for extension events. Add it if it's missing.
-        if "ContainerId" not in [param.name for param in event.parameters]:
+        if "ContainerId" not in event:
+            # ContainerId is populated for agent events on the fly, but not for extension events. Add it if it's missing.
             final_parameters.append(
-                TelemetryEventParam("ContainerId", os.environ.get(CONTAINER_ID_ENV_VARIABLE, "UNINITIALIZED")))
+                TelemetryEventParam("ContainerId", get_container_id_from_env()))
+        if "GAVersion" not in event:
+            # GAVersion is populated for agent events on the fly, but not for extension events. Add it if it's missing.
+            final_parameters.append(TelemetryEventParam("GAVersion", CURRENT_AGENT))
+
+        # Default fields are only populated by Agent and not the extension. Agent will fill up any event if they don't
+        # have the default params.
+        if "OpcodeName" not in event:
+            final_parameters.append(TelemetryEventParam("OpcodeName", datetime.utcnow().__str__()))
+        if "EventTid" not in event:
+            final_parameters.append(TelemetryEventParam("EventTid", ""))
+        if "EventPid" not in event:
+            final_parameters.append(TelemetryEventParam("EventPid", ""))
+        if "TaskName" not in event:
+            final_parameters.append(TelemetryEventParam("TaskName", ""))
+        if "KeywordName" not in event:
+            final_parameters.append(TelemetryEventParam("KeywordName", ""))
 
         event.parameters = final_parameters
 

tests/common/test_telemetryevent.py

@@ -0,0 +1,49 @@
+# Copyright 2019 Microsoft Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# Requires Python 2.6+ and Openssl 1.0+
+#
+from azurelinuxagent.common.telemetryevent import TelemetryEvent, TelemetryEventParam
+from tests.tools import AgentTestCase
+
+
+def get_test_event(name="DummyExtension", op="Unknown", is_success=True, duration=0, version="foo", evt_type="", is_internal=False,
+                      message="DummyMessage", eventId=1):
+    event = TelemetryEvent(eventId, "XXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX")
+    event.parameters.append(TelemetryEventParam('Name', name))
+    event.parameters.append(TelemetryEventParam('Version', str(version)))
+    event.parameters.append(TelemetryEventParam('IsInternal', is_internal))
+    event.parameters.append(TelemetryEventParam('Operation', op))
+    event.parameters.append(TelemetryEventParam('OperationSuccess', is_success))
+    event.parameters.append(TelemetryEventParam('Message', message))
+    event.parameters.append(TelemetryEventParam('Duration', duration))
+    event.parameters.append(TelemetryEventParam('ExtensionType', evt_type))
+    return event
+
+
+class TestTelemetryEvent(AgentTestCase):
+    def test_contains_works_for_TelemetryEvent(self):
+        test_event = get_test_event(message="Dummy Event")
+
+        self.assertTrue('Name' in test_event)
+        self.assertTrue('Version' in test_event)
+        self.assertTrue('IsInternal' in test_event)
+        self.assertTrue('Operation' in test_event)
+        self.assertTrue('OperationSuccess' in test_event)
+        self.assertTrue('Message' in test_event)
+        self.assertTrue('Duration' in test_event)
+        self.assertTrue('ExtensionType' in test_event)
+
+        self.assertFalse('GAVersion' in test_event)
+        self.assertFalse('ContainerId' in test_event)

tests/data/ext/event_from_agent.xml

@@ -0,0 +1,26 @@
+<Data>
+    <Provider id="69B669B9-4AF8-4C50-BDC4-6006FA76E975"/>
+    <Event id="1"/>
+    <Param Name="OperationSuccess" Value="True" T="mt:bool"/>
+    <Param Name="Processors" Value="0" T="mt:uint64"/>
+    <Param Name="OpcodeName" Value="10101010" T="mt:wstr"/>
+    <Param Name="Version" Value="1.4.1.0" T="mt:wstr"/>
+    <Param Name="RoleName" Value="" T="mt:wstr"/>
+    <Param Name="IsInternal" Value="False" T="mt:bool"/>
+    <Param Name="RAM" Value="0" T="mt:uint64"/>
+    <Param Name="ExecutionMode" Value="IAAS" T="mt:wstr"/>
+    <Param Name="RoleInstanceName" Value="" T="mt:wstr"/>
+    <Param Name="Name" Value="CustomScript" T="mt:wstr"/>
+    <Param Name="Message" Value="(01302)Script is finished.&#10;---stdout---&#10;hello&#10;&#10;---errout---&#10;&#10;"
+           T="mt:wstr"/>
+    <Param Name="OSVersion" Value="" T="mt:wstr"/>
+    <Param Name="Operation" Value="RunScript" T="mt:wstr"/>
+    <Param Name="GAVersion" Value="WALinuxAgent-2.2.44" T="mt:wstr"/>
+    <Param Name="TenantName" Value="" T="mt:wstr"/>
+    <Param Name="Duration" Value="0" T="mt:uint64"/>
+    <Param Name="ExtensionType" Value="" T="mt:wstr"/>
+    <Param Name="EventTid" Value="140240309798656" T="mt:uint64"/>
+    <Param Name="EventPid" Value="108573" T="mt:uint64"/>
+    <Param Name="TaskName" Value="MonitorHandler" T="mt:wstr"/>
+    <Param Name="KeywordName" Value="" T="mt:wstr"/>
+</Data>

tests/data/ext/event.xml → tests/data/ext/event_from_extension.xml

@@ -3,7 +3,6 @@
     <Event id="1"/>
     <Param Name="OperationSuccess" Value="True" T="mt:bool"/>
     <Param Name="Processors" Value="0" T="mt:uint64"/>
-    <Param Name="OpcodeName" Value="" T="mt:wstr"/>
     <Param Name="Version" Value="1.4.1.0" T="mt:wstr"/>
     <Param Name="RoleName" Value="" T="mt:wstr"/>
     <Param Name="IsInternal" Value="False" T="mt:bool"/>
@@ -13,11 +12,9 @@
     <Param Name="Name" Value="CustomScript" T="mt:wstr"/>
     <Param Name="Message" Value="(01302)Script is finished.&#10;---stdout---&#10;hello&#10;&#10;---errout---&#10;&#10;"
            T="mt:wstr"/>
-    <Param Name="KeywordName" Value="" T="mt:wstr"/>
-    <Param Name="TaskName" Value="" T="mt:wstr"/>
     <Param Name="OSVersion" Value="" T="mt:wstr"/>
     <Param Name="Operation" Value="RunScript" T="mt:wstr"/>
-    <Param Name="GAVersion" Value="" T="mt:wstr"/>
+    <Param Name="GAVersion" Value="WALinuxAgent-2.2.44" T="mt:wstr"/>
     <Param Name="TenantName" Value="" T="mt:wstr"/>
     <Param Name="Duration" Value="0" T="mt:uint64"/>
     <Param Name="ExtensionType" Value="" T="mt:wstr"/>

tests/ga/test_monitor.py

@@ -109,7 +109,7 @@ def get_event_message(duration, evt_type, is_internal, is_success, message, name
 class TestMonitor(AgentTestCase):
 
     def test_parse_xml_event(self, *args):
-        data_str = load_data('ext/event.xml')
+        data_str = load_data('ext/event_from_extension.xml')
         event = parse_xml_event(data_str)
         self.assertNotEqual(None, event)
         self.assertNotEqual(0, event.parameters)
@@ -123,7 +123,7 @@ def test_parse_json_event(self, *args):
         self.assertTrue(all(param is not None for param in event.parameters))
 
     def test_add_sysinfo_should_honor_sysinfo_values_from_agent_for_agent_events(self, *args):
-        data_str = load_data('ext/event.xml')
+        data_str = load_data('ext/event_from_agent.xml')
         event = parse_xml_event(data_str)
 
         # Pretend that the test event is coming from the agent by ensuring the event already has a container id
@@ -137,13 +137,25 @@ def test_add_sysinfo_should_honor_sysinfo_values_from_agent_for_agent_events(sel
         sysinfo_role_name_value = "sysinfo_dummy_role"
         sysinfo_role_instance_name_value = "sysinfo_dummy_role_instance"
         sysinfo_execution_mode_value = "sysinfo_IAAS"
+        GAVersion_value = "WALinuxAgent-2.2.44"
+        OpcodeName_value = "10101010"
+        EventTid_value = 140240309798656
+        EventPid_value = 108573
+        TaskName_value = "MonitorHandler"
+        KeywordName_value = ""
 
         vm_name_param = "VMName"
         tenant_name_param = "TenantName"
         role_name_param = "RoleName"
         role_instance_name_param = "RoleInstanceName"
         execution_mode_param = "ExecutionMode"
         container_id_param = "ContainerId"
+        GAVersion_param = "GAVersion"
+        OpcodeName_param = "OpcodeName"
+        EventTid_param = "EventTid"
+        EventPid_param = "EventPid"
+        TaskName_param = "TaskName"
+        KeywordName_param = "KeywordName"
 
         sysinfo = [
             TelemetryEventParam(role_instance_name_param, sysinfo_role_instance_name_value),
@@ -179,13 +191,31 @@ def test_add_sysinfo_should_honor_sysinfo_values_from_agent_for_agent_events(sel
             elif p.name == container_id_param:
                 self.assertEqual(container_id_value, p.value)
                 counter += 1
+            elif p.name == GAVersion_param:
+                self.assertEqual(GAVersion_value, p.value)
+                counter += 1
+            elif p.name == OpcodeName_param:
+                self.assertEqual(OpcodeName_value, p.value)
+                counter += 1
+            elif p.name == EventTid_param:
+                self.assertEqual(EventTid_value, p.value)
+                counter += 1
+            elif p.name == EventPid_param:
+                self.assertEqual(EventPid_value, p.value)
+                counter += 1
+            elif p.name == TaskName_param:
+                self.assertEqual(TaskName_value, p.value)
+                counter += 1
+            elif p.name == KeywordName_param:
+                self.assertEqual(KeywordName_value, p.value)
+                counter += 1
 
-        self.assertEqual(6, counter)
+        self.assertEqual(12, counter)
 
     def test_add_sysinfo_should_honor_sysinfo_values_from_agent_for_extension_events(self, *args):
         # The difference between agent and extension events is that extension events don't have the container id
         # populated on the fly like the agent events do. Ensure the container id is populated in add_sysinfo.
-        data_str = load_data('ext/event.xml')
+        data_str = load_data('ext/event_from_extension.xml')
         event = parse_xml_event(data_str)
         monitor_handler = get_monitor_handler()
 
@@ -198,13 +228,25 @@ def test_add_sysinfo_should_honor_sysinfo_values_from_agent_for_extension_events
         sysinfo_role_name_value = "sysinfo_dummy_role"
         sysinfo_role_instance_name_value = "sysinfo_dummy_role_instance"
         sysinfo_execution_mode_value = "sysinfo_IAAS"
+        GAVersion_value = "WALinuxAgent-2.2.44"
+        OpcodeName_value = "2019-01-01 01:30:00"  # Mocking time below.
+        EventTid_value = ""
+        EventPid_value = ""
+        TaskName_value = ""
+        KeywordName_value = ""
 
         vm_name_param = "VMName"
         tenant_name_param = "TenantName"
         role_name_param = "RoleName"
         role_instance_name_param = "RoleInstanceName"
         execution_mode_param = "ExecutionMode"
         container_id_param = "ContainerId"
+        GAVersion_param = "GAVersion"
+        OpcodeName_param = "OpcodeName"
+        EventTid_param = "EventTid"
+        EventPid_param = "EventPid"
+        TaskName_param = "TaskName"
+        KeywordName_param = "KeywordName"
 
         sysinfo = [
             TelemetryEventParam(role_instance_name_param, sysinfo_role_instance_name_value),
@@ -214,12 +256,16 @@ def test_add_sysinfo_should_honor_sysinfo_values_from_agent_for_extension_events
             TelemetryEventParam(role_name_param, sysinfo_role_name_value)
         ]
         monitor_handler.sysinfo = sysinfo
-        monitor_handler.add_sysinfo(event)
+        with patch("azurelinuxagent.ga.monitor.datetime") as patch_datetime:
+            patch_datetime.utcnow = Mock(return_value=datetime.datetime.strptime("2019-01-01 01:30:00",
+                                                                        '%Y-%m-%d %H:%M:%S'))
+            monitor_handler.add_sysinfo(event)
 
         self.assertNotEqual(None, event)
         self.assertNotEqual(0, event.parameters)
         self.assertTrue(all(param is not None for param in event.parameters))
 
+        counter = 0
         counter = 0
         for p in event.parameters:
             if p.name == vm_name_param:
@@ -240,8 +286,26 @@ def test_add_sysinfo_should_honor_sysinfo_values_from_agent_for_extension_events
             elif p.name == container_id_param:
                 self.assertEqual(container_id_value, p.value)
                 counter += 1
+            elif p.name == GAVersion_param:
+                self.assertEqual(GAVersion_value, p.value)
+                counter += 1
+            elif p.name == OpcodeName_param:
+                self.assertEqual(OpcodeName_value, p.value)
+                counter += 1
+            elif p.name == EventTid_param:
+                self.assertEqual(EventTid_value, p.value)
+                counter += 1
+            elif p.name == EventPid_param:
+                self.assertEqual(EventPid_value, p.value)
+                counter += 1
+            elif p.name == TaskName_param:
+                self.assertEqual(TaskName_value, p.value)
+                counter += 1
+            elif p.name == KeywordName_param:
+                self.assertEqual(KeywordName_value, p.value)
+                counter += 1
 
-        self.assertEqual(6, counter)
+        self.assertEqual(12, counter)
         os.environ.pop(CONTAINER_ID_ENV_VARIABLE)
 
     @patch("azurelinuxagent.ga.monitor.MonitorHandler.send_telemetry_heartbeat")