Log collector should not fetch full goal state with extensions

azurelinuxagent/common/logcollector.py

@@ -34,6 +34,7 @@
 
 # Please note: be careful when adding agent dependencies in this module.
 # This module uses its own logger and logs to its own file, not to the agent log.
+from azurelinuxagent.common.protocol.goal_state import GoalStateProperties
 from azurelinuxagent.common.protocol.util import get_protocol_util
 
 _EXTENSION_LOG_DIR = get_ext_log_dir()
@@ -117,8 +118,11 @@ def _set_resource_usage_cgroups(cpu_cgroup_path, memory_cgroup_path):
 
     @staticmethod
     def _initialize_telemetry():
-        protocol = get_protocol_util().get_protocol()
-        protocol.client.update_goal_state(force_update=True)
+        goalstate_properties = GoalStateProperties.RoleConfig | GoalStateProperties.HostingEnv
+        protocol = get_protocol_util().get_protocol(goalstate_properties=goalstate_properties)
+        protocol.client.update_goal_state(
+            goalstate_properties=goalstate_properties,
+            force_update=True)
         # Initialize the common parameters for telemetry events
         initialize_event_logger_vminfo_common_parameters(protocol)
 

azurelinuxagent/common/protocol/goal_state.py

@@ -48,6 +48,22 @@
 _GET_GOAL_STATE_MAX_ATTEMPTS = 6
 
 
+class GoalStateProperties(object):
+    """
+    Enum for defining the properties that we fetch in the goal state
+    """
+    RoleConfig = 1
+    HostingEnv = 2
+    SharedConfig = 4
+    ExtensionsConfig_Certs = 8
+    RemoteAccessInfo = 16
+
+    @staticmethod
+    def default_properties():
+        return GoalStateProperties.RoleConfig | GoalStateProperties.HostingEnv | GoalStateProperties.SharedConfig | \
+               GoalStateProperties.ExtensionsConfig_Certs | GoalStateProperties.RemoteAccessInfo
+
+
 class GoalStateInconsistentError(ProtocolError):
     """
     Indicates an inconsistency in the goal state (e.g. missing tenant certificate)
@@ -57,7 +73,7 @@ def __init__(self, msg, inner=None):
 
 
 class GoalState(object):
-    def __init__(self, wire_client, silent=False):
+    def __init__(self, wire_client, goalstate_properties=GoalStateProperties.default_properties(), silent=False):
         """
         Fetches the goal state using the given wire client.
 
@@ -85,7 +101,7 @@ def __init__(self, wire_client, silent=False):
             self._certs = EmptyCertificates()
             self._remote_access = None
 
-            self.update(silent=silent)
+            self.update(goalstate_properties=goalstate_properties, silent=silent)
 
         except ProtocolError:
             raise
@@ -160,20 +176,20 @@ def update_host_plugin_headers(wire_client):
         # Fetching the goal state updates the HostGAPlugin so simply trigger the request
         GoalState._fetch_goal_state(wire_client)
 
-    def update(self, silent=False):
+    def update(self, goalstate_properties=GoalStateProperties.default_properties(), silent=False):
         """
         Updates the current GoalState instance fetching values from the WireServer/HostGAPlugin as needed
         """
         self.logger.silent = silent
 
         try:
-            self._update(force_update=False)
+            self._update(goalstate_properties=goalstate_properties, force_update=False)
         except GoalStateInconsistentError as e:
             self.logger.warn("Detected an inconsistency in the goal state: {0}", ustr(e))
-            self._update(force_update=True)
+            self._update(goalstate_properties=goalstate_properties, force_update=True)
             self.logger.info("The goal state is consistent")
 
-    def _update(self, force_update):
+    def _update(self, goalstate_properties, force_update):
         #
         # Fetch the goal state from both the HGAP and the WireServer
         #
@@ -226,7 +242,7 @@ def _update(self, force_update):
         #
         extensions_config = None
         if goal_state_updated:
-            extensions_config = self._fetch_full_wire_server_goal_state(incarnation, xml_doc)
+            extensions_config = self._fetch_full_wire_server_goal_state(incarnation, xml_doc, goalstate_properties)
 
         #
         # Lastly, decide whether to use the vmSettings or extensionsConfig for the extensions goal state
@@ -343,7 +359,7 @@ def _fetch_vm_settings(wire_client, force_update=False):
 
         return vm_settings, vm_settings_updated
 
-    def _fetch_full_wire_server_goal_state(self, incarnation, xml_doc):
+    def _fetch_full_wire_server_goal_state(self, incarnation, xml_doc, goalstate_properties=GoalStateProperties.default_properties()):
         """
         Issues HTTP requests (to the WireServer) for each of the URIs in the goal state (ExtensionsConfig, Certificate, Remote Access users, etc)
         and populates the corresponding properties.
@@ -356,40 +372,48 @@ def _fetch_full_wire_server_goal_state(self, incarnation, xml_doc):
             self.logger.info(message)
             add_event(op=WALAEventOperation.GoalState, message=message)
 
-            role_instance = find(xml_doc, "RoleInstance")
-            role_instance_id = findtext(role_instance, "InstanceId")
-            role_config = find(role_instance, "Configuration")
-            role_config_name = findtext(role_config, "ConfigName")
-            container = find(xml_doc, "Container")
-            container_id = findtext(container, "ContainerId")
+            role_instance_id = None
+            role_config_name = None
+            container_id = None
+            if GoalStateProperties.RoleConfig & goalstate_properties:
+                role_instance = find(xml_doc, "RoleInstance")
+                role_instance_id = findtext(role_instance, "InstanceId")
+                role_config = find(role_instance, "Configuration")
+                role_config_name = findtext(role_config, "ConfigName")
+                container = find(xml_doc, "Container")
+                container_id = findtext(container, "ContainerId")
 
             extensions_config_uri = findtext(xml_doc, "ExtensionsConfig")
-            if extensions_config_uri is None:
+            if not (GoalStateProperties.ExtensionsConfig_Certs & goalstate_properties) or extensions_config_uri is None:
                 extensions_config = ExtensionsGoalStateFactory.create_empty(incarnation)
             else:
                 xml_text = self._wire_client.fetch_config(extensions_config_uri, self._wire_client.get_header())
                 extensions_config = ExtensionsGoalStateFactory.create_from_extensions_config(incarnation, xml_text, self._wire_client)
                 self._history.save_extensions_config(extensions_config.get_redacted_text())
 
-            hosting_env_uri = findtext(xml_doc, "HostingEnvironmentConfig")
-            xml_text = self._wire_client.fetch_config(hosting_env_uri, self._wire_client.get_header())
-            hosting_env = HostingEnv(xml_text)
-            self._history.save_hosting_env(xml_text)
-
-            shared_conf_uri = findtext(xml_doc, "SharedConfig")
-            xml_text = self._wire_client.fetch_config(shared_conf_uri, self._wire_client.get_header())
-            shared_config = SharedConfig(xml_text)
-            self._history.save_shared_conf(xml_text)
-            # SharedConfig.xml is used by other components (Azsec and Singularity/HPC Infiniband), so save it to the agent's root directory as well
-            shared_config_file = os.path.join(conf.get_lib_dir(), SHARED_CONF_FILE_NAME)
-            try:
-                fileutil.write_file(shared_config_file, xml_text)
-            except Exception as e:
-                logger.warn("Failed to save {0}: {1}".format(shared_config, e))
+            hosting_env = None
+            if GoalStateProperties.HostingEnv & goalstate_properties:
+                hosting_env_uri = findtext(xml_doc, "HostingEnvironmentConfig")
+                xml_text = self._wire_client.fetch_config(hosting_env_uri, self._wire_client.get_header())
+                hosting_env = HostingEnv(xml_text)
+                self._history.save_hosting_env(xml_text)
+
+            shared_config = None
+            if GoalStateProperties.SharedConfig & goalstate_properties:
+                shared_conf_uri = findtext(xml_doc, "SharedConfig")
+                xml_text = self._wire_client.fetch_config(shared_conf_uri, self._wire_client.get_header())
+                shared_config = SharedConfig(xml_text)
+                self._history.save_shared_conf(xml_text)
+                # SharedConfig.xml is used by other components (Azsec and Singularity/HPC Infiniband), so save it to the agent's root directory as well
+                shared_config_file = os.path.join(conf.get_lib_dir(), SHARED_CONF_FILE_NAME)
+                try:
+                    fileutil.write_file(shared_config_file, xml_text)
+                except Exception as e:
+                    logger.warn("Failed to save {0}: {1}".format(shared_config, e))
 
             certs = EmptyCertificates()
             certs_uri = findtext(xml_doc, "Certificates")
-            if certs_uri is not None:
+            if (GoalStateProperties.ExtensionsConfig_Certs & goalstate_properties) and certs_uri is not None:
                 xml_text = self._wire_client.fetch_config(certs_uri, self._wire_client.get_header_for_cert())
                 certs = Certificates(xml_text, self.logger)
                 # Log and save the certificates summary (i.e. the thumbprint but not the certificate itself) to the goal state history
@@ -403,11 +427,12 @@ def _fetch_full_wire_server_goal_state(self, incarnation, xml_doc):
                 self._history.save_certificates(json.dumps(certs.summary))
 
             remote_access = None
-            remote_access_uri = findtext(container, "RemoteAccessInfo")
-            if remote_access_uri is not None:
-                xml_text = self._wire_client.fetch_config(remote_access_uri, self._wire_client.get_header_for_cert())
-                remote_access = RemoteAccess(xml_text)
-                self._history.save_remote_access(xml_text)
+            if GoalStateProperties.RemoteAccessInfo & goalstate_properties:
+                remote_access_uri = findtext(container, "RemoteAccessInfo")
+                if remote_access_uri is not None:
+                    xml_text = self._wire_client.fetch_config(remote_access_uri, self._wire_client.get_header_for_cert())
+                    remote_access = RemoteAccess(xml_text)
+                    self._history.save_remote_access(xml_text)
 
             self._incarnation = incarnation
             self._role_instance_id = role_instance_id

azurelinuxagent/common/protocol/util.py

@@ -26,6 +26,7 @@
 import azurelinuxagent.common.conf as conf
 import azurelinuxagent.common.logger as logger
 import azurelinuxagent.common.utils.fileutil as fileutil
+from azurelinuxagent.common.protocol.goal_state import GoalStateProperties
 from azurelinuxagent.common.singletonperthread import SingletonPerThread
 
 from azurelinuxagent.common.exception import ProtocolError, OSUtilError, \
@@ -188,7 +189,7 @@ def _clear_wireserver_endpoint(self):
                 return
             logger.error("Failed to clear wiresever endpoint: {0}", e)
 
-    def _detect_protocol(self):
+    def _detect_protocol(self, goalstate_properties=GoalStateProperties.default_properties()):
         """
         Probe protocol endpoints in turn.
         """
@@ -217,7 +218,7 @@ def _detect_protocol(self):
 
                 try:
                     protocol = WireProtocol(endpoint)
-                    protocol.detect()
+                    protocol.detect(goalstate_properties=goalstate_properties)
                     self._set_wireserver_endpoint(endpoint)
                     return protocol
 
@@ -268,7 +269,7 @@ def clear_protocol(self):
         finally:
             self._lock.release()
 
-    def get_protocol(self):
+    def get_protocol(self, goalstate_properties=GoalStateProperties.default_properties()):
         """
         Detect protocol by endpoint.
         :returns: protocol instance
@@ -296,7 +297,7 @@ def get_protocol(self):
 
             logger.info("Detect protocol endpoint")
 
-            protocol = self._detect_protocol()
+            protocol = self._detect_protocol(goalstate_properties=goalstate_properties)
 
             IOErrorCounter.set_protocol_endpoint(endpoint=protocol.get_endpoint())
             self._save_protocol(WIRE_PROTOCOL_NAME)

azurelinuxagent/common/protocol/wire.py

@@ -37,7 +37,8 @@
 from azurelinuxagent.common.exception import ProtocolNotFoundError, \
     ResourceGoneError, ExtensionDownloadError, InvalidContainerError, ProtocolError, HttpError, ExtensionErrorCodes
 from azurelinuxagent.common.future import httpclient, bytebuffer, ustr
-from azurelinuxagent.common.protocol.goal_state import GoalState, TRANSPORT_CERT_FILE_NAME, TRANSPORT_PRV_FILE_NAME
+from azurelinuxagent.common.protocol.goal_state import GoalState, TRANSPORT_CERT_FILE_NAME, TRANSPORT_PRV_FILE_NAME, \
+    GoalStateProperties
 from azurelinuxagent.common.protocol.hostplugin import HostPluginProtocol
 from azurelinuxagent.common.protocol.restapi import DataContract, ProvisionStatus, VMInfo, VMStatus
 from azurelinuxagent.common.telemetryevent import GuestAgentExtensionEventsSchema
@@ -72,7 +73,7 @@ def __init__(self, endpoint):
             raise ProtocolError("WireProtocol endpoint is None")
         self.client = WireClient(endpoint)
 
-    def detect(self):
+    def detect(self, goalstate_properties=GoalStateProperties.default_properties()):
         self.client.check_wire_protocol_version()
 
         trans_prv_file = os.path.join(conf.get_lib_dir(),
@@ -84,7 +85,7 @@ def detect(self):
 
         # Initialize the goal state, including all the inner properties
         logger.info('Initializing goal state during protocol detection')
-        self.client.update_goal_state(force_update=True)
+        self.client.update_goal_state(goalstate_properties=goalstate_properties, force_update=True)
 
     def update_goal_state(self, silent=False):
         self.client.update_goal_state(silent=silent)
@@ -778,7 +779,7 @@ def update_host_plugin(self, container_id, role_config_name):
             self._host_plugin.update_container_id(container_id)
             self._host_plugin.update_role_config_name(role_config_name)
 
-    def update_goal_state(self, force_update=False, silent=False):
+    def update_goal_state(self, goalstate_properties=GoalStateProperties.default_properties(), force_update=False, silent=False):
         """
         Updates the goal state if the incarnation or etag changed or if 'force_update' is True
         """
@@ -787,9 +788,9 @@ def update_goal_state(self, force_update=False, silent=False):
                 logger.info("Forcing an update of the goal state.")
 
             if self._goal_state is None or force_update:
-                self._goal_state = GoalState(self, silent=silent)
+                self._goal_state = GoalState(self, goalstate_properties=goalstate_properties, silent=silent)
             else:
-                self._goal_state.update(silent=silent)
+                self._goal_state.update(goalstate_properties=goalstate_properties, silent=silent)
 
         except ProtocolError:
             raise