Add API for uploading logs via host plugin

azurelinuxagent/common/protocol/hostplugin.py

@@ -24,7 +24,7 @@
 
 from azurelinuxagent.common import logger
 from azurelinuxagent.common.errorstate import ErrorState, ERROR_STATE_HOST_PLUGIN_FAILURE
-from azurelinuxagent.common.event import WALAEventOperation, report_event
+from azurelinuxagent.common.event import WALAEventOperation, add_event
 from azurelinuxagent.common.exception import HttpError, ProtocolError
 from azurelinuxagent.common.future import ustr
 from azurelinuxagent.common.protocol.healthservice import HealthService
@@ -43,15 +43,15 @@
 
 API_VERSION = "2015-09-01"
 
-HEADER_CLIENT_NAME = "x-ms-client-name"
-HEADER_CLIENT_VERSION = "x-ms-client-version"
-HEADER_CORRELATION_ID = "x-ms-client-correlationid"
-HEADER_CONTAINER_ID = "x-ms-containerid"
-HEADER_DEPLOYMENT_ID = "x-ms-vmagentlog-deploymentid"
-HEADER_VERSION = "x-ms-version"
-HEADER_HOST_CONFIG_NAME = "x-ms-host-config-name"
-HEADER_ARTIFACT_LOCATION = "x-ms-artifact-location"
-HEADER_ARTIFACT_MANIFEST_LOCATION = "x-ms-artifact-manifest-location"
+_HEADER_CLIENT_NAME = "x-ms-client-name"
+_HEADER_CLIENT_VERSION = "x-ms-client-version"
+_HEADER_CORRELATION_ID = "x-ms-client-correlationid"
+_HEADER_CONTAINER_ID = "x-ms-containerid"
+_HEADER_DEPLOYMENT_ID = "x-ms-vmagentlog-deploymentid"
+_HEADER_VERSION = "x-ms-version"
+_HEADER_HOST_CONFIG_NAME = "x-ms-host-config-name"
+_HEADER_ARTIFACT_LOCATION = "x-ms-artifact-location"
+_HEADER_ARTIFACT_MANIFEST_LOCATION = "x-ms-artifact-manifest-location"
 
 MAXIMUM_PAGEBLOB_PAGE_SIZE = 4 * 1024 * 1024  # Max page size: 4MB
 
@@ -107,8 +107,8 @@ def ensure_initialized(self):
             self.api_versions = self.get_api_versions()
             self.is_available = API_VERSION in self.api_versions
             self.is_initialized = self.is_available
-            report_event(WALAEventOperation.InitializeHostPlugin,
-                         is_success=self.is_available)
+            add_event(WALAEventOperation.InitializeHostPlugin,
+                      is_success=self.is_available)
         return self.is_available
 
     def get_health(self):
@@ -131,7 +131,7 @@ def get_api_versions(self):
         error_response = ''
         is_healthy = False
         try:
-            headers = {HEADER_CONTAINER_ID: self.container_id}
+            headers = {_HEADER_CONTAINER_ID: self.container_id}
             response = restutil.http_get(url, headers)
 
             if restutil.request_failed(response):
@@ -157,13 +157,13 @@ def get_artifact_request(self, artifact_url, artifact_manifest_url=None):
 
         url = URI_FORMAT_GET_EXTENSION_ARTIFACT.format(self.endpoint,
                                                        HOST_PLUGIN_PORT)
-        headers = {HEADER_VERSION: API_VERSION,
-                   HEADER_CONTAINER_ID: self.container_id,
-                   HEADER_HOST_CONFIG_NAME: self.role_config_name,
-                   HEADER_ARTIFACT_LOCATION: artifact_url}
+        headers = {_HEADER_VERSION: API_VERSION,
+                   _HEADER_CONTAINER_ID: self.container_id,
+                   _HEADER_HOST_CONFIG_NAME: self.role_config_name,
+                   _HEADER_ARTIFACT_LOCATION: artifact_url}
 
         if artifact_manifest_url is not None:
-            headers[HEADER_ARTIFACT_MANIFEST_LOCATION] = artifact_manifest_url
+            headers[_HEADER_ARTIFACT_MANIFEST_LOCATION] = artifact_manifest_url
 
         return url, headers
 
@@ -224,11 +224,8 @@ def put_vm_log(self, content):
         if not self.ensure_initialized():
             raise ProtocolError("HostGAPlugin: HostGAPlugin is not available")
 
-        if content is None or self.container_id is None or self.deployment_id is None:
-            raise ProtocolError("HostGAPlugin: Invalid arguments passed to upload VM logs. "
-                                "Content: {0}, container id: {1}, deployment id: {2}".format(content,
-                                                                                             self.container_id,
-                                                                                             self.deployment_id))
+        if content is None:
+            raise ProtocolError("HostGAPlugin: Invalid argument passed to upload VM logs. Content was not provided.")
 
         url = URI_FORMAT_PUT_LOG.format(self.endpoint, HOST_PLUGIN_PORT)
         response = restutil.http_put(url,
@@ -359,20 +356,20 @@ def _build_status_data(self, sas_url, blob_headers, content=None):
 
     def _build_status_headers(self):
         return {
-            HEADER_VERSION: API_VERSION,
+            _HEADER_VERSION: API_VERSION,
             "Content-type": "application/json",
-            HEADER_CONTAINER_ID: self.container_id,
-            HEADER_HOST_CONFIG_NAME: self.role_config_name
+            _HEADER_CONTAINER_ID: self.container_id,
+            _HEADER_HOST_CONFIG_NAME: self.role_config_name
         }
 
     def _build_log_headers(self):
         return {
-            HEADER_VERSION: API_VERSION,
-            HEADER_CONTAINER_ID: self.container_id,
-            HEADER_DEPLOYMENT_ID: self.deployment_id,
-            HEADER_CLIENT_NAME: AGENT_NAME,
-            HEADER_CLIENT_VERSION: AGENT_VERSION,
-            HEADER_CORRELATION_ID: str(uuid.uuid4())
+            _HEADER_VERSION: API_VERSION,
+            _HEADER_CONTAINER_ID: self.container_id,
+            _HEADER_DEPLOYMENT_ID: self.deployment_id,
+            _HEADER_CLIENT_NAME: AGENT_NAME,
+            _HEADER_CLIENT_VERSION: AGENT_VERSION,
+            _HEADER_CORRELATION_ID: str(uuid.uuid4())
         }
 
     def _base64_encode(self, data):

azurelinuxagent/common/protocol/wire.py

@@ -1208,15 +1208,17 @@ def upload_logs(self, content):
             host_plugin = self.get_host_plugin()
             host_plugin.put_vm_log(content)
 
-            msg = "Upload VM logs request succeeded using the host plugin channel for " \
+            msg = "Upload VM logs request succeeded using the host plugin channel with " \
                   "container id {0} and role config file {1}".format(host_plugin.container_id,
                                                                      host_plugin.role_config_name)
+            logger.info(msg)
             add_event(name=AGENT_NAME,
                       version=CURRENT_VERSION,
                       op=WALAEventOperation.HostPlugin,
                       is_success=True,
                       message=msg,
-                      log_event=True)
+                      log_event=False)
+            return
         except (ResourceGoneError, InvalidContainerError) as e:
             host_plugin = self.get_host_plugin()
             old_container_id = host_plugin.container_id
@@ -1243,13 +1245,14 @@ def upload_logs(self, content):
                       "ContainerId changed from {0} to {1}, " \
                       "role config file changed from {2} to {3}.".format(old_container_id, new_container_id,
                                                                          old_role_config_name, new_role_config_name)
+                logger.info(msg)
                 add_event(name=AGENT_NAME,
                           version=CURRENT_VERSION,
                           op=WALAEventOperation.HostPlugin,
                           is_success=True,
                           message=msg,
-                          log_event=True)
-
+                          log_event=False)
+                return
             except (ResourceGoneError, InvalidContainerError) as e:
                 msg = "Upload VM logs request failed using the host plugin channel after goal state refresh. " \
                       "ContainerId changed from {0} to {1}, role config file changed from {2} to {3}. " \
@@ -1262,6 +1265,8 @@ def upload_logs(self, content):
                           message=msg,
                           log_event=True)
                 raise
+        except Exception as e:
+            raise ProtocolError("Failed to upload logs. Error: {0}".format(ustr(e)))
 
 
 class VersionInfo(object):

tests/protocol/test_hostplugin.py

@@ -21,18 +21,18 @@
 import json
 import sys
 import unittest
-import uuid
 
 import azurelinuxagent.common.protocol.hostplugin as hostplugin
 import azurelinuxagent.common.protocol.restapi as restapi
 import azurelinuxagent.common.protocol.wire as wire
 from azurelinuxagent.common.errorstate import ErrorState
 from azurelinuxagent.common.exception import HttpError, ResourceGoneError
 from azurelinuxagent.common.future import ustr
+from azurelinuxagent.common.osutil.default import UUID_PATTERN
 from azurelinuxagent.common.protocol.hostplugin import API_VERSION
 from azurelinuxagent.common.utils import restutil
 from azurelinuxagent.common.version import AGENT_VERSION, AGENT_NAME
-from tests.protocol.mocks import mock_wire_protocol
+from tests.protocol.mocks import mock_wire_protocol, HttpRequestPredicates
 from tests.protocol.mockwiredata import DATA_FILE, DATA_FILE_NO_EXT
 from tests.protocol.test_wire import MockResponse as TestWireMockResponse
 from tests.tools import AgentTestCase, PY_VERSION_MAJOR, Mock, patch
@@ -64,7 +64,7 @@
     faux_status_b64 = faux_status_b64.decode('utf-8')
 
 
-class TestHostPlugin(AgentTestCase):
+class TestHostPlugin(HttpRequestPredicates, AgentTestCase):
 
     def _init_host(self):
         with mock_wire_protocol(DATA_FILE) as protocol:
@@ -174,7 +174,7 @@ def create_mock_protocol():
 
     @patch("azurelinuxagent.common.protocol.healthservice.HealthService.report_host_plugin_versions")
     @patch("azurelinuxagent.ga.update.restutil.http_get")
-    @patch("azurelinuxagent.common.protocol.hostplugin.report_event")
+    @patch("azurelinuxagent.common.protocol.hostplugin.add_event")
     def assert_ensure_initialized(self, patch_event, patch_http_get, patch_report_health,
                                   response_body,
                                   response_status_code,
@@ -539,42 +539,48 @@ def test_validate_page_blobs(self):
                         exp_method, exp_url, exp_data)
 
     def test_validate_http_request_when_uploading_logs(self):
-        with mock_wire_protocol(DATA_FILE) as protocol:
-            test_goal_state = protocol.client._goal_state
-            correlation_id = str(uuid.uuid4())
+        def http_put_handler(url, *args, **kwargs):
+            if self.is_host_plugin_put_logs_request(url):
+                http_put_handler.args, http_put_handler.kwargs = args, kwargs
+                return MockResponse(body=b'', status_code=200)
+            self.fail('The upload logs request was sent to the wrong url: {0}'.format(url))
+
+        http_put_handler.args, http_put_handler.kwargs = [], {}
+
+        with mock_wire_protocol(DATA_FILE, http_put_handler=http_put_handler) as protocol:
+            test_goal_state = protocol.client.get_goal_state()
 
             expected_url = hostplugin.URI_FORMAT_PUT_LOG.format(wireserver_url, hostplugin.HOST_PLUGIN_PORT)
             expected_headers = {'x-ms-version': '2015-09-01',
                                 "x-ms-containerid": test_goal_state.container_id,
                                 "x-ms-vmagentlog-deploymentid": test_goal_state.role_config_name.split(".")[0],
                                 "x-ms-client-name": AGENT_NAME,
-                                "x-ms-client-version": AGENT_VERSION,
-                                "x-ms-client-correlationid": correlation_id}
+                                "x-ms-client-version": AGENT_VERSION}
 
             host_client = wire.HostPluginProtocol(wireserver_url,
                                                   test_goal_state.container_id,
                                                   test_goal_state.role_config_name)
 
             self.assertFalse(host_client.is_initialized, "Host plugin should not be initialized!")
 
-            with patch.object(restutil, "http_request") as patch_http:
-                with patch.object(wire.HostPluginProtocol, "get_api_versions", return_value=api_versions):
-                    with patch("azurelinuxagent.common.protocol.hostplugin.uuid.uuid4", return_value=correlation_id):
-                        patch_http.return_value = Mock(status=httpclient.OK)
-
-                        content = b"test"
-                        host_client.put_vm_log(content)
-                        self.assertTrue(host_client.is_initialized, "Host plugin is not initialized!")
-
-                        args, kwargs = patch_http.call_args_list[0]
-                        self.assertEqual('PUT', args[0], "Expected HTTP request method PUT!")
-                        self.assertEqual(expected_url, args[1], "Unexpected request URL!")
-                        self.assertEqual(content, args[2], "Unexpected content for HTTP PUT request!")
-
-                        headers = kwargs['headers']
-                        for k in expected_headers:
-                            self.assertTrue(k in headers, "Header {0} not found in headers!".format(k))
-                            self.assertEqual(expected_headers[k], headers[k], "Request headers don't match!")
+            content = b"test"
+            host_client.put_vm_log(content)
+            self.assertTrue(host_client.is_initialized, "Host plugin is not initialized!")
+
+            urls = protocol.get_tracked_urls()
+
+            self.assertEqual(expected_url, urls[0], "Unexpected request URL!")
+            self.assertEqual(content, http_put_handler.args[0], "Unexpected content for HTTP PUT request!")
+
+            headers = http_put_handler.kwargs['headers']
+            for k in expected_headers:
+                self.assertTrue(k in headers, "Header {0} not found in headers!".format(k))
+                self.assertEqual(expected_headers[k], headers[k], "Request headers don't match!")
+
+            # Special check for correlation id header value, check for pattern, not exact value
+            self.assertTrue("x-ms-client-correlationid" in headers.keys(), "Correlation id not found in headers!")
+            self.assertTrue(UUID_PATTERN.match(headers["x-ms-client-correlationid"]),
+                            "Correlation id is not in GUID form!")
 
     def test_validate_get_extension_artifacts(self):
         with mock_wire_protocol(DATA_FILE) as protocol:

tests/protocol/test_wire.py

@@ -817,7 +817,7 @@ def http_get_handler(url, *_, **kwargs):
 
     def test_upload_logs_should_not_refresh_plugin_when_first_attempt_succeeds(self):
         def http_put_handler(url, *_, **__):
-            if protocol.get_endpoint() in url and url.endswith('/vmAgentLog'):
+            if self.is_host_plugin_put_logs_request(url):
                 return MockResponse(body=b'', status_code=200)
             self.fail('The upload logs request was sent to the wrong url: {0}'.format(url))