Add sample remote tests (#2888)

* Add sample remote tests

* add pass

* review feedback

---------

Co-authored-by: narrieta <narrieta>

tests_e2e/orchestrator/lib/agent_test_suite.py

@@ -46,7 +46,7 @@
 from azurelinuxagent.common.version import AGENT_VERSION
 from tests_e2e.orchestrator.lib.agent_test_loader import TestSuiteInfo
 from tests_e2e.tests.lib.agent_log import AgentLog
-from tests_e2e.tests.lib.agent_test import TestSkipped
+from tests_e2e.tests.lib.agent_test import TestSkipped, RemoteTestError
 from tests_e2e.tests.lib.agent_test_context import AgentTestContext
 from tests_e2e.tests.lib.identifiers import VmIdentifier
 from tests_e2e.tests.lib.logging import log
@@ -531,17 +531,29 @@ def _execute_test_suite(self, suite: TestSuiteInfo) -> bool:
                                 TestStatus.FAILED,
                                 test_start_time,
                                 message=str(e))
+                        except RemoteTestError as e:
+                            test_success = False
+                            summary.append(f"[Failed]  {test.name}")
+                            message = f"UNEXPECTED ERROR IN [{e.command}] {e.stderr}\n{e.stdout}"
+                            log.error("******** [Failed] %s: %s", test.name, message)
+                            self.context.lisa_log.error("******** [Failed] %s", test_full_name)
+                            self._report_test_result(
+                                suite_full_name,
+                                test.name,
+                                TestStatus.FAILED,
+                                test_start_time,
+                                message=str(message))
                         except:  # pylint: disable=bare-except
                             test_success = False
                             summary.append(f"[Error]   {test.name}")
-                            log.exception("UNHANDLED EXCEPTION IN %s", test.name)
-                            self.context.lisa_log.exception("UNHANDLED EXCEPTION IN %s", test_full_name)
+                            log.exception("UNEXPECTED ERROR IN %s", test.name)
+                            self.context.lisa_log.exception("UNEXPECTED ERROR IN %s", test_full_name)
                             self._report_test_result(
                                 suite_full_name,
                                 test.name,
                                 TestStatus.FAILED,
                                 test_start_time,
-                                message="Unhandled exception.",
+                                message="Unexpected error.",
                                 add_exception_stack_trace=True)
 
                         log.info("")

tests_e2e/test_suites/fail.yml

@@ -1,5 +1,7 @@
 name: "Fail"
 tests:
-  - "fail_test.py"
-  - "error_test.py"
-images: "ubuntu_1804"
+  - "samples/fail_test.py"
+  - "samples/fail_remote_test.py"
+  - "samples/error_test.py"
+  - "samples/error_remote_test.py"
+images: "ubuntu_2004"

tests_e2e/test_suites/pass.yml

@@ -1,4 +1,5 @@
 name: "Pass"
 tests:
-  - "pass_test.py"
+  - "samples/pass_test.py"
+  - "samples/pass_remote_test.py"
 images: "ubuntu_2004"

tests_e2e/tests/lib/agent_test.py

@@ -20,10 +20,14 @@
 import sys
 
 from abc import ABC, abstractmethod
+from assertpy import fail
 from typing import Any, Dict, List
 
 from tests_e2e.tests.lib.agent_test_context import AgentTestContext
 from tests_e2e.tests.lib.logging import log
+from tests_e2e.tests.lib.remote_test import FAIL_EXIT_CODE
+from tests_e2e.tests.lib.shell import CommandError
+from tests_e2e.tests.lib.ssh_client import ATTEMPTS, ATTEMPT_DELAY, SshClient
 
 
 class TestSkipped(Exception):
@@ -33,6 +37,12 @@ class TestSkipped(Exception):
     """
 
 
+class RemoteTestError(CommandError):
+    """
+    Raised when a remote test fails with an unexpected error.
+    """
+
+
 class AgentTest(ABC):
     """
     Defines the interface for agent tests, which are simply constructed from an AgentTestContext and expose a single method,
@@ -59,8 +69,28 @@ def run_from_command_line(cls):
             cls(AgentTestContext.from_args()).run()
         except SystemExit:  # Bad arguments
             pass
+        except AssertionError as e:
+            log.error("%s", e)
+            sys.exit(1)
         except:  # pylint: disable=bare-except
             log.exception("Test failed")
             sys.exit(1)
 
         sys.exit(0)
+
+    def _run_remote_test(self, command: str, use_sudo: bool = False, attempts: int = ATTEMPTS, attempt_delay: int = ATTEMPT_DELAY) -> None:
+        """
+        Derived classes can use this method to execute a remote test (a test that runs over SSH).
+        """
+        try:
+            ssh_client: SshClient = self._context.create_ssh_client()
+            output = ssh_client.run_command(command=command, use_sudo=use_sudo, attempts=attempts, attempt_delay=attempt_delay)
+            log.info("*** PASSED: [%s]\n%s", command, self._indent(output))
+        except CommandError as error:
+            if error.exit_code == FAIL_EXIT_CODE:
+                fail(f"[{command}] {error.stderr}{self._indent(error.stdout)}")
+            raise RemoteTestError(command=error.command, exit_code=error.exit_code, stdout=self._indent(error.stdout), stderr=error.stderr)
+
+    @staticmethod
+    def _indent(text: str, indent: str = " " * 8):
+        return "\n".join(f"{indent}{line}" for line in text.splitlines())

tests_e2e/tests/lib/logging.py

@@ -20,6 +20,8 @@
 # for logging.
 #
 import contextlib
+import sys
+
 from logging import FileHandler, Formatter, Handler, Logger, StreamHandler, INFO
 from pathlib import  Path
 from threading import current_thread
@@ -46,7 +48,7 @@ class _AgentLoggingHandler(Handler):
     def __init__(self):
         super().__init__()
         self.formatter: Formatter = Formatter('%(asctime)s.%(msecs)03d [%(levelname)s] %(message)s', datefmt="%Y-%m-%dT%H:%M:%SZ")
-        self.default_handler = StreamHandler()
+        self.default_handler = StreamHandler(sys.stdout)
         self.default_handler.setFormatter(self.formatter)
         self.per_thread_handlers: Dict[int, FileHandler] = {}
 

tests_e2e/tests/lib/remote_test.py

@@ -0,0 +1,48 @@
+#!/usr/bin/env pypy3
+
+# Microsoft Azure Linux Agent
+#
+# Copyright 2018 Microsoft Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+import sys
+
+from typing import Callable
+
+from tests_e2e.tests.lib.logging import log
+
+SUCCESS_EXIT_CODE = 0
+FAIL_EXIT_CODE = 100
+ERROR_EXIT_CODE = 200
+
+
+def run_remote_test(test_method: Callable[[], int]) -> None:
+    """
+    Helper function to run a remote test; implements coding conventions for remote tests, e.g. error message goes
+    to stderr, test log goes to stdout, etc.
+    """
+    try:
+        test_method()
+        log.info("*** PASSED")
+    except AssertionError as e:
+        print(f"{e}", file=sys.stderr)
+        log.error("%s", e)
+        sys.exit(FAIL_EXIT_CODE)
+    except Exception as e:
+        print(f"UNEXPECTED ERROR: {e}", file=sys.stderr)
+        log.exception("*** UNEXPECTED ERROR")
+        sys.exit(ERROR_EXIT_CODE)
+
+    sys.exit(SUCCESS_EXIT_CODE)
+

tests_e2e/tests/samples/error_remote_test.py

@@ -0,0 +1,32 @@
+#!/usr/bin/env python3
+
+# Microsoft Azure Linux Agent
+#
+# Copyright 2018 Microsoft Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+from tests_e2e.tests.lib.agent_test import AgentTest
+
+
+class ErrorRemoteTest(AgentTest):
+    """
+    A trivial remote test that fails
+    """
+    def run(self):
+        self._run_remote_test("samples-error_remote_test.py")
+
+
+if __name__ == "__main__":
+    ErrorRemoteTest.run_from_command_line()

tests_e2e/tests/error_test.py → tests_e2e/tests/samples/error_test.py

@@ -25,7 +25,7 @@ class ErrorTest(AgentTest):
     A trivial test that errors out
     """
     def run(self):
-        raise Exception("* ERROR *")
+        raise Exception("* TEST ERROR *")  # simulate an unexpected error
 
 
 if __name__ == "__main__":

tests_e2e/tests/samples/fail_remote_test.py

@@ -0,0 +1,32 @@
+#!/usr/bin/env python3
+
+# Microsoft Azure Linux Agent
+#
+# Copyright 2018 Microsoft Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+from tests_e2e.tests.lib.agent_test import AgentTest
+
+
+class FailRemoteTest(AgentTest):
+    """
+    A trivial remote test that fails
+    """
+    def run(self):
+        self._run_remote_test("samples-fail_remote_test.py")
+
+
+if __name__ == "__main__":
+    FailRemoteTest.run_from_command_line()

tests_e2e/tests/fail_test.py → tests_e2e/tests/samples/fail_test.py

@@ -26,7 +26,7 @@ class FailTest(AgentTest):
     A trivial test that fails
     """
     def run(self):
-        fail("* FAILED *")
+        fail("* TEST FAILED *")
 
 
 if __name__ == "__main__":

tests_e2e/tests/samples/pass_remote_test.py

@@ -0,0 +1,32 @@
+#!/usr/bin/env python3
+
+# Microsoft Azure Linux Agent
+#
+# Copyright 2018 Microsoft Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+from tests_e2e.tests.lib.agent_test import AgentTest
+
+
+class PassRemoteTest(AgentTest):
+    """
+    A trivial remote test that succeeds
+    """
+    def run(self):
+        self._run_remote_test("samples-pass_remote_test.py")
+
+
+if __name__ == "__main__":
+    PassRemoteTest.run_from_command_line()

tests_e2e/tests/scripts/samples-error_remote_test.py

@@ -0,0 +1,36 @@
+#!/usr/bin/env pypy3
+
+# Microsoft Azure Linux Agent
+#
+# Copyright 2018 Microsoft Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+#
+# A sample remote test that simulates an unexpected error
+#
+
+from tests_e2e.tests.lib.logging import log
+from tests_e2e.tests.lib.remote_test import run_remote_test
+
+
+def main():
+    log.info("Setting up test")
+    log.info("Doing some operation")
+    log.warning("Something went wrong, but the test can continue")
+    log.info("Doing some other operation")
+    raise Exception("Something went wrong")  # simulate an unexpected error
+
+
+run_remote_test(main)

tests_e2e/tests/scripts/samples-fail_remote_test.py

@@ -0,0 +1,37 @@
+#!/usr/bin/env pypy3
+
+# Microsoft Azure Linux Agent
+#
+# Copyright 2018 Microsoft Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+#
+# A sample remote test that fails
+#
+
+from assertpy import fail
+from tests_e2e.tests.lib.logging import log
+from tests_e2e.tests.lib.remote_test import run_remote_test
+
+
+def main():
+    log.info("Setting up test")
+    log.info("Doing some operation")
+    log.warning("Something went wrong, but the test can continue")
+    log.info("Doing some other operation")
+    fail("Verification of the operation failed")
+
+
+run_remote_test(main)