[draft] test: integration test framework

.gitmodules

@@ -0,0 +1,6 @@
+[submodule "tests/integration/mender_integration"]
+	path = tests/integration/mender_integration
+	url = https://github.com/mendersoftware/integration.git
+[submodule "tests/integration/mender_server"]
+	path = tests/integration/mender_server
+	url = https://github.com/mendersoftware/mender-server.git

CMakeLists.txt

@@ -4,11 +4,23 @@ find_package(Zephyr REQUIRED HINTS $ENV{ZEPHYR_BASE})
 
 project(mender-mcu-integration)
 
+if (INTEGRATION_TESTS MATCHES "y")
+target_sources(app PRIVATE
+    tests/integration/src/main.c
+    src/utils/netup.c
+    src/utils/certs.c
+)
+target_include_directories(app PUBLIC
+    tests/integration/src
+)
+target_sources(app PRIVATE tests/integration/src//modules/test-update-module.c)
+else()
 target_sources(app PRIVATE
     src/main.c
     src/utils/netup.c
     src/utils/certs.c
 )
+endif()
 
 target_include_directories(app PUBLIC
     src

tests/integration/README.md

@@ -0,0 +1,23 @@
+## Running the integration tests locally
+
+###### NOTE: This does _not_ work with mender-mcu main due to restructuring of CMakeLists etc.
+The latest commit this is tested with is `208190909d47d48a5bcf33ac0aa606dc05af4182` (mender-mcu)
+There is a ticket to review the `C parts`, so this should be rebased on top of main there.
+
+Pre-requisites:
+ - networking for the native-sim board (see README)
+ - pytest
+
+To run the current tests, you will need to export
+ - TEST_DEVICE_ID
+ - TEST_TENANT_TOKEN
+ - TEST_AUTH_TOKEN
+
+as these are used to connect to hosted Mender.
+
+You will also need to run `git submodule update --init --recursive` to get
+the parts from mender-server (there are also parts from mender-integration, but
+as of now this is just some small logging parts etc., we should remove the dependency on
+the integration repo).
+
+Run the tests with `pytest -s --host hosted.mender.io` in the `tests/integration` directory.

tests/integration/conftest.py

@@ -0,0 +1,66 @@
+# Copyright 2024 Northern.tech AS
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+
+import os
+from os import path
+
+import sys
+import logging
+import pytest
+
+sys.path += [path.join(path.dirname(__file__), "mender_integration")]
+sys.path += [
+    path.join(path.dirname(__file__), "mender_server/backend/tests/integration/tests/")
+]
+
+from server import Server
+
+from mender_integration.tests.conftest import unique_test_name
+from mender_integration.tests.log import setup_test_logger
+
+logging.getLogger("requests").setLevel(logging.CRITICAL)
+
+logging.basicConfig()
+logging.getLogger().setLevel(logging.DEBUG)
+
+collect_ignore = ["mender_integration", "mender_server"]
+
+
+@pytest.fixture(scope="function", autouse=True)
+def testlogger(request):
+    test_name = unique_test_name(request)
+    setup_test_logger(test_name)
+    logging.getLogger().info("%s is starting.... " % test_name)
+
+
+def pytest_addoption(parser):
+    parser.addoption(
+        "--host",
+        action="store",
+        default="docker.mender.io",
+        help="Server URL for tests",
+    )
+
+
+@pytest.fixture(scope="session", autouse=True)
+def setup_user():
+    # for now this just returns auth token from env
+    # but for demo server, it should create a user aswell
+    return os.getenv("TEST_AUTH_TOKEN")
+
+
+@pytest.fixture(scope="session", autouse=True)
+def server(setup_user, request):
+    host_url = request.config.getoption("--host")
+    return Server(auth_token=setup_user, host=host_url)

tests/integration/definitions.py

@@ -0,0 +1,45 @@
+# Copyright 2024 Northern.tech AS
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+
+################
+# Callbacks
+################
+
+# Callbacks defined in the update module
+UM_DOWNLOAD_CALLBACK = "UM_DOWNLOAD_CALLBACK"
+UM_INSTALL_CALLBACK = "UM_INSTALL_CALLBACK"
+UM_REBOOT_CALLBACK = "UM_REBOOT_CALLBACK"
+UM_VERIFY_REBOOT_CALLBACK = "UM_VERIFY_REBOOT_CALLBACK"
+UM_COMMIT_CALLBACK = "UM_COMMIT_CALLBACK"
+UM_ROLLBACK_CALLBACK = "UM_ROLLBACK_CALLBACK"
+UM_ROLLBACK_REBOOT_CALLBACK = "UM_ROLLBACK_REBOOT_CALLBACK"
+UM_FAILURE_CALLBACK = "UM_FAILURE_CALLBACK"
+
+
+# Callback defined in main
+NETWORK_CONNECT_CALLBACK = "NETWORK_CONNECT_CALLBACK"
+NETWORK_RELEASE_CALLBACK = "NETWORK_RELEASE_CALLBACK"
+DEPLOYMENT_STATUS_CALLBACK = "DEPLOYMENT_STATUS_CALLBACK"
+RESTART_CALLBACK = "RESTART_CALLBACK"
+GET_IDENTITY_CALLBACK = "GET_IDENTITY_CALLBACK"
+
+
+################
+# Variables
+################
+
+# Update module requires reboot
+UM_REQUIRES_REBOOT = "UM_REQUIRES_REBOOT"
+# Update module supports rollback
+UM_SUPPORTS_ROLLBACK = "UM_SUPPORTS_ROLLBACK"

tests/integration/device.py

@@ -0,0 +1,130 @@
+# Copyright 2024 Northern.tech AS
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+
+import os
+import time
+import pytest
+import shutil
+import tempfile
+import subprocess
+from mender_integration.tests.MenderAPI import logger
+
+from helpers import stdout
+from helpers import create_header_file
+
+THIS_DIR = os.path.dirname(os.path.abspath(__file__))
+
+# This has to point the west workspace containing mender-mcu-integration
+WORKSPACE_DIRECTORY = os.path.join(THIS_DIR, "../../")
+
+
+class DeviceStatus:
+    def __init__(self, device):
+        self.device = device
+
+    def is_authenticated(self, timeout=60):
+        logger.info("Waiting for device to authenticate")
+        start_time = time.time()
+        while time.time() - start_time < timeout:
+            line = stdout(self.device)
+            if "Authenticated successfully" in line:
+                logger.info("Device authenticated")
+                return True
+        return False
+
+    def is_aborted(self, timeout=60):
+        start_time = time.time()
+        while time.time() - start_time < timeout:
+            logger.info("Waiting for deployment to abort")
+            line = stdout(self.device)
+            if "Deployment aborted" in line:
+                return True
+        return False
+
+
+class NativeSim:
+    def __init__(self, stdout=False):
+        self.tenant_token = "..."
+        self.proc = None
+        self.stdout = stdout
+
+        self.server_host = ""
+        self.server_tenant = ""
+
+        self.status = DeviceStatus(self)
+
+        # Defaults to `https://docker.mender.io`
+        self.set_host()
+
+        create_header_file()
+
+        self.build_dir = tempfile.mkdtemp()
+
+    def set_host(self, host="https://docker.mender.io"):
+        self.server_host = host
+
+    def set_tenant(self, tenant):
+        self.server_tenant = tenant
+
+    def compile(self, pristine=False, extra_variables=None):
+        if extra_variables is None:
+            extra_variables = []
+
+        if compile:
+            variables = [
+                "-DCONFIG_LOG_ALWAYS_RUNTIME=y",
+                "-DCONFIG_LOG_MODE_IMMEDIATE=y",
+                "-DCONFIG_MENDER_LOG_LEVEL_OFF=n",
+                "-DCONFIG_MENDER_LOG_LEVEL_DBG=y",
+                f'-DINTEGRATION_TESTS="y"',
+                f'-DCONFIG_MENDER_SERVER_HOST="{self.server_host}"',
+                f'-DCONFIG_MENDER_SERVER_TENANT_TOKEN="{self.server_tenant}"',
+            ] + extra_variables
+            command = (
+                ["west", "build", "--board", "native_sim", WORKSPACE_DIRECTORY]
+                + variables
+                + ["--build-dir", f"{self.build_dir}"]
+                + (["--pristine"] if pristine else [])
+            )
+
+            try:
+                subprocess.check_call(command)
+            except subprocess.CalledProcessError as result:
+                logger.error(result.stderr)
+                command_output = " ".join(command)
+                pytest.fail(f"Failed to compile with command: {command_output}")
+
+    def start(self, compile=True, pristine=False, extra_variables=[]):
+        if compile:
+            self.compile(pristine=pristine, extra_variables=extra_variables)
+
+        self.proc = subprocess.Popen(
+            [
+                f"{self.build_dir}/zephyr/zephyr.exe",
+                f"--flash={self.build_dir}/flash.bin",
+            ],
+            stdout=subprocess.PIPE,
+            stderr=subprocess.PIPE,
+            text=True,
+        )
+        logger.info("Started device")
+
+    def stop(self, stop_after=0):
+        time.sleep(stop_after)
+        self.proc.terminate()
+        self.proc.wait()
+        logger.info("Stopped device")
+
+    def clean_build(self):
+        shutil.rmtree(self.build_dir)