test: PoC integration test framework

Integration test framework for testing mender-mcu client with
native-sim.
Still requires work - e.g. making the demo server work, and probably
more work on the testing framework itself.
It uses  `integration` as a submodulel, so you need to init it.

You have to set tenant token, auth_token etc. manually, and
I've only tested it with 'hosted.mender.io'.

I went with the compile-in approach:
You can modify the callbacks directly in the test. You do this by
utilizing `set_callback()` from `helpers`. The available callbacks
you can modify are stored in `callbacks.py`. The callbacks have an #ifdef,
and the python script creates a header-file which is imported by the
test-update-module.

The test-update-module and the main-file for the tests are located in
the `src/` in the integration-tests directory.

Ticket: MEN-7599

Signed-off-by: Daniel Skinstad Drabitzius <daniel.drabitzius@northern.tech>

.gitmodules

@@ -0,0 +1,3 @@
+[submodule "tests/integration/mender_integration"]
+	path = tests/integration/mender_integration
+	url = https://github.com/mendersoftware/integration.git

CMakeLists.txt

@@ -4,11 +4,23 @@ find_package(Zephyr REQUIRED HINTS $ENV{ZEPHYR_BASE})
 
 project(mender-mcu-integration)
 
+if (INTEGRATION_TESTS MATCHES "y")
+target_sources(app PRIVATE
+    tests/integration/src/main.c
+    src/utils/netup.c
+    src/utils/certs.c
+)
+target_include_directories(app PUBLIC
+    tests/integration/src
+)
+target_sources(app PRIVATE tests/integration/src//modules/test-update-module.c)
+else()
 target_sources(app PRIVATE
     src/main.c
     src/utils/netup.c
     src/utils/certs.c
 )
+endif()
 
 target_include_directories(app PUBLIC
     src

tests/integration/callbacks.py

@@ -0,0 +1,21 @@
+# Copyright 2024 Northern.tech AS
+#
+#    All Rights Reserved
+
+# Callbacks defined in the update module
+UM_DOWNLOAD_CALLBACK = "UM_DOWNLOAD_CALLBACK"
+UM_INSTALL_CALLBACK = "UM_INSTALL_CALLBACK"
+UM_REBOOT_CALLBACK = "UM_REBOOT_CALLBACK"
+UM_VERIFY_REBOOT_CALLBACK = "UM_VERIFY_REBOOT_CALLBACK"
+UM_COMMIT_CALLBACK = "UM_COMMIT_CALLBACK"
+UM_ROLLBACK_CALLBACK = "UM_ROLLBACK_CALLBACK"
+UM_ROLLBACK_REBOOT_CALLBACK = "UM_ROLLBACK_REBOOT_CALLBACK"
+UM_FAILURE_CALLBACK = "UM_FAILURE_CALLBACK"
+
+
+# Callback defined in main
+NETWORK_CONNECT_CALLBACK = "NETWORK_CONNECT_CALLBACK"
+NETWORK_RELEASE_CALLBACK = "NETWORK_RELEASE_CALLBACK"
+DEPLOYMENT_STATUS_CALLBACK = "DEPLOYMENT_STATUS_CALLBACK"
+RESTART_CALLBACK = "RESTART_CALLBACK"
+GET_IDENTITY_CALLBACK = "GET_IDENTITY_CALLBACK"

tests/integration/conftest.py

@@ -0,0 +1,28 @@
+# Copyright 2024 Northern.tech AS
+#
+#    All Rights Reserved
+
+from os import path
+import sys
+
+sys.path += [path.join(path.dirname(__file__), "mender_integration")]
+
+import logging
+
+import pytest
+from mender_integration.tests.conftest import unique_test_name
+from mender_integration.tests.log import setup_test_logger
+
+logging.getLogger("requests").setLevel(logging.CRITICAL)
+
+logging.basicConfig()
+logging.getLogger().setLevel(logging.DEBUG)
+
+collect_ignore = ["mender_integration"]
+
+
+@pytest.fixture(scope="function", autouse=True)
+def testlogger(request):
+    test_name = unique_test_name(request)
+    setup_test_logger(test_name)
+    logging.getLogger().info("%s is starting.... " % test_name)

tests/integration/device.py

@@ -0,0 +1,109 @@
+# Copyright 2024 Northern.tech AS
+#
+#    All Rights Reserved
+
+import os
+import time
+import pytest
+import subprocess
+from mender_integration.tests.MenderAPI import logger
+
+from helpers import get_build_folder
+from helpers import stdout
+
+THIS_DIR = os.path.dirname(os.path.abspath(__file__))
+
+# This has to point the west workspace containing mender-mcu-integration
+WORKSPACE_DIRECTORY = os.path.join(THIS_DIR, "../../")
+
+
+class DeviceStatus:
+    def __init__(self, device):
+        self.device = device
+
+    def is_authenticated(self, timeout=60):
+        logger.info("Waiting for device to authenticate")
+        start_time = time.time()
+        while time.time() - start_time < timeout:
+            line = stdout(self.device)
+            if "Authenticated successfully" in line:
+                logger.info("Device authenticated")
+                return True
+        return False
+
+    def is_aborted(self, timeout=60):
+        start_time = time.time()
+        while time.time() - start_time < timeout:
+            logger.info("Waiting for deployment to abort")
+            line = stdout(self.device)
+            if "Deployment aborted" in line:
+                return True
+        return False
+
+
+class NativeSim:
+    def __init__(self, request, stdout=False):
+        self.tenant_token = "..."
+        self.stdout = ""
+        self.proc = None
+        self.stdout = stdout
+
+        self.build_folder = get_build_folder(request)
+
+        self.server_host = ""
+        self.server_tenant = ""
+
+        self.status = DeviceStatus(self)
+
+        # Defaults to `https://docker.mender.io`
+        self.set_host()
+
+    def set_host(self, host="https://docker.mender.io"):
+        self.server_host = host
+
+    def set_tenant(self, tenant):
+        self.server_tenant = tenant
+
+    def compile(self, pristine=False, extra_variables=[]):
+        if compile:
+            variables = [
+                "-DCONFIG_LOG_ALWAYS_RUNTIME=y",
+                "-DCONFIG_LOG_MODE_IMMEDIATE=y",
+                "-DCONFIG_MENDER_LOG_LEVEL_OFF=n",
+                "-DCONFIG_MENDER_LOG_LEVEL_DBG=y",
+                f'-DINTEGRATION_TESTS="y"',
+                f'-DCONFIG_MENDER_SERVER_HOST="{self.server_host}"',
+                f'-DCONFIG_MENDER_SERVER_TENANT_TOKEN="{self.server_tenant}"',
+            ] + extra_variables
+            command = (
+                ["west", "build", "--board", "native_sim", WORKSPACE_DIRECTORY]
+                + variables
+                + ["--build-dir", f"{self.build_folder}"]
+                + (["--pristine"] if pristine else [])
+            )
+            result = subprocess.run(command, capture_output=True, text=True)
+            if result.returncode != 0:
+                logger.error(result.stderr)
+                command_output = " ".join(command)
+                pytest.fail(f"Failed to compile with command: {command_output}")
+
+    def start(self, compile=True, pristine=False, extra_variables=[]):
+        if compile:
+            self.compile(pristine=pristine, extra_variables=extra_variables)
+
+        self.proc = subprocess.Popen(
+            [
+                f"{self.build_folder}/zephyr/zephyr.exe",
+                f"--flash={self.build_folder}/flash.bin",
+            ],
+            stdout=subprocess.PIPE,
+            stderr=subprocess.PIPE,
+            text=True,
+        )
+        logger.info("Started device")
+
+    def stop(self, stop_after=0):
+        time.sleep(stop_after)
+        self.proc.terminate()
+        self.proc.wait()
+        logger.info("Stopped device")

tests/integration/helpers.py

@@ -0,0 +1,83 @@
+# Copyright 2024 Northern.tech AS
+#
+#    All Rights Reserved
+
+import os
+import random
+import string
+import tempfile
+import subprocess
+
+from contextlib import contextmanager
+
+from mender_integration.tests.MenderAPI import logger
+
+
+def get_header_file():
+    return "src/test_definitions.h"
+
+
+# The name of the available macros can be found in callbacks.py
+def set_callback(function_name, definition):
+    with open(get_header_file(), "a") as f:
+        f.write(f"#define {function_name}() \\{definition}\n")
+
+
+def get_build_folder(request):
+    return f"{request.node.name}_build"
+
+
+def stdout(device):
+    line = device.proc.stdout.readline()
+    if device.stdout:
+        logger.info(line)
+    return line
+
+
+# get_mender_artifact from testutils/common.py didn't supoprt uncompressed artifacts
+@contextmanager
+def get_uncompressed_mender_artifact(
+    artifact_name="test",
+    update_module="dummy",
+    device_types=("arm1",),
+    size=256,
+    depends=(),
+    provides=(),
+    compressed=False,
+):
+    data = "".join(random.choices(string.ascii_uppercase + string.digits, k=size))
+    f = tempfile.NamedTemporaryFile(delete=False)
+    f.write(data.encode("utf-8"))
+    f.close()
+    #
+    filename = f.name
+    artifact = "%s.mender" % filename
+    args = [
+        "mender-artifact",
+        "write",
+        "module-image",
+        "-o",
+        artifact,
+        "--artifact-name",
+        artifact_name,
+        "-T",
+        update_module,
+        "-f",
+        filename,
+    ]
+
+    if not compressed:
+        args.extend(["--compression", "none"])
+
+    for device_type in device_types:
+        args.extend(["-t", device_type])
+    for depend in depends:
+        args.extend(["--depends", depend])
+    for provide in provides:
+        args.extend(["--provides", provide])
+    try:
+        subprocess.call(args)
+        yield artifact
+    finally:
+        os.unlink(filename)
+        os.path.exists(artifact) and os.unlink(artifact)

tests/integration/server.py

@@ -0,0 +1,76 @@
+# Copyright 2024 Northern.tech AS
+#
+#    All Rights Reserved
+
+import os
+
+from helpers import get_uncompressed_mender_artifact
+
+from mender_integration.tests.MenderAPI import logger
+
+from mender_integration.testutils.api.client import ApiClient
+from mender_integration.testutils.api import deployments as deployments
+
+URL_DEPLOYMENTS_STATUS = "/deployments/{id}/status"
+
+
+class Server:
+    def __init__(self, auth_token, host="docker.mender.io"):
+        self.auth_token = auth_token
+        self.host = host
+        self.deployment_id = ""
+
+        self.api_dev_deploy = ApiClient(deployments.URL_MGMT, self.host)
+
+    def abort_deployment(self):
+        logger.info("Aborting deployment")
+        return self.api_dev_deploy.with_auth(self.auth_token).call(
+            "PUT",
+            URL_DEPLOYMENTS_STATUS.format(id=self.deployment_id),
+            body={"status": "aborted"},
+        )
+
+    def create_deployment(self, artifact_name, device_id, force=False):
+        logger.info("Creating deployment")
+        response = self.api_dev_deploy.with_auth(self.auth_token).call(
+            "POST",
+            deployments.URL_DEPLOYMENTS,
+            body={
+                "name": artifact_name,
+                "artifact_name": artifact_name,
+                "devices": [device_id],
+                "force_installation": force,
+            },
+        )
+        assert response.status_code == 201, f"{response.text} {response.status_code}"
+        self.deployment_id = os.path.basename(response.headers["Location"])
+
+    def upload_artifact(self, name, device_types):
+        with get_uncompressed_mender_artifact(
+            name, device_types=device_types, update_module="test-update"
+        ) as filename:
+
+            upload_image(filename, self.auth_token, self.api_dev_deploy)
+
+            deployment_req = {
+                "name": name,
+                "artifact_name": name,
+            }
+
+            self.api_dev_deploy.with_auth(self.auth_token).call(
+                "POST", deployments.URL_DEPLOYMENTS, deployment_req
+            )
+        return name
+
+
+def upload_image(filename, auth_token, api_client):
+    api_client.headers = {}
+    r = api_client.with_auth(auth_token).call(
+        "POST",
+        deployments.URL_DEPLOYMENTS_ARTIFACTS,
+        files=(
+            ("description", (None)),
+            ("size", (None, str(os.path.getsize(filename)))),
+            ("artifact", (filename, open(filename, "rb"), "application/octet-stream")),
+        ),
+    )