test: PoC integration test framework

Integration test framework for testing mender-mcu client with
native-sim.
Still requires work - e.g. making the demo server work, and probably
more work on the testing framework itself.
It uses  `integration` as a submodulel, so you need to init it.

You have to set tenant token, auth_token etc. manually, and
I've only tested it with 'hosted.mender.io'.

I went with the compile-in approach:
You can modify the callbacks directly in the test. You do this by
utilizing `set_callback()` from `helpers`. The available callbacks
you can modify are stored in `callbacks.py`. The callbacks have an #ifdef,
and the python script creates a header-file which is imported by the
test-update-module.

The test-update-module and the main-file for the tests are located in
the `src/` in the integration-tests directory.

Ticket: MEN-7599

Signed-off-by: Daniel Skinstad Drabitzius <daniel.drabitzius@northern.tech>

.gitmodules

@@ -0,0 +1,3 @@
+[submodule "tests/integration/mender_integration"]
+	path = tests/integration/mender_integration
+	url = https://github.com/mendersoftware/integration.git

CMakeLists.txt

@@ -4,11 +4,23 @@ find_package(Zephyr REQUIRED HINTS $ENV{ZEPHYR_BASE})
 
 project(mender-mcu-integration)
 
+if (INTEGRATION_TESTS MATCHES "y")
+target_sources(app PRIVATE
+    tests/integration/src/main.c
+    src/utils/netup.c
+    src/utils/certs.c
+)
+target_include_directories(app PUBLIC
+    tests/integration/src
+)
+target_sources(app PRIVATE tests/integration/src//modules/test-update-module.c)
+else()
 target_sources(app PRIVATE
     src/main.c
     src/utils/netup.c
     src/utils/certs.c
 )
+endif()
 
 target_include_directories(app PUBLIC
     src

tests/integration/conftest.py

@@ -0,0 +1,38 @@
+# Copyright 2024 Northern.tech AS
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+
+from os import path
+import sys
+
+sys.path += [path.join(path.dirname(__file__), "mender_integration")]
+
+import logging
+
+import pytest
+from mender_integration.tests.conftest import unique_test_name
+from mender_integration.tests.log import setup_test_logger
+
+logging.getLogger("requests").setLevel(logging.CRITICAL)
+
+logging.basicConfig()
+logging.getLogger().setLevel(logging.DEBUG)
+
+collect_ignore = ["mender_integration"]
+
+
+@pytest.fixture(scope="function", autouse=True)
+def testlogger(request):
+    test_name = unique_test_name(request)
+    setup_test_logger(test_name)
+    logging.getLogger().info("%s is starting.... " % test_name)

tests/integration/definitions.py

@@ -0,0 +1,45 @@
+# Copyright 2024 Northern.tech AS
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+
+################
+# Callbacks
+################
+
+# Callbacks defined in the update module
+UM_DOWNLOAD_CALLBACK = "UM_DOWNLOAD_CALLBACK"
+UM_INSTALL_CALLBACK = "UM_INSTALL_CALLBACK"
+UM_REBOOT_CALLBACK = "UM_REBOOT_CALLBACK"
+UM_VERIFY_REBOOT_CALLBACK = "UM_VERIFY_REBOOT_CALLBACK"
+UM_COMMIT_CALLBACK = "UM_COMMIT_CALLBACK"
+UM_ROLLBACK_CALLBACK = "UM_ROLLBACK_CALLBACK"
+UM_ROLLBACK_REBOOT_CALLBACK = "UM_ROLLBACK_REBOOT_CALLBACK"
+UM_FAILURE_CALLBACK = "UM_FAILURE_CALLBACK"
+
+
+# Callback defined in main
+NETWORK_CONNECT_CALLBACK = "NETWORK_CONNECT_CALLBACK"
+NETWORK_RELEASE_CALLBACK = "NETWORK_RELEASE_CALLBACK"
+DEPLOYMENT_STATUS_CALLBACK = "DEPLOYMENT_STATUS_CALLBACK"
+RESTART_CALLBACK = "RESTART_CALLBACK"
+GET_IDENTITY_CALLBACK = "GET_IDENTITY_CALLBACK"
+
+
+################
+# Variables
+################
+
+# Update module requires reboot
+UM_REQUIRES_REBOOT = "UM_REQUIRES_REBOOT"
+# Update module supports rollback
+UM_SUPPORTS_ROLLBACK = "UM_SUPPORTS_ROLLBACK"

tests/integration/device.py

@@ -0,0 +1,130 @@
+# Copyright 2024 Northern.tech AS
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+
+import os
+import time
+import pytest
+import shutil
+import tempfile
+import subprocess
+from mender_integration.tests.MenderAPI import logger
+
+from helpers import stdout
+from helpers import create_header_file
+
+THIS_DIR = os.path.dirname(os.path.abspath(__file__))
+
+# This has to point the west workspace containing mender-mcu-integration
+WORKSPACE_DIRECTORY = os.path.join(THIS_DIR, "../../")
+
+
+class DeviceStatus:
+    def __init__(self, device):
+        self.device = device
+
+    def is_authenticated(self, timeout=60):
+        logger.info("Waiting for device to authenticate")
+        start_time = time.time()
+        while time.time() - start_time < timeout:
+            line = stdout(self.device)
+            if "Authenticated successfully" in line:
+                logger.info("Device authenticated")
+                return True
+        return False
+
+    def is_aborted(self, timeout=60):
+        start_time = time.time()
+        while time.time() - start_time < timeout:
+            logger.info("Waiting for deployment to abort")
+            line = stdout(self.device)
+            if "Deployment aborted" in line:
+                return True
+        return False
+
+
+class NativeSim:
+    def __init__(self, stdout=False):
+        self.tenant_token = "..."
+        self.proc = None
+        self.stdout = stdout
+
+        self.server_host = ""
+        self.server_tenant = ""
+
+        self.status = DeviceStatus(self)
+
+        # Defaults to `https://docker.mender.io`
+        self.set_host()
+
+        create_header_file()
+
+        self.build_dir = tempfile.mkdtemp()
+
+    def set_host(self, host="https://docker.mender.io"):
+        self.server_host = host
+
+    def set_tenant(self, tenant):
+        self.server_tenant = tenant
+
+    def compile(self, pristine=False, extra_variables=None):
+        if extra_variables is None:
+            extra_variables = []
+
+        if compile:
+            variables = [
+                "-DCONFIG_LOG_ALWAYS_RUNTIME=y",
+                "-DCONFIG_LOG_MODE_IMMEDIATE=y",
+                "-DCONFIG_MENDER_LOG_LEVEL_OFF=n",
+                "-DCONFIG_MENDER_LOG_LEVEL_DBG=y",
+                f'-DINTEGRATION_TESTS="y"',
+                f'-DCONFIG_MENDER_SERVER_HOST="{self.server_host}"',
+                f'-DCONFIG_MENDER_SERVER_TENANT_TOKEN="{self.server_tenant}"',
+            ] + extra_variables
+            command = (
+                ["west", "build", "--board", "native_sim", WORKSPACE_DIRECTORY]
+                + variables
+                + ["--build-dir", f"{self.build_dir}"]
+                + (["--pristine"] if pristine else [])
+            )
+
+            try:
+                subprocess.check_call(command)
+            except subprocess.CalledProcessError as result:
+                logger.error(result.stderr)
+                command_output = " ".join(command)
+                pytest.fail(f"Failed to compile with command: {command_output}")
+
+    def start(self, compile=True, pristine=False, extra_variables=[]):
+        if compile:
+            self.compile(pristine=pristine, extra_variables=extra_variables)
+
+        self.proc = subprocess.Popen(
+            [
+                f"{self.build_dir}/zephyr/zephyr.exe",
+                f"--flash={self.build_dir}/flash.bin",
+            ],
+            stdout=subprocess.PIPE,
+            stderr=subprocess.PIPE,
+            text=True,
+        )
+        logger.info("Started device")
+
+    def stop(self, stop_after=0):
+        time.sleep(stop_after)
+        self.proc.terminate()
+        self.proc.wait()
+        logger.info("Stopped device")
+
+    def clean_build(self):
+        shutil.rmtree(self.build_dir)

tests/integration/helpers.py

@@ -0,0 +1,101 @@
+# Copyright 2024 Northern.tech AS
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+
+import os
+import random
+import string
+import tempfile
+import subprocess
+
+from os import path
+from contextlib import contextmanager
+
+from mender_integration.tests.MenderAPI import logger
+
+THIS_DIR = os.path.dirname(os.path.abspath(__file__))
+
+
+def get_header_file():
+    return path.join(THIS_DIR, "src/test_definitions.h")
+
+
+def create_header_file():
+    if not path.exists(get_header_file()):
+        open(get_header_file(), "w").close()
+
+
+# The name of the configurable callbaks can be found in definition.py
+def set_callback(function_name, definition):
+    with open(get_header_file(), "a") as f:
+        f.write(f"#define {function_name}() \\{definition}\n")
+
+
+# The name of the configurable variables can be found in definitions.py
+def set_define(define_name, definition):
+    with open(get_header_file(), "a") as f:
+        f.write(f"#define {define_name} {definition}\n")
+
+
+def stdout(device):
+    line = device.proc.stdout.readline()
+    if device.stdout:
+        logger.info(line)
+    return line
+
+
+# get_mender_artifact from testutils/common.py didn't support uncompressed artifacts
+@contextmanager
+def get_uncompressed_mender_artifact(
+    artifact_name="test",
+    update_module="dummy",
+    device_types=("arm1",),
+    size=256,
+    depends=(),
+    provides=(),
+):
+    data = "".join(random.choices(string.ascii_uppercase + string.digits, k=size))
+    f = tempfile.NamedTemporaryFile(delete=False)
+    f.write(data.encode("utf-8"))
+    f.close()
+    #
+    filename = f.name
+    artifact = "%s.mender" % filename
+    args = [
+        "mender-artifact",
+        "write",
+        "module-image",
+        "-o",
+        artifact,
+        "--artifact-name",
+        artifact_name,
+        "-T",
+        update_module,
+        "-f",
+        filename,
+        "--compression",
+        "none",
+    ]
+
+    for device_type in device_types:
+        args.extend(["-t", device_type])
+    for depend in depends:
+        args.extend(["--depends", depend])
+    for provide in provides:
+        args.extend(["--provides", provide])
+    try:
+        subprocess.call(args)
+        yield artifact
+    finally:
+        os.unlink(filename)
+        os.path.exists(artifact) and os.unlink(artifact)