From 5ac481ca1a9bac4c7ccfcbcf2be567133a05ee60 Mon Sep 17 00:00:00 2001 From: Daniel Skinstad Drabitzius Date: Mon, 21 Oct 2024 15:25:38 +0200 Subject: [PATCH 1/3] test: PoC integration test framework Integration test framework for testing mender-mcu client with native-sim. Still requires work - e.g. making the demo server work, and probably more work on the testing framework itself. It uses `integration` as a submodulel, so you need to init it. You have to set tenant token, auth_token etc. manually, and I've only tested it with 'hosted.mender.io'. I went with the compile-in approach: You can modify the callbacks directly in the test. You do this by utilizing `set_callback()` from `helpers`. The available callbacks you can modify are stored in `callbacks.py`. The callbacks have an #ifdef, and the python script creates a header-file which is imported by the test-update-module. The test-update-module and the main-file for the tests are located in the `src/` in the integration-tests directory. Ticket: MEN-7599 Signed-off-by: Daniel Skinstad Drabitzius --- .gitmodules | 3 + CMakeLists.txt | 12 ++ tests/integration/conftest.py | 38 ++++ tests/integration/definitions.py | 45 +++++ tests/integration/device.py | 130 ++++++++++++++ tests/integration/helpers.py | 101 +++++++++++ tests/integration/mender_integration | 1 + tests/integration/server.py | 86 +++++++++ tests/integration/src/main.c | 133 ++++++++++++++ .../src/modules/test-update-module.c | 166 ++++++++++++++++++ .../src/modules/test-update-module.h | 30 ++++ tests/integration/test_mender_mcu.py | 90 ++++++++++ 12 files changed, 835 insertions(+) create mode 100644 .gitmodules create mode 100644 tests/integration/conftest.py create mode 100644 tests/integration/definitions.py create mode 100644 tests/integration/device.py create mode 100644 tests/integration/helpers.py create mode 160000 tests/integration/mender_integration create mode 100644 tests/integration/server.py create mode 100644 tests/integration/src/main.c create mode 100644 tests/integration/src/modules/test-update-module.c create mode 100644 tests/integration/src/modules/test-update-module.h create mode 100644 tests/integration/test_mender_mcu.py diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 0000000..17eff08 --- /dev/null +++ b/.gitmodules @@ -0,0 +1,3 @@ +[submodule "tests/integration/mender_integration"] + path = tests/integration/mender_integration + url = https://github.com/mendersoftware/integration.git diff --git a/CMakeLists.txt b/CMakeLists.txt index 67f40af..f3a30a7 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -4,11 +4,23 @@ find_package(Zephyr REQUIRED HINTS $ENV{ZEPHYR_BASE}) project(mender-mcu-integration) +if (INTEGRATION_TESTS MATCHES "y") +target_sources(app PRIVATE + tests/integration/src/main.c + src/utils/netup.c + src/utils/certs.c +) +target_include_directories(app PUBLIC + tests/integration/src +) +target_sources(app PRIVATE tests/integration/src//modules/test-update-module.c) +else() target_sources(app PRIVATE src/main.c src/utils/netup.c src/utils/certs.c ) +endif() target_include_directories(app PUBLIC src diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py new file mode 100644 index 0000000..7d4a9e1 --- /dev/null +++ b/tests/integration/conftest.py @@ -0,0 +1,38 @@ +# Copyright 2024 Northern.tech AS +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from os import path +import sys + +sys.path += [path.join(path.dirname(__file__), "mender_integration")] + +import logging + +import pytest +from mender_integration.tests.conftest import unique_test_name +from mender_integration.tests.log import setup_test_logger + +logging.getLogger("requests").setLevel(logging.CRITICAL) + +logging.basicConfig() +logging.getLogger().setLevel(logging.DEBUG) + +collect_ignore = ["mender_integration"] + + +@pytest.fixture(scope="function", autouse=True) +def testlogger(request): + test_name = unique_test_name(request) + setup_test_logger(test_name) + logging.getLogger().info("%s is starting.... " % test_name) diff --git a/tests/integration/definitions.py b/tests/integration/definitions.py new file mode 100644 index 0000000..6c8ac5c --- /dev/null +++ b/tests/integration/definitions.py @@ -0,0 +1,45 @@ +# Copyright 2024 Northern.tech AS +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +################ +# Callbacks +################ + +# Callbacks defined in the update module +UM_DOWNLOAD_CALLBACK = "UM_DOWNLOAD_CALLBACK" +UM_INSTALL_CALLBACK = "UM_INSTALL_CALLBACK" +UM_REBOOT_CALLBACK = "UM_REBOOT_CALLBACK" +UM_VERIFY_REBOOT_CALLBACK = "UM_VERIFY_REBOOT_CALLBACK" +UM_COMMIT_CALLBACK = "UM_COMMIT_CALLBACK" +UM_ROLLBACK_CALLBACK = "UM_ROLLBACK_CALLBACK" +UM_ROLLBACK_REBOOT_CALLBACK = "UM_ROLLBACK_REBOOT_CALLBACK" +UM_FAILURE_CALLBACK = "UM_FAILURE_CALLBACK" + + +# Callback defined in main +NETWORK_CONNECT_CALLBACK = "NETWORK_CONNECT_CALLBACK" +NETWORK_RELEASE_CALLBACK = "NETWORK_RELEASE_CALLBACK" +DEPLOYMENT_STATUS_CALLBACK = "DEPLOYMENT_STATUS_CALLBACK" +RESTART_CALLBACK = "RESTART_CALLBACK" +GET_IDENTITY_CALLBACK = "GET_IDENTITY_CALLBACK" + + +################ +# Variables +################ + +# Update module requires reboot +UM_REQUIRES_REBOOT = "UM_REQUIRES_REBOOT" +# Update module supports rollback +UM_SUPPORTS_ROLLBACK = "UM_SUPPORTS_ROLLBACK" diff --git a/tests/integration/device.py b/tests/integration/device.py new file mode 100644 index 0000000..f9de865 --- /dev/null +++ b/tests/integration/device.py @@ -0,0 +1,130 @@ +# Copyright 2024 Northern.tech AS +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import time +import pytest +import shutil +import tempfile +import subprocess +from mender_integration.tests.MenderAPI import logger + +from helpers import stdout +from helpers import create_header_file + +THIS_DIR = os.path.dirname(os.path.abspath(__file__)) + +# This has to point the west workspace containing mender-mcu-integration +WORKSPACE_DIRECTORY = os.path.join(THIS_DIR, "../../") + + +class DeviceStatus: + def __init__(self, device): + self.device = device + + def is_authenticated(self, timeout=60): + logger.info("Waiting for device to authenticate") + start_time = time.time() + while time.time() - start_time < timeout: + line = stdout(self.device) + if "Authenticated successfully" in line: + logger.info("Device authenticated") + return True + return False + + def is_aborted(self, timeout=60): + start_time = time.time() + while time.time() - start_time < timeout: + logger.info("Waiting for deployment to abort") + line = stdout(self.device) + if "Deployment aborted" in line: + return True + return False + + +class NativeSim: + def __init__(self, stdout=False): + self.tenant_token = "..." + self.proc = None + self.stdout = stdout + + self.server_host = "" + self.server_tenant = "" + + self.status = DeviceStatus(self) + + # Defaults to `https://docker.mender.io` + self.set_host() + + create_header_file() + + self.build_dir = tempfile.mkdtemp() + + def set_host(self, host="https://docker.mender.io"): + self.server_host = host + + def set_tenant(self, tenant): + self.server_tenant = tenant + + def compile(self, pristine=False, extra_variables=None): + if extra_variables is None: + extra_variables = [] + + if compile: + variables = [ + "-DCONFIG_LOG_ALWAYS_RUNTIME=y", + "-DCONFIG_LOG_MODE_IMMEDIATE=y", + "-DCONFIG_MENDER_LOG_LEVEL_OFF=n", + "-DCONFIG_MENDER_LOG_LEVEL_DBG=y", + f'-DINTEGRATION_TESTS="y"', + f'-DCONFIG_MENDER_SERVER_HOST="{self.server_host}"', + f'-DCONFIG_MENDER_SERVER_TENANT_TOKEN="{self.server_tenant}"', + ] + extra_variables + command = ( + ["west", "build", "--board", "native_sim", WORKSPACE_DIRECTORY] + + variables + + ["--build-dir", f"{self.build_dir}"] + + (["--pristine"] if pristine else []) + ) + + try: + subprocess.check_call(command) + except subprocess.CalledProcessError as result: + logger.error(result.stderr) + command_output = " ".join(command) + pytest.fail(f"Failed to compile with command: {command_output}") + + def start(self, compile=True, pristine=False, extra_variables=[]): + if compile: + self.compile(pristine=pristine, extra_variables=extra_variables) + + self.proc = subprocess.Popen( + [ + f"{self.build_dir}/zephyr/zephyr.exe", + f"--flash={self.build_dir}/flash.bin", + ], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) + logger.info("Started device") + + def stop(self, stop_after=0): + time.sleep(stop_after) + self.proc.terminate() + self.proc.wait() + logger.info("Stopped device") + + def clean_build(self): + shutil.rmtree(self.build_dir) diff --git a/tests/integration/helpers.py b/tests/integration/helpers.py new file mode 100644 index 0000000..5dbc46a --- /dev/null +++ b/tests/integration/helpers.py @@ -0,0 +1,101 @@ +# Copyright 2024 Northern.tech AS +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import random +import string +import tempfile +import subprocess + +from os import path +from contextlib import contextmanager + +from mender_integration.tests.MenderAPI import logger + +THIS_DIR = os.path.dirname(os.path.abspath(__file__)) + + +def get_header_file(): + return path.join(THIS_DIR, "src/test_definitions.h") + + +def create_header_file(): + if not path.exists(get_header_file()): + open(get_header_file(), "w").close() + + +# The name of the configurable callbaks can be found in definition.py +def set_callback(function_name, definition): + with open(get_header_file(), "a") as f: + f.write(f"#define {function_name}() \\{definition}\n") + + +# The name of the configurable variables can be found in definitions.py +def set_define(define_name, definition): + with open(get_header_file(), "a") as f: + f.write(f"#define {define_name} {definition}\n") + + +def stdout(device): + line = device.proc.stdout.readline() + if device.stdout: + logger.info(line) + return line + + +# get_mender_artifact from testutils/common.py didn't support uncompressed artifacts +@contextmanager +def get_uncompressed_mender_artifact( + artifact_name="test", + update_module="dummy", + device_types=("arm1",), + size=256, + depends=(), + provides=(), +): + data = "".join(random.choices(string.ascii_uppercase + string.digits, k=size)) + f = tempfile.NamedTemporaryFile(delete=False) + f.write(data.encode("utf-8")) + f.close() + # + filename = f.name + artifact = "%s.mender" % filename + args = [ + "mender-artifact", + "write", + "module-image", + "-o", + artifact, + "--artifact-name", + artifact_name, + "-T", + update_module, + "-f", + filename, + "--compression", + "none", + ] + + for device_type in device_types: + args.extend(["-t", device_type]) + for depend in depends: + args.extend(["--depends", depend]) + for provide in provides: + args.extend(["--provides", provide]) + try: + subprocess.call(args) + yield artifact + finally: + os.unlink(filename) + os.path.exists(artifact) and os.unlink(artifact) diff --git a/tests/integration/mender_integration b/tests/integration/mender_integration new file mode 160000 index 0000000..00a6f79 --- /dev/null +++ b/tests/integration/mender_integration @@ -0,0 +1 @@ +Subproject commit 00a6f798d4baccd2456ce4ee56db0acd6427c3f5 diff --git a/tests/integration/server.py b/tests/integration/server.py new file mode 100644 index 0000000..296f6b7 --- /dev/null +++ b/tests/integration/server.py @@ -0,0 +1,86 @@ +# Copyright 2024 Northern.tech AS +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + +from helpers import get_uncompressed_mender_artifact + +from mender_integration.tests.MenderAPI import logger + +from mender_integration.testutils.api.client import ApiClient +from mender_integration.testutils.api import deployments as deployments + +URL_DEPLOYMENTS_STATUS = "/deployments/{id}/status" + + +class Server: + def __init__(self, auth_token, host="docker.mender.io"): + self.auth_token = auth_token + self.host = host + self.deployment_id = "" + + self.api_dev_deploy = ApiClient(deployments.URL_MGMT, self.host) + + def abort_deployment(self): + logger.info("Aborting deployment") + return self.api_dev_deploy.with_auth(self.auth_token).call( + "PUT", + URL_DEPLOYMENTS_STATUS.format(id=self.deployment_id), + body={"status": "aborted"}, + ) + + def create_deployment(self, artifact_name, device_id, force=False): + logger.info("Creating deployment") + response = self.api_dev_deploy.with_auth(self.auth_token).call( + "POST", + deployments.URL_DEPLOYMENTS, + body={ + "name": artifact_name, + "artifact_name": artifact_name, + "devices": [device_id], + "force_installation": force, + }, + ) + assert response.status_code == 201, f"{response.text} {response.status_code}" + self.deployment_id = os.path.basename(response.headers["Location"]) + + def upload_artifact(self, name, device_types): + with get_uncompressed_mender_artifact( + name, device_types=device_types, update_module="test-update" + ) as filename: + + upload_image(filename, self.auth_token, self.api_dev_deploy) + + deployment_req = { + "name": name, + "artifact_name": name, + } + + self.api_dev_deploy.with_auth(self.auth_token).call( + "POST", deployments.URL_DEPLOYMENTS, deployment_req + ) + return name + + +def upload_image(filename, auth_token, api_client): + api_client.headers = {} + r = api_client.with_auth(auth_token).call( + "POST", + deployments.URL_DEPLOYMENTS_ARTIFACTS, + files=( + ("description", (None)), + ("size", (None, str(os.path.getsize(filename)))), + ("artifact", (filename, open(filename, "rb"), "application/octet-stream")), + ), + ) diff --git a/tests/integration/src/main.c b/tests/integration/src/main.c new file mode 100644 index 0000000..41710ee --- /dev/null +++ b/tests/integration/src/main.c @@ -0,0 +1,133 @@ +// Copyright 2024 Northern.tech AS +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include +LOG_MODULE_REGISTER(mender_app, LOG_LEVEL_DBG); + +#include "utils/netup.h" +#include "utils/certs.h" + +#include +#include + +#include "mender-client.h" +#include "mender-inventory.h" +#include "mender-flash.h" + +#include "modules/test-update-module.h" + +#include "test_definitions.h" + +static mender_err_t +network_connect_cb(void) { + LOG_DBG("network_connect_cb"); +#ifdef NETWORK_CONNECT_CALLBACK + NETWORK_CONNECT_CALLBACK(); +#endif + return MENDER_OK; +} + +static mender_err_t +network_release_cb(void) { + LOG_DBG("network_release_cb"); +#ifdef NETWORK_RELEASE_CALLBACK + NETWORK_RELEASE_CALLBACK(); +#endif + return MENDER_OK; +} + +static mender_err_t +deployment_status_cb(mender_deployment_status_t status, char *desc) { + LOG_DBG("deployment_status_cb: %s", desc); +#ifdef DEPLOYMENT_STATUS_CALLBACK + DEPLOYMENT_STATUS_CALLBACK(); +#endif + return MENDER_OK; +} + +static mender_err_t +restart_cb(void) { + LOG_DBG("restart_cb"); +#ifdef RESTART_CALLBACK + RESTART_CALLBACK(); +#else + sys_reboot(SYS_REBOOT_WARM); +#endif + + return MENDER_OK; +} + +static char mac_address[18] = { 0 }; +static mender_identity_t mender_identity = { .name = "mac", .value = mac_address }; + +static mender_err_t +get_identity_cb(mender_identity_t **identity) { + LOG_DBG("get_identity_cb"); +#ifdef GET_IDENTITY_CALLBACK + GET_IDENTITY_CALLBACK(); +#else + if (NULL != identity) { + *identity = &mender_identity; + return MENDER_OK; + } +#endif + return MENDER_FAIL; +} + +int +main(void) { + printf("Hello World! %s\n", CONFIG_BOARD_TARGET); + + netup_wait_for_network(); + + netup_get_mac_address(mender_identity.value); + + certs_add_credentials(); + + LOG_INF("Initializing Mender Client with:"); + LOG_INF(" Device type: '%s'", CONFIG_MENDER_DEVICE_TYPE); + LOG_INF(" Identity: '{\"%s\": \"%s\"}'", mender_identity.name, mender_identity.value); + + /* Initialize mender-client */ + mender_client_config_t mender_client_config = { .device_type = NULL, .recommissioning = false }; + mender_client_callbacks_t mender_client_callbacks = { .network_connect = network_connect_cb, + .network_release = network_release_cb, + .deployment_status = deployment_status_cb, + .restart = restart_cb, + .get_identity = get_identity_cb, + .get_user_provided_keys = NULL }; + + assert(MENDER_OK == mender_client_init(&mender_client_config, &mender_client_callbacks)); + LOG_INF("Mender client initialized"); + + assert(MENDER_OK == test_update_module_register()); + LOG_INF("Update Module 'test-update' initialized"); + +#ifdef CONFIG_MENDER_CLIENT_ADD_ON_INVENTORY + mender_keystore_t inventory[] = { { .name = "demo", .value = "demo" }, { .name = "foo", .value = "bar" }, { .name = NULL, .value = NULL } }; + assert(MENDER_OK == mender_inventory_set(inventory)); + LOG_INF("Mender inventory set"); +#endif /* CONFIG_MENDER_CLIENT_ADD_ON_INVENTORY */ + + /* Finally activate mender client */ + if (MENDER_OK != mender_client_activate()) { + LOG_ERR("Unable to activate mender-client"); + } else { + LOG_INF("Mender client activated and running!"); + } + + k_sleep(K_FOREVER); + + return 0; +} diff --git a/tests/integration/src/modules/test-update-module.c b/tests/integration/src/modules/test-update-module.c new file mode 100644 index 0000000..2e8b8ea --- /dev/null +++ b/tests/integration/src/modules/test-update-module.c @@ -0,0 +1,166 @@ +// Copyright 2024 Northern.tech AS +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "mender-client.h" +#include "mender-log.h" +#include "mender-utils.h" +#include "mender-update-module.h" +#include +#include + +#include "zephyr/kernel.h" + +#include "mender-utils.h" + +#include + +#include "test_definitions.h" + +static mender_err_t test_update_module_download(mender_update_state_t state, mender_update_state_data_t callback_data); + +static mender_err_t test_update_module_install(mender_update_state_t state, mender_update_state_data_t callback_data); + +static mender_err_t test_update_module_reboot(mender_update_state_t state, mender_update_state_data_t callback_data); + +static mender_err_t test_update_module_verify_reboot(mender_update_state_t state, mender_update_state_data_t callback_data); + +static mender_err_t test_update_module_commit(mender_update_state_t state, mender_update_state_data_t callback_data); + +static mender_err_t test_update_module_rollback(mender_update_state_t state, mender_update_state_data_t callback_data); + +static mender_err_t test_update_module_failure(mender_update_state_t state, mender_update_state_data_t callback_data); + +static mender_err_t test_update_module_rollback_reboot(mender_update_state_t state, mender_update_state_data_t callback_data); + +#ifndef UM_REQUIRES_REBOOT +#define UM_REQUIRES_REBOOT false +#endif +#ifndef UM_SUPPORTS_ROLLBACK +#define UM_SUPPORTS_ROLLBACK false +#endif + +mender_err_t +test_update_module_register(void) { + mender_err_t ret; + mender_update_module_t *test_update_module; + + /* Register the zephyr-image update module */ + if (NULL == (test_update_module = calloc(1, sizeof(mender_update_module_t)))) { + mender_log_error("Unable to allocate memory for the 'test-update' update module"); + return MENDER_FAIL; + } + test_update_module->callbacks[MENDER_UPDATE_STATE_DOWNLOAD] = &test_update_module_download; + test_update_module->callbacks[MENDER_UPDATE_STATE_INSTALL] = &test_update_module_install; + test_update_module->callbacks[MENDER_UPDATE_STATE_REBOOT] = &test_update_module_reboot; + test_update_module->callbacks[MENDER_UPDATE_STATE_VERIFY_REBOOT] = &test_update_module_verify_reboot; + test_update_module->callbacks[MENDER_UPDATE_STATE_COMMIT] = &test_update_module_commit; + test_update_module->callbacks[MENDER_UPDATE_STATE_ROLLBACK] = &test_update_module_rollback; + test_update_module->callbacks[MENDER_UPDATE_STATE_FAILURE] = &test_update_module_failure; + test_update_module->callbacks[MENDER_UPDATE_STATE_ROLLBACK_REBOOT] = &test_update_module_rollback_reboot; + test_update_module->artifact_type = "test-update"; + test_update_module->requires_reboot = UM_REQUIRES_REBOOT; + test_update_module->supports_rollback = UM_SUPPORTS_ROLLBACK; + + if (MENDER_OK != (ret = mender_client_register_update_module(test_update_module))) { + mender_log_error("Unable to register the 'test-update' update module"); + /* mender_client_register_update_module() takes ownership if it succeeds */ + free(test_update_module); + return ret; + } + + return MENDER_OK; +} + +static mender_err_t +test_update_module_download(MENDER_NDEBUG_UNUSED mender_update_state_t state, MENDER_ARG_UNUSED mender_update_state_data_t callback_data) { + + /* Macro defined by test */ +#ifdef UM_DOWNLOAD_CALLBACK + UM_DOWNLOAD_CALLBACK(); +#endif + + return MENDER_OK; +} + +static mender_err_t +test_update_module_install(MENDER_NDEBUG_UNUSED mender_update_state_t state, MENDER_ARG_UNUSED mender_update_state_data_t callback_data) { + + /* Macro defined by test */ +#ifdef UM_INSTALL_CALLBACK + UM_INSTALL_CALLBACK(); +#endif + return MENDER_OK; +} + +static mender_err_t test_update_module_reboot(mender_update_state_t state, mender_update_state_data_t callback_data) { + + /* Macro defined by test */ +#ifdef UM_REBOOT_CALLBACK + UM_REBOOT_CALLBACK(); +#endif + return MENDER_OK; +} + +static mender_err_t test_update_module_verify_reboot(mender_update_state_t state, mender_update_state_data_t callback_data) { + + /* Macro defined by test */ +#ifdef UM_VERIFY_REBOOT_CALLBACK + UM_VERIFY_REBOOT_CALLBACK(); +#endif + + return MENDER_OK; +} + +static mender_err_t +test_update_module_commit(MENDER_NDEBUG_UNUSED mender_update_state_t state, MENDER_ARG_UNUSED mender_update_state_data_t callback_data) { + + /* Macro defined by test */ +#ifdef UM_COMMIT_CALLBACK + UM_COMMIT_CALLBACK(); +#endif + + return MENDER_OK; +} + + +static mender_err_t test_update_module_rollback(mender_update_state_t state, mender_update_state_data_t callback_data) { + + /* Macro defined by test */ +#ifdef UM_ROLLBACK_CALLBACK + UM_ROLLBACK_CALLBACK(); +#endif + + return MENDER_OK; +} + +static mender_err_t +test_update_module_failure(MENDER_NDEBUG_UNUSED mender_update_state_t state, MENDER_ARG_UNUSED mender_update_state_data_t callback_data) { + + /* Macro defined by test */ +#ifdef UM_FAILURE_CALLBACK + UM_FAILURE_CALLBACK(); +#endif + + return MENDER_OK; +} + + +static mender_err_t test_update_module_rollback_reboot(mender_update_state_t state, mender_update_state_data_t callback_data) { + + /* Macro defined by test */ +#ifdef UM_ROLLBACK_REBOOT_CALLBACK + UM_ROLLBACK_REBOOT_CALLBACK(); +#endif + return MENDER_OK; +} diff --git a/tests/integration/src/modules/test-update-module.h b/tests/integration/src/modules/test-update-module.h new file mode 100644 index 0000000..2c181ca --- /dev/null +++ b/tests/integration/src/modules/test-update-module.h @@ -0,0 +1,30 @@ +// Copyright 2024 Northern.tech AS +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef __NOOP_UPDATE_MODULE_H__ +#define __NOOP_UPDATE_MODULE_H__ + +#ifdef __cplusplus +extern "C" { +#endif /* __cplusplus */ + +#include "mender-utils.h" + +mender_err_t test_update_module_register(void); + +#ifdef __cplusplus +} +#endif /* __cplusplus */ + +#endif /* __NOOP_UPDATE_MODULE_H__ */ diff --git a/tests/integration/test_mender_mcu.py b/tests/integration/test_mender_mcu.py new file mode 100644 index 0000000..d23abd0 --- /dev/null +++ b/tests/integration/test_mender_mcu.py @@ -0,0 +1,90 @@ +# Copyright 2024 Northern.tech AS +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import time +import pytest +import helpers + +from mender_integration.tests.MenderAPI import logger + +from server import Server + +from helpers import stdout +from device import NativeSim + +import definitions + + +@pytest.fixture +def teardown(): + yield + os.remove(helpers.get_header_file()) + + +def test_deployment_abort(teardown): + + """ + Sample test to demonstrate use of framework + """ + + # Use hosted.mender.io for now, as there are problems with + # the demo server + # Temporary workaround for the PoC + auth_token = os.getenv("TEST_AUTH_TOKEN") + server = Server(auth_token=auth_token, host="hosted.mender.io") + + # Temporary workaround for the PoC + device_id = os.getenv("TEST_DEVICE_ID") + artifact_name = server.upload_artifact( + "test-artifact", device_types=("native_sim/native",) + ) + + download_body = r""" + static int counter = 0; \ + if (counter == 0) { \ + printf("Sleeping in download\n"); \ + k_sleep(K_SECONDS(5)); \ + }\ + counter++; \ + """ + helpers.set_callback(definitions.UM_DOWNLOAD_CALLBACK, download_body) + + device = NativeSim(stdout=True) + # Set host and tenant in the device + device.set_host("https://hosted.mender.io") + + # Temporary workaround for the PoC + device.set_tenant(os.getenv("TEST_TENANT_TOKEN")) + + # Start device + device.start(pristine=True) + device.status.is_authenticated(timeout=60) + + # Create deployment + server.create_deployment(artifact_name, device_id, True) + + # Wait for download callback to sleep so we can abort + timeout = 60 + start_time = time.time() + while time.time() - start_time < timeout: + line = stdout(device) + if "Sleeping in download\n" in line: + server.abort_deployment() + break + + if not device.status.is_aborted(timeout=60): + pytest.fail("Deployment did not abort") + + logger.info("Deployment aborted") From 6798002b002a17bf6d6d9dc7b00c257348b0e9a2 Mon Sep 17 00:00:00 2001 From: Daniel Skinstad Drabitzius Date: Fri, 15 Nov 2024 09:43:47 +0100 Subject: [PATCH 2/3] test: use ApiClient from mender-server instead And some small other adjustments, like * setting up the server as a fixture etc., * allow you to pass e.g. `hosted.mender.io` as a flag when running pytest We only use some functions from integration now, but that is related to logging Ticket: MEN-7675 Signed-off-by: Daniel Skinstad Drabitzius --- .gitmodules | 3 +++ tests/integration/conftest.py | 34 +++++++++++++++++++++++++--- tests/integration/server.py | 6 ++--- tests/integration/test_mender_mcu.py | 10 +------- 4 files changed, 37 insertions(+), 16 deletions(-) diff --git a/.gitmodules b/.gitmodules index 17eff08..905c2ec 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,3 +1,6 @@ [submodule "tests/integration/mender_integration"] path = tests/integration/mender_integration url = https://github.com/mendersoftware/integration.git +[submodule "tests/integration/mender_server"] + path = tests/integration/mender_server + url = https://github.com/mendersoftware/mender-server.git diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index 7d4a9e1..ef3f137 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -12,14 +12,20 @@ # See the License for the specific language governing permissions and # limitations under the License. +import os from os import path + import sys +import logging +import pytest sys.path += [path.join(path.dirname(__file__), "mender_integration")] +sys.path += [ + path.join(path.dirname(__file__), "mender_server/backend/tests/integration/tests/") +] -import logging +from server import Server -import pytest from mender_integration.tests.conftest import unique_test_name from mender_integration.tests.log import setup_test_logger @@ -28,7 +34,7 @@ logging.basicConfig() logging.getLogger().setLevel(logging.DEBUG) -collect_ignore = ["mender_integration"] +collect_ignore = ["mender_integration", "mender_server"] @pytest.fixture(scope="function", autouse=True) @@ -36,3 +42,25 @@ def testlogger(request): test_name = unique_test_name(request) setup_test_logger(test_name) logging.getLogger().info("%s is starting.... " % test_name) + + +def pytest_addoption(parser): + parser.addoption( + "--host", + action="store", + default="docker.mender.io", + help="Server URL for tests", + ) + + +@pytest.fixture(scope="session", autouse=True) +def setup_user(): + # for now this just returns auth token from env + # but for demo server, it should create a user aswell + return os.getenv("TEST_AUTH_TOKEN") + + +@pytest.fixture(scope="session", autouse=True) +def server(setup_user, request): + host_url = request.config.getoption("--host") + return Server(auth_token=setup_user, host=host_url) diff --git a/tests/integration/server.py b/tests/integration/server.py index 296f6b7..476000c 100644 --- a/tests/integration/server.py +++ b/tests/integration/server.py @@ -15,13 +15,11 @@ import os from helpers import get_uncompressed_mender_artifact - from mender_integration.tests.MenderAPI import logger -from mender_integration.testutils.api.client import ApiClient -from mender_integration.testutils.api import deployments as deployments - URL_DEPLOYMENTS_STATUS = "/deployments/{id}/status" +from mender_server.backend.tests.integration.tests.testutils.api.client import ApiClient +from mender_server.backend.tests.integration.tests.testutils.api import deployments class Server: diff --git a/tests/integration/test_mender_mcu.py b/tests/integration/test_mender_mcu.py index d23abd0..b48ffed 100644 --- a/tests/integration/test_mender_mcu.py +++ b/tests/integration/test_mender_mcu.py @@ -19,8 +19,6 @@ from mender_integration.tests.MenderAPI import logger -from server import Server - from helpers import stdout from device import NativeSim @@ -33,18 +31,12 @@ def teardown(): os.remove(helpers.get_header_file()) -def test_deployment_abort(teardown): +def test_deployment_abort(teardown, server): """ Sample test to demonstrate use of framework """ - # Use hosted.mender.io for now, as there are problems with - # the demo server - # Temporary workaround for the PoC - auth_token = os.getenv("TEST_AUTH_TOKEN") - server = Server(auth_token=auth_token, host="hosted.mender.io") - # Temporary workaround for the PoC device_id = os.getenv("TEST_DEVICE_ID") artifact_name = server.upload_artifact( From ac6635241388c5a333d38992fafa6eea46aa8228 Mon Sep 17 00:00:00 2001 From: Daniel Skinstad Drabitzius Date: Fri, 29 Nov 2024 13:07:54 +0100 Subject: [PATCH 3/3] test: Readme for integration tests locally Signed-off-by: Daniel Skinstad Drabitzius --- tests/integration/README.md | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) create mode 100644 tests/integration/README.md diff --git a/tests/integration/README.md b/tests/integration/README.md new file mode 100644 index 0000000..84d5fb1 --- /dev/null +++ b/tests/integration/README.md @@ -0,0 +1,23 @@ +## Running the integration tests locally + +###### NOTE: This does _not_ work with mender-mcu main due to restructuring of CMakeLists etc. +The latest commit this is tested with is `208190909d47d48a5bcf33ac0aa606dc05af4182` (mender-mcu) +There is a ticket to review the `C parts`, so this should be rebased on top of main there. + +Pre-requisites: + - networking for the native-sim board (see README) + - pytest + +To run the current tests, you will need to export + - TEST_DEVICE_ID + - TEST_TENANT_TOKEN + - TEST_AUTH_TOKEN + +as these are used to connect to hosted Mender. + +You will also need to run `git submodule update --init --recursive` to get +the parts from mender-server (there are also parts from mender-integration, but +as of now this is just some small logging parts etc., we should remove the dependency on +the integration repo). + +Run the tests with `pytest -s --host hosted.mender.io` in the `tests/integration` directory.