All integrations tests should run in openstack (#413)

.github/workflows/integration_test.yaml

@@ -16,14 +16,13 @@ jobs:
     secrets: inherit
     with:
       juju-channel: 3.1/stable
-      pre-run-script: scripts/pre-integration-test.sh
+      pre-run-script: scripts/setup-lxd.sh
       provider: lxd
       test-tox-env: integration-juju3.1
-      # These important local LXD test have no OpenStack integration versions.
-      # test_charm_scheduled_events ensures reconcile events are fired on a schedule.
-      # test_debug_ssh ensures tmate SSH actions works.
-      # The test test_charm_upgrade needs to run to ensure the charm can be upgraded.
       modules: '["test_charm_scheduled_events", "test_debug_ssh", "test_charm_upgrade"]'
+      extra-arguments: "-m openstack"
+      self-hosted-runner: true
+      self-hosted-runner-label: stg-private-endpoint
   openstack-interface-tests-private-endpoint:
     name: openstack interface test using private-endpoint
     uses: canonical/operator-workflows/.github/workflows/integration_test.yaml@main
@@ -39,7 +38,6 @@ jobs:
   openstack-integration-tests-private-endpoint:
     name: Integration test using private-endpoint
     uses: canonical/operator-workflows/.github/workflows/integration_test.yaml@main
-    needs: openstack-interface-tests-private-endpoint
     secrets: inherit
     with:
       juju-channel: 3.6/stable

tests/integration/conftest.py

@@ -440,7 +440,7 @@ async def app_openstack_runner_fixture(
             reconcile_interval=60,
             constraints={
                 "root-disk": 50 * 1024,
-                "mem": 16 * 1024,
+                "mem": 2 * 1024,
             },
             config={
                 OPENSTACK_CLOUDS_YAML_CONFIG_NAME: clouds_yaml_contents,
@@ -470,43 +470,28 @@ async def app_one_runner(model: Model, app_no_runner: Application) -> AsyncItera
     return app_no_runner
 
 
-@pytest_asyncio.fixture(scope="module")
-async def app_scheduled_events(
+@pytest_asyncio.fixture(scope="module", name="app_scheduled_events")
+async def app_scheduled_events_fixture(
     model: Model,
-    charm_file: str,
-    app_name: str,
-    path: str,
-    token: str,
-    http_proxy: str,
-    https_proxy: str,
-    no_proxy: str,
-) -> AsyncIterator[Application]:
-    """Application with no token.
-
-    Test should ensure it returns with the application having one runner.
-
-    This fixture has to deploy a new application. The scheduled events are set
-    to one hour in other application to avoid conflicting with the tests.
-    Changes to the duration of scheduled interval only takes effect after the
-    next trigger. Therefore, it would take a hour for the duration change to
-    take effect.
-    """
-    application = await deploy_github_runner_charm(
-        model=model,
-        charm_file=charm_file,
-        app_name=app_name,
-        path=path,
-        token=token,
-        runner_storage="memory",
-        http_proxy=http_proxy,
-        https_proxy=https_proxy,
-        no_proxy=no_proxy,
-        reconcile_interval=8,
-    )
-
+    app_openstack_runner,
+):
+    """Application to check scheduled events."""
+    application = app_openstack_runner
+    await application.set_config({"reconcile-interval": "8"})
     await application.set_config({VIRTUAL_MACHINES_CONFIG_NAME: "1"})
+    await model.wait_for_idle(apps=[application.name], status=ACTIVE, timeout=90 * 60)
     await reconcile(app=application, model=model)
+    return application
 
+
+@pytest_asyncio.fixture(scope="module", name="app_no_wait_tmate")
+async def app_no_wait_tmate_fixture(
+    model: Model,
+    app_openstack_runner,
+):
+    """Application to check tmate ssh with openstack without waiting for active."""
+    application = app_openstack_runner
+    await application.set_config({"reconcile-interval": "60", VIRTUAL_MACHINES_CONFIG_NAME: "1"})
     return application
 
 
@@ -569,11 +554,11 @@ async def app_no_wait_fixture(
 
 @pytest_asyncio.fixture(scope="module", name="tmate_ssh_server_app")
 async def tmate_ssh_server_app_fixture(
-    model: Model, app_no_wait: Application
+    model: Model, app_no_wait_tmate: Application
 ) -> AsyncIterator[Application]:
     """tmate-ssh-server charm application related to GitHub-Runner app charm."""
     tmate_app: Application = await model.deploy("tmate-ssh-server", channel="edge")
-    await app_no_wait.relate("debug-ssh", f"{tmate_app.name}:debug-ssh")
+    await app_no_wait_tmate.relate("debug-ssh", f"{tmate_app.name}:debug-ssh")
     await model.wait_for_idle(apps=[tmate_app.name], status=ACTIVE, timeout=60 * 30)
 
     return tmate_app

tests/integration/helpers/common.py

@@ -55,14 +55,39 @@ class InstanceHelper(typing.Protocol):
     """Helper for running commands in instances."""
 
     async def run_in_instance(
-        self, unit: Unit, command: str, timeout: int | None = None
+        self,
+        unit: Unit,
+        command: str,
+        timeout: int | None = None,
+        assert_on_failure: bool = False,
+        assert_msg: str | None = None,
     ) -> tuple[int, str | None, str | None]:
         """Run command in instance.
 
         Args:
             unit: Juju unit to execute the command in.
             command: Command to execute.
             timeout: Amount of time to wait for the execution.
+            assert_on_failure: Perform assertion on non-zero exit code.
+            assert_msg: Message for the failure assertion.
+        """
+        ...
+
+    async def expose_to_instance(
+        self,
+        unit: Unit,
+        port: int,
+        host: str = "localhost",
+    ) -> None:
+        """Expose a port on the juju machine to the OpenStack instance.
+
+        Uses SSH remote port forwarding from the juju machine to the OpenStack instance containing
+        the runner.
+
+        Args:
+            unit: The juju unit of the github-runner charm.
+            port: The port on the juju machine to expose to the runner.
+            host: Host for the reverse tunnel.
         """
         ...
 
@@ -74,6 +99,14 @@ async def ensure_charm_has_runner(self, app: Application):
         """
         ...
 
+    async def get_runner_names(self, unit: Unit) -> list[str]:
+        """Get the name of all the runners in the unit.
+
+        Args:
+            unit: The GitHub Runner Charm unit to get the runner names for.
+        """
+        ...
+
     async def get_runner_name(self, unit: Unit) -> str:
         """Get the name of the runner.
 
@@ -82,6 +115,14 @@ async def get_runner_name(self, unit: Unit) -> str:
         """
         ...
 
+    async def delete_single_runner(self, unit: Unit) -> None:
+        """Delete the only runner.
+
+        Args:
+            unit: The GitHub Runner Charm unit to delete the runner name for.
+        """
+        ...
+
 
 async def check_runner_binary_exists(unit: Unit) -> bool:
     """Checks if runner binary exists in the charm.

tests/integration/helpers/lxd.py

@@ -20,21 +20,49 @@ class LXDInstanceHelper(InstanceHelper):
     """Helper class to interact with LXD instances."""
 
     async def run_in_instance(
-        self, unit: Unit, command: str, timeout: int | None = None
+        self,
+        unit: Unit,
+        command: str,
+        timeout: int | None = None,
+        assert_on_failure: bool = False,
+        assert_msg: str | None = None,
     ) -> tuple[int, str | None, str | None]:
         """Run command in LXD instance.
 
         Args:
             unit: Juju unit to execute the command in.
             command: Command to execute.
             timeout: Amount of time to wait for the execution.
+            assert_on_failure: Not used in lxd
+            assert_msg: Not used in lxd
 
         Returns:
             Tuple of return code, stdout and stderr.
         """
         name = await self.get_runner_name(unit)
         return await run_in_lxd_instance(unit, name, command, timeout=timeout)
 
+    async def expose_to_instance(
+        self,
+        unit: Unit,
+        port: int,
+        host: str = "localhost",
+    ) -> None:
+        """Expose a port on the juju machine to the OpenStack instance.
+
+        Uses SSH remote port forwarding from the juju machine to the OpenStack instance containing
+        the runner.
+
+        Args:
+            unit: The juju unit of the github-runner charm.
+            port: The port on the juju machine to expose to the runner.
+            host: Host for the reverse tunnel.
+
+        Raises:
+            NotImplementedError: Not implemented yet.
+        """
+        raise NotImplementedError
+
     async def ensure_charm_has_runner(self, app: Application):
         """Reconcile the charm to contain one runner.
 
@@ -56,6 +84,29 @@ async def get_runner_name(self, unit: Unit) -> str:
         """
         return await get_runner_name(unit)
 
+    async def get_runner_names(self, unit: Unit) -> list[str]:
+        """Get the name of all the runners in the unit.
+
+        Args:
+            unit: The GitHub Runner Charm unit to get the runner names for.
+
+        Raises:
+            NotImplementedError: Not implemented yet.
+        """
+        raise NotImplementedError
+
+    async def delete_single_runner(self, unit: Unit) -> None:
+        """Delete the only runner.
+
+
+        Args:
+            unit: The GitHub Runner Charm unit to check.
+
+        Raises:
+            NotImplementedError: Not implemented yet.
+        """
+        raise NotImplementedError
+
 
 async def assert_resource_lxd_profile(unit: Unit, configs: dict[str, Any]) -> None:
     """Check for LXD profile of the matching resource config.

tests/integration/helpers/openstack.py

@@ -3,7 +3,7 @@
 import logging
 import secrets
 from asyncio import sleep
-from typing import Optional, TypedDict, cast
+from typing import Optional, TypedDict
 
 import openstack.connection
 from juju.application import Application
@@ -32,6 +32,7 @@ async def expose_to_instance(
         self,
         unit: Unit,
         port: int,
+        host: str = "localhost",
     ) -> None:
         """Expose a port on the juju machine to the OpenStack instance.
 
@@ -41,6 +42,7 @@ async def expose_to_instance(
         Args:
             unit: The juju unit of the github-runner charm.
             port: The port on the juju machine to expose to the runner.
+            host: Host for the reverse tunnel.
         """
         runner = self._get_single_runner(unit=unit)
         assert runner, f"Runner not found for unit {unit.name}"
@@ -61,7 +63,7 @@ async def expose_to_instance(
         key_path = f"/home/{RUNNER_MANAGER_USER}/.ssh/{runner.name}.key"
         exit_code, _, _ = await run_in_unit(unit, f"ls {key_path}")
         assert exit_code == 0, f"Unable to find key file {key_path}"
-        ssh_cmd = f'ssh -fNT -R {port}:localhost:{port} -i {key_path} -o "StrictHostKeyChecking no" -o "ControlPersist yes" ubuntu@{ip} &'
+        ssh_cmd = f'ssh -fNT -R {port}:{host}:{port} -i {key_path} -o "StrictHostKeyChecking no" -o "ControlPersist yes" ubuntu@{ip} &'
         exit_code, _, stderr = await run_in_unit(unit, ssh_cmd)
         assert (
             exit_code == 0
@@ -150,6 +152,18 @@ async def _set_app_runner_amount(app: Application, num_runners: int) -> None:
         await app.set_config({VIRTUAL_MACHINES_CONFIG_NAME: f"{num_runners}"})
         await reconcile(app=app, model=app.model)
 
+    async def get_runner_names(self, unit: Unit) -> list[str]:
+        """Get the name of all the runners in the unit.
+
+        Args:
+            unit: The GitHub Runner Charm unit to get the runner names for.
+
+        Returns:
+            List of names for the runners.
+        """
+        runners = self._get_runners(unit)
+        return [runner.name for runner in runners]
+
     async def get_runner_name(self, unit: Unit) -> str:
         """Get the name of the runner.
 
@@ -161,24 +175,27 @@ async def get_runner_name(self, unit: Unit) -> str:
         Returns:
             The Github runner name deployed in the given unit.
         """
-        runners = await self._get_runner_names(unit)
+        runners = self._get_runners(unit)
         assert len(runners) == 1
-        return runners[0]
+        return runners[0].name
 
-    async def _get_runner_names(self, unit: Unit) -> tuple[str, ...]:
-        """Get names of the runners in LXD.
+    async def delete_single_runner(self, unit: Unit) -> None:
+        """Delete the only runner.
 
         Args:
-            unit: Unit instance to check for the LXD profile.
-
-        Returns:
-            Tuple of runner names.
+            unit: The GitHub Runner Charm unit to delete the runner name for.
         """
         runner = self._get_single_runner(unit)
-        assert runner, "Failed to find runner server"
-        return (cast(str, runner.name),)
+        self.openstack_connection.delete_server(name_or_id=runner.id)
+
+    def _get_runners(self, unit: Unit) -> list[Server]:
+        """Get all runners for the unit."""
+        servers: list[Server] = self.openstack_connection.list_servers()
+        unit_name_without_slash = unit.name.replace("/", "-")
+        runners = [server for server in servers if server.name.startswith(unit_name_without_slash)]
+        return runners
 
-    def _get_single_runner(self, unit: Unit) -> Server | None:
+    def _get_single_runner(self, unit: Unit) -> Server:
         """Get the only runner for the unit.
 
         This method asserts for exactly one runner for the unit.
@@ -189,9 +206,7 @@ def _get_single_runner(self, unit: Unit) -> Server | None:
         Returns:
             The runner server.
         """
-        servers: list[Server] = self.openstack_connection.list_servers()
-        unit_name_without_slash = unit.name.replace("/", "-")
-        runners = [server for server in servers if server.name.startswith(unit_name_without_slash)]
+        runners = self._get_runners(unit)
         assert (
             len(runners) == 1
         ), f"In {unit.name} found more than one runners or no runners: {runners}"