bug(status): retry systemctl show to avoid dbus disconnect (canonical…

…#4679)

Fixes canonicalGH-4676

cloudinit/cmd/status.py

@@ -285,14 +285,20 @@ def _get_systemd_status() -> Optional[UXAppStatus]:
         "cloud-init.service",
         "cloud-init-local.service",
     ]:
-        stdout = subp.subp(
-            [
-                "systemctl",
-                "show",
-                "--property=ActiveState,UnitFileState,SubState,MainPID",
-                service,
-            ],
-        ).stdout
+        cmd = [
+            "systemctl",
+            "show",
+            "--property=ActiveState,UnitFileState,SubState,MainPID",
+            service,
+        ]
+        for retry_sleep in [0.25] * 10:  # 10 retries sleeping 0.25 sec between
+            try:
+                stdout = subp.subp(cmd).stdout
+                break
+            except subp.ProcessExecutionError as e:
+                print("Retrying: %s on %s", cmd, e)
+                sleep(retry_sleep)
+
         states = dict(
             [[x.strip() for x in r.split("=")] for r in stdout.splitlines()]
         )

tests/unittests/cmd/test_status.py

@@ -12,7 +12,7 @@
 from cloudinit.atomic_helper import write_json
 from cloudinit.cmd import status
 from cloudinit.cmd.status import UXAppStatus, _get_systemd_status
-from cloudinit.subp import SubpResult
+from cloudinit.subp import ProcessExecutionError, SubpResult
 from cloudinit.util import ensure_file
 from tests.unittests.helpers import wrap_and_call
 
@@ -874,6 +874,28 @@ def test_status_main(
 
 
 class TestSystemdStatusDetails:
+    def test_retry_on_failures(self):
+        with mock.patch(
+            f"{M_PATH}subp.subp",
+            side_effect=[
+                ProcessExecutionError(
+                    "Message recipient disconnected from message bus without"
+                    " replying"
+                ),
+                ProcessExecutionError(
+                    "Message recipient disconnected from message bus without"
+                    " replying"
+                ),
+                SubpResult(
+                    f"ActiveState=activating\nUnitFileState=enabled\n"
+                    f"SubState=start\nMainPID=123\n",
+                    stderr=None,
+                ),
+            ],
+        ) as m_supb:
+            assert _get_systemd_status() == UXAppStatus.RUNNING
+        assert 3 == m_supb.call_count
+
     @pytest.mark.parametrize(
         ["active_state", "unit_file_state", "sub_state", "main_pid", "status"],
         [