sources/azure: retry dhcp setup for failed processes

Commands may fail in the process of performing DHCP such as
udevadm settle.  Other examples could include
ip link set dev eth0 up, etc used to prepare dhclient.

Report these failures and retry until timeout.

Signed-off-by: Chris Patterson <cpatterson@microsoft.com>

cloudinit/sources/DataSourceAzure.py

@@ -413,6 +413,19 @@ def _setup_ephemeral_networking(
                         "Failed to obtain DHCP lease (iface=%s)" % iface,
                         logger_func=LOG.error,
                     )
+                except subp.ProcessExecutionError as error:
+                    # udevadm settle, ip link set dev eth0 up, etc.
+                    report_diagnostic_event(
+                        "Command failed: "
+                        "cmd=%r stderr=%r stdout=%r exit_code=%s"
+                        % (
+                            error.cmd,
+                            error.stderr,
+                            error.stdout,
+                            error.exit_code,
+                        ),
+                        logger_func=LOG.error,
+                    )
 
                 # Sleep before retrying, otherwise break if we're past timeout.
                 if lease is None and time() + retry_sleep < timeout:

tests/unittests/sources/test_azure.py

@@ -13,7 +13,7 @@
 import requests
 import yaml
 
-from cloudinit import distros, helpers, url_helper
+from cloudinit import distros, helpers, subp, url_helper
 from cloudinit.net import dhcp
 from cloudinit.sources import UNSET
 from cloudinit.sources import DataSourceAzure as dsaz
@@ -181,6 +181,12 @@ def mock_readurl():
         yield m
 
 
+@pytest.fixture
+def mock_report_diagnostic_event():
+    with mock.patch(MOCKPATH + "report_diagnostic_event") as m:
+        yield m
+
+
 @pytest.fixture
 def mock_requests_session_request():
     with mock.patch("requests.Session.request", autospec=True) as m:
@@ -3746,6 +3752,42 @@ def test_retry_interface_error(
         assert azure_ds._wireserver_endpoint == "168.63.129.16"
         assert azure_ds._ephemeral_dhcp_ctx.iface == "fakeEth0"
 
+    def test_retry_process_error(
+        self,
+        azure_ds,
+        mock_ephemeral_dhcp_v4,
+        mock_report_diagnostic_event,
+        mock_sleep,
+    ):
+        lease = {
+            "interface": "fakeEth0",
+        }
+        mock_ephemeral_dhcp_v4.return_value.obtain_lease.side_effect = [
+            subp.ProcessExecutionError(
+                cmd=["failed", "cmd"],
+                stdout="test_stdout",
+                stderr="test_stderr",
+                exit_code=4,
+            ),
+            lease,
+        ]
+
+        azure_ds._setup_ephemeral_networking()
+
+        assert mock_ephemeral_dhcp_v4.mock_calls == [
+            mock.call(iface=None, dhcp_log_func=dsaz.dhcp_log_cb),
+            mock.call().obtain_lease(),
+            mock.call().obtain_lease(),
+        ]
+        assert mock_sleep.mock_calls == [mock.call(1)]
+        assert mock_report_diagnostic_event.mock_calls == [
+            mock.call(
+                "Command failed: cmd=['failed', 'cmd'] "
+                "stderr='test_stderr' stdout='test_stdout' exit_code=4",
+                logger_func=dsaz.LOG.error,
+            )
+        ]
+
     @pytest.mark.parametrize(
         "error_class", [dhcp.NoDHCPLeaseInterfaceError, dhcp.NoDHCPLeaseError]
     )