feat(dhcp): Add support for dhcpcd (canonical#4746)

- dhcp: rework client api
- dhcp/isc-dhclient: consolidate duplicate code paths
- distro: make dhcp_client a distro property [1]
- distro: make fallback_interface a distro property [1]
- cloudstack: simplify "domain-name" and "dhcp-server-identifier" codepaths
- scaleway: eliminate unnecessary fallback interface detection
- ec2: eliminate old unnecessary pickle attribute check for "fallback_nic"[2]
- tests/unittest: new dhcpcd tests
- tests/integration: new dhcpcd tests for azure, ec2, gce, and openstack

[1] Required because datasource objects can access the Distro attributes but
      not the other way around. Also this is a very distro-specific codepath
      and much simplification to the code will be possible as a result of this.
[2] This "bug" was never a supported use case. Calling cloud-init manually
      after upgrade not via the init system isn't supported, but this was
      "fixed" before the use case was fully understood.

cloudinit/distros/__init__.py

@@ -17,6 +17,7 @@
 import string
 import urllib.parse
 from collections import defaultdict
+from contextlib import suppress
 from io import StringIO
 from typing import (
     Any,
@@ -147,7 +148,6 @@ class Distro(persistence.CloudInitPickleMixin, metaclass=abc.ABCMeta):
     resolve_conf_fn = "/etc/resolv.conf"
 
     osfamily: str
-    dhcp_client_priority = [dhcp.IscDhclient, dhcp.Dhcpcd, dhcp.Udhcpc]
     # Directory where the distro stores their DHCP leases.
     # The children classes should override this with their dhcp leases
     # directory
@@ -162,14 +162,12 @@ def __init__(self, name, cfg, paths):
         self._cfg = cfg
         self.name = name
         self.networking: Networking = self.networking_cls()
-        self.dhcp_client_priority = [
-            dhcp.IscDhclient,
-            dhcp.Dhcpcd,
-            dhcp.Udhcpc,
-        ]
+        self.dhcp_client_priority = dhcp.ALL_DHCP_CLIENTS
         self.net_ops = iproute2.Iproute2
         self._runner = helpers.Runners(paths)
         self.package_managers: List[PackageManager] = []
+        self._dhcp_client = None
+        self._fallback_interface = None
 
     def _unpickle(self, ci_pkl_version: int) -> None:
         """Perform deserialization fixes for Distro."""
@@ -182,6 +180,10 @@ def _unpickle(self, ci_pkl_version: int) -> None:
             # either because it isn't present at all, or because it will be
             # missing expected instance state otherwise.
             self.networking = self.networking_cls()
+        if not hasattr(self, "_dhcp_client"):
+            self._dhcp_client = None
+        if not hasattr(self, "_fallback_interface"):
+            self._fallback_interface = None
 
     def _validate_entry(self, entry):
         if isinstance(entry, str):
@@ -267,6 +269,66 @@ def install_packages(self, pkglist: PackageList):
         if uninstalled:
             raise PackageInstallerError(error_message % uninstalled)
 
+    @property
+    def dhcp_client(self) -> dhcp.DhcpClient:
+        """access the distro's preferred dhcp client
+
+        if no client has been selected yet select one - uses
+        self.dhcp_client_priority, which may be overriden in each distro's
+        object to eliminate checking for clients which will not be provided
+        by the distro
+        """
+        if self._dhcp_client:
+            return self._dhcp_client
+
+        # no client has been selected yet, so pick one
+        #
+        # set the default priority list to the distro-defined priority list
+        dhcp_client_priority = self.dhcp_client_priority
+
+        # if the configuration includes a network.dhcp_client_priority list
+        # then attempt to use it
+        config_priority = util.get_cfg_by_path(
+            self._cfg, ("network", "dhcp_client_priority"), []
+        )
+
+        if config_priority:
+            # user or image builder configured a custom dhcp client priority
+            # list
+            found_clients = []
+            LOG.debug(
+                "Using configured dhcp client priority list: %s",
+                config_priority,
+            )
+            for client_configured in config_priority:
+                for client_class in dhcp.ALL_DHCP_CLIENTS:
+                    if client_configured == client_class.client_name:
+                        found_clients.append(client_class)
+                        break
+                else:
+                    LOG.warning(
+                        "Configured dhcp client %s is not supported, skipping",
+                        client_configured,
+                    )
+            # If dhcp_client_priority is defined in the configuration, but none
+            # of the defined clients are supported by cloud-init, then we don't
+            # override the distro default. If at least one client in the
+            # configured list exists, then we use that for our list of clients
+            # to check.
+            if found_clients:
+                dhcp_client_priority = found_clients
+
+        # iterate through our priority list and use the first client that is
+        # installed on the system
+        for client in dhcp_client_priority:
+            try:
+                self._dhcp_client = client()
+                LOG.debug("DHCP client selected: %s", client.client_name)
+                return self._dhcp_client
+            except (dhcp.NoDHCPLeaseMissingDhclientError,):
+                LOG.debug("DHCP client not found: %s", client.client_name)
+        raise dhcp.NoDHCPLeaseMissingDhclientError()
+
     @property
     def network_activator(self) -> Optional[Type[activators.NetworkActivator]]:
         """Return the configured network activator for this environment."""
@@ -1183,6 +1245,75 @@ def build_dhclient_cmd(
             "/bin/true",
         ] + (["-cf", config_file, interface] if config_file else [interface])
 
+    @property
+    def fallback_interface(self):
+        """Determine the network interface used during local network config."""
+        if self._fallback_interface is None:
+            self._fallback_interface = net.find_fallback_nic()
+            if not self._fallback_interface:
+                LOG.warning(
+                    "Did not find a fallback interface on distro: %s.",
+                    self.name,
+                )
+        return self._fallback_interface
+
+    @fallback_interface.setter
+    def fallback_interface(self, value):
+        self._fallback_interface = value
+
+    @staticmethod
+    def get_proc_ppid(pid: int) -> Optional[int]:
+        """Return the parent pid of a process by parsing /proc/$pid/stat"""
+        match = Distro._get_proc_stat_by_index(pid, 4)
+        if match is not None:
+            with suppress(ValueError):
+                return int(match)
+            LOG.warning("/proc/%s/stat has an invalid ppid [%s]", pid, match)
+        return None
+
+    @staticmethod
+    def get_proc_pgid(pid: int) -> Optional[int]:
+        """Return the parent pid of a process by parsing /proc/$pid/stat"""
+        match = Distro._get_proc_stat_by_index(pid, 5)
+        if match is not None:
+            with suppress(ValueError):
+                return int(match)
+            LOG.warning("/proc/%s/stat has an invalid pgid [%s]", pid, match)
+        return None
+
+    @staticmethod
+    def _get_proc_stat_by_index(pid: int, field: int) -> Optional[int]:
+        """
+        parse /proc/$pid/stat for a specific field as numbered in man:proc(5)
+
+        param pid: integer to query /proc/$pid/stat for
+        param field: field number within /proc/$pid/stat to return
+        """
+        try:
+            content: str = util.load_file(
+                "/proc/%s/stat" % pid, quiet=True
+            ).strip()  # pyright: ignore
+            match = re.search(
+                r"^(\d+) (\(.+\)) ([RSDZTtWXxKPI]) (\d+) (\d+)", content
+            )
+            if not match:
+                LOG.warning(
+                    "/proc/%s/stat has an invalid contents [%s]", pid, content
+                )
+                return None
+            return int(match.group(field))
+        except IOError as e:
+            LOG.warning("Failed to load /proc/%s/stat. %s", pid, e)
+        except IndexError:
+            LOG.warning(
+                "Unable to match field %s of process pid=%s (%s) (%s)",
+                field,
+                pid,
+                content,  # pyright: ignore
+                match,  # pyright: ignore
+            )
+        return None
+
 
 def _apply_hostname_transformations_to_url(url: str, transformations: list):
     """

cloudinit/distros/bsd.py

@@ -141,3 +141,11 @@ def apply_locale(self, locale, out_fn=None):
     def chpasswd(self, plist_in: list, hashed: bool):
         for name, password in plist_in:
             self.set_passwd(name, password, hashed=hashed)
+
+    @staticmethod
+    def get_proc_ppid(pid):
+        """
+        Return the parent pid of a process by checking ps
+        """
+        ppid, _ = subp.subp(["ps", "-oppid=", "-p", str(pid)])
+        return int(ppid.strip())