make python apt_pkg a recommends instead of strict dependency

cloudinit/config/cc_apt_configure.py

@@ -16,8 +16,6 @@
 from textwrap import dedent, indent
 from typing import Dict
 
-import apt_pkg
-
 from cloudinit import features, gpg
 from cloudinit import log as logging
 from cloudinit import safeyaml, subp, templater, util
@@ -549,20 +547,50 @@ def is_deb822_format(apt_src_content: str) -> bool:
 
 DEFAULT_APT_CFG = {
     "sourcelist": "/etc/apt/sources.list",
-    "sourceparts": "/etc/apt/sources.list.d",
+    "sourceparts": "/etc/apt/sources.list.d/",
 }
 
+APT_CFG_RE = (
+    r"(Dir::Etc|Dir::Etc::sourceparts|Dir::Etc::sourcelist) \"([^\"]+)"
+)
+
 
 def get_apt_cfg() -> Dict[str, str]:
-    """Return a dict of applicable apt configuration or defaults."""
-    return {
-        "sourcelist": apt_pkg.config.find_file(
+    """Return a dict of applicable apt configuration or defaults.
+
+    Prefer python apt_pkg if present.
+    Fallback to apt-config dump command if present out output parsed
+    Fallback to DEFAULT_APT_CFG if apt-config commmand absent or
+    output unparsable.
+    """
+    try:
+        import apt_pkg  # noqa: F401
+
+        sourcelist = apt_pkg.config.find_file(
             "Dir::Etc::sourcelist", DEFAULT_APT_CFG["sourcelist"]
-        ),
-        "sourceparts": apt_pkg.config.find_dir(
+        )
+        sourceparts = apt_pkg.config.find_dir(
             "Dir::Etc::sourceparts", DEFAULT_APT_CFG["sourceparts"]
-        ),
-    }
+        )
+    except ImportError:
+
+        try:
+            apt_dump, _ = subp.subp(["apt-config", "dump"])
+        except subp.ProcessExecutionError:
+            # No apt-config, return defaults
+            return DEFAULT_APT_CFG
+        matched_cfg = re.findall(APT_CFG_RE, apt_dump)
+        apt_cmd_config = dict(matched_cfg)
+        etc = apt_cmd_config.get("Dir::Etc", "etc/apt")
+        if apt_cmd_config.get("Dir::Etc::sourcelist"):
+            sourcelist = f"/{etc}/{apt_cmd_config['Dir::Etc::sourcelist']}"
+        else:
+            sourcelist = DEFAULT_APT_CFG["sourcelist"]
+        if apt_cmd_config.get("Dir::Etc::sourceparts"):
+            sourceparts = f"/{etc}/{apt_cmd_config['Dir::Etc::sourceparts']}/"
+        else:
+            sourceparts = DEFAULT_APT_CFG["sourceparts"]
+    return {"sourcelist": sourcelist, "sourceparts": sourceparts}
 
 
 def generate_sources_list(cfg, release, mirrors, cloud):

tests/integration_tests/modules/test_apt.py

@@ -5,9 +5,11 @@
 
 from cloudinit import gpg
 from cloudinit.config import cc_apt_configure
+from cloudinit.util import is_true
 from tests.integration_tests.instances import IntegrationInstance
 from tests.integration_tests.integration_settings import PLATFORM
 from tests.integration_tests.releases import CURRENT_RELEASE, IS_UBUNTU
+from tests.integration_tests.util import get_feature_flag_value
 
 USER_DATA = """\
 #cloud-config
@@ -143,11 +145,19 @@ def test_sources_list(self, class_client: IntegrationInstance):
         (This is ported from
         `tests/cloud_tests/testcases/modules/apt_configure_sources_list.yaml`.)
         """
-        sources_list = class_client.read_from_file("/etc/apt/sources.list")
-        assert 6 == len(sources_list.rstrip().split("\n"))
-
-        for expected_re in EXPECTED_REGEXES:
-            assert re.search(expected_re, sources_list) is not None
+        feature_deb822 = is_true(
+            get_feature_flag_value(class_client, "APT_DEB822_SOURCE_LIST_FILE")
+        )
+        if feature_deb822:
+            sources_list = class_client.read_from_file("/etc/apt/sources.list")
+            assert 6 == len(sources_list.rstrip().split("\n"))
+            for expected_re in EXPECTED_REGEXES:
+                assert re.search(expected_re, sources_list) is not None
+        else:
+            sources_list = class_client.read_from_file(
+                "/etc/apt/sources.list.d/ubuntu.sources"
+            )
+            assert 6 == len(sources_list.rstrip().split("\n"))
 
     def test_apt_conf(self, class_client: IntegrationInstance):
         """Test the apt conf functionality.

tests/unittests/config/test_apt_source_v3.py

@@ -16,6 +16,7 @@
 
 from cloudinit import gpg, subp, util
 from cloudinit.config import cc_apt_configure
+from tests.unittests.helpers import skipIfAptPkg
 from tests.unittests.util import get_cloud
 
 EXPECTEDKEY = """-----BEGIN PGP PUBLIC KEY BLOCK-----
@@ -1540,3 +1541,57 @@ def test_disable_deb822_suites_disables_proper_suites(
         assert expected == cc_apt_configure.disable_suites_deb822(
             disabled_suites, src, "mantic"
         )
+
+
+APT_CONFIG_DUMP = """
+APT "";
+Dir "/";
+Dir::Etc "etc/myapt";
+Dir::Etc::sourcelist "sources.my.list";
+Dir::Etc::sourceparts "sources.my.list.d";
+Dir::Etc::main "apt.conf";
+"""
+
+
+class TestGetAptCfg:
+    @skipIfAptPkg()
+    @pytest.mark.parametrize(
+        "subp_side_effect,expected",
+        (
+            pytest.param(
+                [(APT_CONFIG_DUMP, "")],
+                {
+                    "sourcelist": "/etc/myapt/sources.my.list",
+                    "sourceparts": "/etc/myapt/sources.my.list.d/",
+                },
+                id="no_aptpkg_use_apt_config_cmd",
+            ),
+            pytest.param(
+                [("", "")],
+                {
+                    "sourcelist": "/etc/apt/sources.list",
+                    "sourceparts": "/etc/apt/sources.list.d/",
+                },
+                id="no_aptpkg_unparsable_apt_config_cmd_defaults",
+            ),
+            pytest.param(
+                [
+                    subp.ProcessExecutionError(
+                        "No such file or directory 'apt-config'"
+                    )
+                ],
+                {
+                    "sourcelist": "/etc/apt/sources.list",
+                    "sourceparts": "/etc/apt/sources.list.d/",
+                },
+                id="no_aptpkg_no_apt_config_cmd_defaults",
+            ),
+        ),
+    )
+    def test_use_defaults_or_apt_config_dump(
+        self, subp_side_effect, expected, mocker
+    ):
+        subp = mocker.patch("cloudinit.config.cc_apt_configure.subp.subp")
+        subp.side_effect = subp_side_effect
+        assert expected == cc_apt_configure.get_apt_cfg()
+        subp.assert_called_once_with(["apt-config", "dump"])

tests/unittests/helpers.py

@@ -39,6 +39,14 @@
 skipIf = unittest.skipIf
 
 
+try:
+    import apt_pkg  # noqa
+
+    HAS_APT_PKG = True
+except ImportError:
+    HAS_APT_PKG = False
+
+
 # Makes the old path start
 # with new base instead of whatever
 # it previously had
@@ -519,6 +527,13 @@ def readResource(name, mode="r"):
         return fh.read()
 
 
+def skipIfAptPkg():
+    return skipIf(
+        HAS_APT_PKG,
+        "No python-apt dependency present.",
+    )
+
+
 try:
     import jsonschema