diff --git a/cloudinit/config/cc_apt_configure.py b/cloudinit/config/cc_apt_configure.py index f07eb5db8b4..cc3db5e7c13 100644 --- a/cloudinit/config/cc_apt_configure.py +++ b/cloudinit/config/cc_apt_configure.py @@ -15,9 +15,10 @@ import re import shutil import signal -from textwrap import dedent +from textwrap import dedent, indent +from typing import Dict -from cloudinit import gpg, subp, templater, util +from cloudinit import features, gpg, subp, templater, util from cloudinit.cloud import Cloud from cloudinit.config import Config from cloudinit.config.schema import MetaSchema, get_meta_doc @@ -31,6 +32,7 @@ APT_LOCAL_KEYS = "/etc/apt/trusted.gpg" APT_TRUSTED_GPG_DIR = "/etc/apt/trusted.gpg.d/" CLOUD_INIT_GPG_DIR = "/etc/apt/cloud-init.gpg.d/" +DISABLE_SUITES_REDACT_PREFIX = "# cloud-init disable_suites redacted: " frequency = PER_INSTANCE distros = ["ubuntu", "debian"] @@ -54,6 +56,13 @@ cloud-init's distro support for instructions on using these config options. + By default, cloud-init will generate default + apt sources information in deb822 format at + :file:`/etc/apt/sources.list.d/.sources`. When the value + of `sources_list` does not appear to be deb822 format, or stable + distribution releases disable deb822 format, + :file:`/etc/apt/sources.list` will be written instead. + .. note:: To ensure that apt configuration is valid yaml, any strings containing special characters, especially ``:`` should be quoted. @@ -130,7 +139,21 @@ ------BEGIN PGP PUBLIC KEY BLOCK------- ------END PGP PUBLIC KEY BLOCK-------""" - ) + ), + dedent( + """\ + # cloud-init version 23.4 will generate a deb822 formatted sources + # file at /etc/apt/sources.list.d/.sources instead of + # /etc/apt/sources.list when `sources_list` content is deb822 + # format. + apt: + sources_list: | + Types: deb + URIs: http://archive.ubuntu.com/ubuntu/ + Suites: $RELEASE + Components: main + """ + ), ], "frequency": frequency, "activate_by_schema_keys": [], @@ -416,13 +439,78 @@ def map_known_suites(suite): return retsuite -def disable_suites(disabled, src, release): +def disable_deb822_section_without_suites(deb822_entry: str) -> str: + """If no active Suites, disable this deb822 source.""" + if not re.findall(r"\nSuites:[ \t]+([\w-]+)", deb822_entry): + # No Suites remaining in this entry, disable full entry + # Reconstitute commented Suites line to original as we disable entry + deb822_entry = re.sub(r"\nSuites:.*", "", deb822_entry) + deb822_entry = re.sub( + rf"{DISABLE_SUITES_REDACT_PREFIX}", "", deb822_entry + ) + return ( + "## Entry disabled by cloud-init, due to disable_suites\n" + + indent(deb822_entry, "# disabled by cloud-init: ") + ) + return deb822_entry + + +def disable_suites_deb822(disabled, src, release) -> str: + """reads the deb822 format config and comment disabled suites""" + new_src = [] + disabled_suite_names = [ + templater.render_string(map_known_suites(suite), {"RELEASE": release}) + for suite in disabled + ] + LOG.debug("Disabling suites %s as %s", disabled, disabled_suite_names) + new_deb822_entry = "" + for line in src.splitlines(): + if line.startswith("#"): + if new_deb822_entry: + new_deb822_entry += f"{line}\n" + else: + new_src.append(line) + continue + if not line or line.isspace(): + # Validate/disable deb822 entry upon whitespace + if new_deb822_entry: + new_src.append( + disable_deb822_section_without_suites(new_deb822_entry) + ) + new_deb822_entry = "" + new_src.append(line) + continue + new_line = line + if not line.startswith("Suites:"): + new_deb822_entry += line + "\n" + continue + # redact all disabled suite names + if disabled_suite_names: + # Redact any matching Suites from line + orig_suites = line.split()[1:] + new_suites = [ + suite + for suite in orig_suites + if suite not in disabled_suite_names + ] + if new_suites != orig_suites: + new_deb822_entry += f"{DISABLE_SUITES_REDACT_PREFIX}{line}\n" + new_line = f"Suites: {' '.join(new_suites)}" + new_deb822_entry += new_line + "\n" + if new_deb822_entry: + new_src.append(disable_deb822_section_without_suites(new_deb822_entry)) + return "\n".join(new_src) + + +def disable_suites(disabled, src, release) -> str: """reads the config for suites to be disabled and removes those from the template""" if not disabled: return src retsrc = src + if is_deb822_sources_format(src): + return disable_suites_deb822(disabled, src, release) for suite in disabled: suite = map_known_suites(suite) releasesuite = templater.render_string(suite, {"RELEASE": release}) @@ -461,34 +549,139 @@ def add_mirror_keys(cfg, cloud, target): add_apt_key(mirror, cloud, target, file_name=key) +def is_deb822_sources_format(apt_src_content: str) -> bool: + """Simple check for deb822 format for apt source content + + Only validates that minimal required keys are present in the file, which + indicates we are likely deb822 format. + + Doesn't handle if multiple sections all contain deb822 keys. + + Return True if content looks like it is deb822 formatted APT source. + """ + # TODO(At jammy EOL: use aptsources.sourceslist.Deb822SourceEntry.invalid) + if re.findall(r"^(deb |deb-src )", apt_src_content, re.M): + return False + if re.findall( + r"^(Types: |Suites: |Components: |URIs: )", apt_src_content, re.M + ): + return True + # Did not match any required deb822 format keys + LOG.warning( + "apt.sources_list value does not match either deb822 source keys or" + " deb/deb-src list keys. Assuming APT deb/deb-src list format." + ) + return False + + +DEFAULT_APT_CFG = { + "Dir::Etc": "etc/apt", + "Dir::Etc::sourcelist": "sources.list", + "Dir::Etc::sourceparts": "sources.list.d", +} + +APT_CFG_RE = ( + r"(Dir::Etc|Dir::Etc::sourceparts|Dir::Etc::sourcelist) \"([^\"]+)" +) + + +def get_apt_cfg() -> Dict[str, str]: + """Return a dict of applicable apt configuration or defaults. + + Prefer python apt_pkg if present. + Fallback to apt-config dump command if present out output parsed + Fallback to DEFAULT_APT_CFG if apt-config commmand absent or + output unparsable. + """ + try: + import apt_pkg # type: ignore + + apt_pkg.init_config() + etc = apt_pkg.config.get("Dir::Etc", DEFAULT_APT_CFG["Dir::Etc"]) + sourcelist = apt_pkg.config.get( + "Dir::Etc::sourcelist", DEFAULT_APT_CFG["Dir::Etc::sourcelist"] + ) + sourceparts = apt_pkg.config.get( + "Dir::Etc::sourceparts", DEFAULT_APT_CFG["Dir::Etc::sourceparts"] + ) + except ImportError: + + try: + apt_dump, _ = subp.subp(["apt-config", "dump"]) + except subp.ProcessExecutionError: + # No apt-config, return defaults + etc = DEFAULT_APT_CFG["Dir::Etc"] + sourcelist = DEFAULT_APT_CFG["Dir::Etc::sourcelist"] + sourceparts = DEFAULT_APT_CFG["Dir::Etc::sourceparts"] + return { + "sourcelist": f"/{etc}/{sourcelist}", + "sourceparts": f"/{etc}/{sourceparts}/", + } + matched_cfg = re.findall(APT_CFG_RE, apt_dump) + apt_cmd_config = dict(matched_cfg) + etc = apt_cmd_config.get("Dir::Etc", DEFAULT_APT_CFG["Dir::Etc"]) + sourcelist = apt_cmd_config.get( + "Dir::Etc::sourcelist", DEFAULT_APT_CFG["Dir::Etc::sourcelist"] + ) + sourceparts = apt_cmd_config.get( + "Dir::Etc::sourceparts", DEFAULT_APT_CFG["Dir::Etc::sourceparts"] + ) + return { + "sourcelist": f"/{etc}/{sourcelist}", + "sourceparts": f"/{etc}/{sourceparts}/", + } + + def generate_sources_list(cfg, release, mirrors, cloud): """generate_sources_list create a source.list file based on a custom or default template by replacing mirrors and release in the template""" - aptsrc = "/etc/apt/sources.list" + apt_cfg = get_apt_cfg() + apt_sources_list = apt_cfg["sourcelist"] + apt_sources_deb822 = f"{apt_cfg['sourceparts']}{cloud.distro.name}.sources" + if features.APT_DEB822_SOURCE_LIST_FILE: + aptsrc_file = apt_sources_deb822 + else: + aptsrc_file = apt_sources_list + params = {"RELEASE": release, "codename": release} for k in mirrors: params[k] = mirrors[k] params[k.lower()] = mirrors[k] tmpl = cfg.get("sources_list", None) - if tmpl is None: + if not tmpl: LOG.info("No custom template provided, fall back to builtin") + tmpl_fmt = ".deb822" if features.APT_DEB822_SOURCE_LIST_FILE else "" template_fn = cloud.get_template_filename( - "sources.list.%s" % (cloud.distro.name) + f"sources.list.{cloud.distro.name}{tmpl_fmt}" ) if not template_fn: template_fn = cloud.get_template_filename("sources.list") if not template_fn: - LOG.warning( - "No template found, not rendering /etc/apt/sources.list" - ) + LOG.warning("No template found, not rendering %s", aptsrc_file) return tmpl = util.load_file(template_fn) rendered = templater.render_string(tmpl, params) + if tmpl: + if is_deb822_sources_format(rendered): + if aptsrc_file == apt_sources_list: + LOG.debug( + "Provided 'sources_list' user-data is deb822 format," + " writing to %s", + apt_sources_deb822, + ) + aptsrc_file = apt_sources_deb822 + else: + LOG.debug( + "Provided 'sources_list' user-data is not deb822 format," + " fallback to %s", + apt_sources_list, + ) + aptsrc_file = apt_sources_list disabled = disable_suites(cfg.get("disable_suites"), rendered, release) - util.write_file(aptsrc, disabled, mode=0o644) + util.write_file(aptsrc_file, disabled, mode=0o644) def add_apt_key_raw(key, file_name, hardened=False, target=None): diff --git a/cloudinit/features.py b/cloudinit/features.py index db1b3f1fbea..d864fdf695e 100644 --- a/cloudinit/features.py +++ b/cloudinit/features.py @@ -80,6 +80,13 @@ (This flag can be removed when Jammy is no longer supported.) """ +APT_DEB822_SOURCE_LIST_FILE = True +""" +On Debian and Ubuntu systems, cc_apt_configure will write a deb822 compatible +/etc/apt/sources.list.d/(debian|ubuntu).sources file. When set False, continue +to write /etc/apt/sources.list directly. +""" + def get_features() -> Dict[str, bool]: """Return a dict of applicable features/overrides and their values.""" diff --git a/cloudinit/subp.py b/cloudinit/subp.py index 7624d9f39a4..72fdecea377 100644 --- a/cloudinit/subp.py +++ b/cloudinit/subp.py @@ -345,7 +345,6 @@ def target_path(target, path=None): # os.path.join("/etc", "/foo") returns "/foo". Chomp all leading /. while len(path) and path[0] == "/": path = path[1:] - return os.path.join(target, path) diff --git a/templates/sources.list.debian.deb822.tmpl b/templates/sources.list.debian.deb822.tmpl new file mode 100644 index 00000000000..6d15096c3ab --- /dev/null +++ b/templates/sources.list.debian.deb822.tmpl @@ -0,0 +1,34 @@ +## template:jinja +## Note, this file is written by cloud-init on first boot of an instance +## modifications made here will not survive a re-bundle. +## if you wish to make changes you can: +## a.) add 'apt_preserve_sources_list: true' to /etc/cloud/cloud.cfg +## or do the same in user-data +## b.) add supplemental sources in /etc/apt/sources.list.d +## c.) make changes to template file +## /etc/cloud/templates/sources.list.debian.deb822.tmpl + +# For how to upgrade to newer versions of the distribution, see: +# http://www.debian.org/releases/stable/i386/release-notes/ch-upgrading.html + +## See the sources.list(5) manual page for further settings. + +## Comment any of the following Suites to avoid getting updates from the +## specific Suite. +## +## N.B. software from {{codename}}-backports repository may not have been +## tested as extensively as that contained in the main release, although it +## includes newer versions of some applications which may provide useful +## features. +Types: deb deb-src +URIs: {{mirror}} +Suites: {{codename}} {{codename}}-updates {{codename}}-backports +Components: main +Signed-By: /usr/share/keyrings/debian-archive-keyring.gpg + +## Major bug fix updates produced after the final release of the distribution. +Types: deb deb-src +URIs: {{security}} +Suites: {{codename}}{% if codename in ('buster', 'stretch') %}/updates{% else %}-security{% endif %} +Components: main +Signed-By: /usr/share/keyrings/debian-archive-keyring.gpg diff --git a/templates/sources.list.ubuntu.deb822.tmpl b/templates/sources.list.ubuntu.deb822.tmpl new file mode 100644 index 00000000000..d75dc088f0b --- /dev/null +++ b/templates/sources.list.ubuntu.deb822.tmpl @@ -0,0 +1,61 @@ +## template:jinja +## Note, this file is written by cloud-init on first boot of an instance +## modifications made here will not survive a re-bundle. +## if you wish to make changes you can: +## a.) add 'apt_preserve_sources_list: true' to /etc/cloud/cloud.cfg +## or do the same in user-data +## b.) add supplemental sources in /etc/apt/sources.list.d +## c.) make changes to template file +## /etc/cloud/templates/sources.list.ubuntu.deb822.tmpl +## +## See http://help.ubuntu.com/community/UpgradeNotes for how to upgrade to +## newer versions of the distribution. +## +## The following settings can be tweaked to configure which packages to use +## from Ubuntu. +## Mirror your choices (except for URIs and Suites) in the security section +## below to ensure timely security updates. +## +## Types: Append deb-src to enable the fetching of source package. +## URIs: A URL to the repository (you may add multiple URLs) +## Suites: The following additional suites can be configured +## -updates - Major bug fix updates produced after the final release +## of the distribution. +## -backports - software from this repository may not have been tested +## as extensively as that contained in the main release, +## although it includes newer versions of some +## applications which may provide useful features. +## Also, please note that software in backports WILL NOT +## receive any review or updates from the Ubuntu security +## team. +## Components: Aside from main, the following components can be added to the +## list: +## restricted - Software that may not be under a free license, or protected +## by patents. +## universe - Community maintained packages. Software from this repository +## is ENTIRELY UNSUPPORTED by the Ubuntu team. Also, please +## note that software in universe WILL NOT receive any +## review or updates from the Ubuntu security team. +## multiverse - Community maintained of restricted. Software from this +## repository is ENTIRELY UNSUPPORTED by the Ubuntu team, and +## may not be under a free licence. Please satisfy yourself as +## to your rights to use the software. +## Also, please note that software in multiverse WILL NOT +## receive any review or updates from the Ubuntu security team. +## +## See the sources.list(5) manual page for further settings. +# Types: deb deb-src +Types: deb +URIs: {{mirror}} +Suites: {{codename}} {{codename}}-updates {{codename}}-backports +Components: main restricted universe multiverse +Signed-By: /usr/share/keyrings/ubuntu-archive-keyring.gpg + +## Ubuntu security updates. Aside from URIs and Suites, +## this should mirror your choices in the previous section. +# Types: deb deb-src +Types: deb +URIs: {{security}} +Suites: {{codename}}-security +Components: main restricted universe multiverse +Signed-By: /usr/share/keyrings/ubuntu-archive-keyring.gpg diff --git a/tests/integration_tests/modules/test_apt_functionality.py b/tests/integration_tests/modules/test_apt_functionality.py index 7eb6e32d378..e80bce27429 100644 --- a/tests/integration_tests/modules/test_apt_functionality.py +++ b/tests/integration_tests/modules/test_apt_functionality.py @@ -1,18 +1,29 @@ # This file is part of cloud-init. See LICENSE file for license information. """Series of integration tests covering apt functionality.""" import re +from textwrap import dedent import pytest from cloudinit import gpg from cloudinit.config import cc_apt_configure +from cloudinit.util import is_true from tests.integration_tests.instances import IntegrationInstance from tests.integration_tests.integration_settings import PLATFORM from tests.integration_tests.releases import CURRENT_RELEASE, IS_UBUNTU -from tests.integration_tests.util import verify_clean_log +from tests.integration_tests.util import ( + get_feature_flag_value, + verify_clean_log, +) + +DEB822_SOURCES_FILE = "/etc/apt/sources.list.d/ubuntu.sources" +ORIG_SOURCES_FILE = "/etc/apt/sources.list" USER_DATA = """\ #cloud-config +bootcmd: + - rm -f /etc/apt/sources.list /etc/apt/sources.list.d/ubuntu.sources + apt: conf: | APT { @@ -142,20 +153,16 @@ def test_sources_list(self, class_client: IntegrationInstance): expected number of sources.list entries is present, and (b) that each expected line appears in the file. - (This is ported from - `tests/cloud_tests/testcases/modules/apt_configure_sources_list.yaml`.) + Since sources_list is no deb822-compliant, ORIG_SOURCES_FILE will + always be written regardless of feature.APT_DEB822_SOURCE_LIST_FILE """ - sources_list = class_client.read_from_file("/etc/apt/sources.list") + sources_list = class_client.read_from_file(ORIG_SOURCES_FILE) assert 6 == len(sources_list.rstrip().split("\n")) - for expected_re in EXPECTED_REGEXES: assert re.search(expected_re, sources_list) is not None def test_apt_conf(self, class_client: IntegrationInstance): - """Test the apt conf functionality. - - Ported from tests/cloud_tests/testcases/modules/apt_configure_conf.py - """ + """Test the apt conf functionality.""" apt_config = class_client.read_from_file( "/etc/apt/apt.conf.d/94cloud-init-config" ) @@ -163,27 +170,17 @@ def test_apt_conf(self, class_client: IntegrationInstance): assert 'Fix-Broken "true";' in apt_config def test_ppa_source(self, class_client: IntegrationInstance): - """Test the apt ppa functionality. - - Ported from - tests/cloud_tests/testcases/modules/apt_configure_sources_ppa.py - """ - if CURRENT_RELEASE.series < "mantic": - ppa_path_contents = class_client.read_from_file( - "/etc/apt/sources.list.d/" - "simplestreams-dev-ubuntu-trunk-{}.list".format( - CURRENT_RELEASE.series - ) + """Test the apt ppa functionality.""" + ppa_path = ( + "/etc/apt/sources.list.d/simplestreams-dev-ubuntu-trunk-{}".format( + CURRENT_RELEASE.series ) + ) + if CURRENT_RELEASE.series < "mantic": + ppa_path += ".list" else: - # deb822 support present on mantic and later - ppa_path_contents = class_client.read_from_file( - "/etc/apt/sources.list.d/" - "simplestreams-dev-ubuntu-trunk-{}.sources".format( - CURRENT_RELEASE.series - ) - ) - + ppa_path += ".sources" + ppa_path_contents = class_client.read_from_file(ppa_path) assert ( "://ppa.launchpad.net/simplestreams-dev/trunk/ubuntu" in ppa_path_contents @@ -220,11 +217,7 @@ def test_bad_key(self, class_client: IntegrationInstance): ) def test_key(self, class_client: IntegrationInstance): - """Test the apt key functionality. - - Ported from - tests/cloud_tests/testcases/modules/apt_configure_sources_key.py - """ + """Test the apt key functionality.""" test_archive_contents = class_client.read_from_file( "/etc/apt/sources.list.d/test_key.list" ) @@ -236,11 +229,7 @@ def test_key(self, class_client: IntegrationInstance): assert TEST_KEY in self.get_keys(class_client) def test_keyserver(self, class_client: IntegrationInstance): - """Test the apt keyserver functionality. - - Ported from - tests/cloud_tests/testcases/modules/apt_configure_sources_keyserver.py - """ + """Test the apt keyserver functionality.""" test_keyserver_contents = class_client.read_from_file( "/etc/apt/sources.list.d/test_keyserver.list" ) @@ -253,10 +242,7 @@ def test_keyserver(self, class_client: IntegrationInstance): assert TEST_KEYSERVER_KEY in self.get_keys(class_client) def test_os_pipelining(self, class_client: IntegrationInstance): - """Test 'os' settings does not write apt config file. - - Ported from tests/cloud_tests/testcases/modules/apt_pipelining_os.py - """ + """Test 'os' settings does not write apt config file.""" conf_exists = class_client.execute( "test -f /etc/apt/apt.conf.d/90cloud-init-pipelining" ).ok @@ -315,28 +301,40 @@ def test_primary_on_openstack(self, class_client: IntegrationInstance): When no uri is provided. """ zone = class_client.execute("cloud-init query v1.availability_zone") - sources_list = class_client.read_from_file("/etc/apt/sources.list") + feature_deb822 = is_true( + get_feature_flag_value(class_client, "APT_DEB822_SOURCE_LIST_FILE") + ) + src_file = DEB822_SOURCES_FILE if feature_deb822 else ORIG_SOURCES_FILE + sources_list = class_client.read_from_file(src_file) assert "{}.clouds.archive.ubuntu.com".format(zone) in sources_list def test_security(self, class_client: IntegrationInstance): - """Test apt default security sources. - - Ported from - tests/cloud_tests/testcases/modules/apt_configure_security.py - """ - sources_list = class_client.read_from_file("/etc/apt/sources.list") - - # 3 lines from main, universe, and multiverse + """Test apt default security sources.""" series = CURRENT_RELEASE.series - sec_url = f"deb http://security.ubuntu.com/ubuntu {series}-security" + feature_deb822 = is_true( + get_feature_flag_value(class_client, "APT_DEB822_SOURCE_LIST_FILE") + ) if class_client.settings.PLATFORM == "azure": - sec_url = ( - f"deb http://azure.archive.ubuntu.com/ubuntu/" - f" {series}-security" + sec_url = "deb http://azure.archive.ubuntu.com/ubuntu/" + else: + sec_url = "http://security.ubuntu.com/ubuntu" + if feature_deb822: + expected_cfg = dedent( + f"""\ + Types: deb + URIs: {sec_url} + Suites: {series}-security + """ ) - sec_src_url = sec_url.replace("deb ", "# deb-src ") - assert 3 == sources_list.count(sec_url) - assert 3 == sources_list.count(sec_src_url) + sources_list = class_client.read_from_file(DEB822_SOURCES_FILE) + assert expected_cfg in sources_list + else: + sources_list = class_client.read_from_file(ORIG_SOURCES_FILE) + # 3 lines from main, universe, and multiverse + sec_deb_line = f"deb {sec_url} {series}-security" + sec_src_deb_line = sec_deb_line.replace("deb ", "# deb-src ") + assert 3 == sources_list.count(sec_deb_line) + assert 3 == sources_list.count(sec_src_deb_line) DEFAULT_DATA_WITH_URI = _DEFAULT_DATA.format( @@ -346,20 +344,22 @@ def test_security(self, class_client: IntegrationInstance): @pytest.mark.user_data(DEFAULT_DATA_WITH_URI) def test_default_primary_with_uri(client: IntegrationInstance): - """Test apt default primary sources. - - Ported from - tests/cloud_tests/testcases/modules/apt_configure_primary.py - """ - sources_list = client.read_from_file("/etc/apt/sources.list") + """Test apt default primary sources.""" + feature_deb822 = is_true( + get_feature_flag_value(client, "APT_DEB822_SOURCE_LIST_FILE") + ) + src_file = DEB822_SOURCES_FILE if feature_deb822 else ORIG_SOURCES_FILE + sources_list = client.read_from_file(src_file) assert "archive.ubuntu.com" not in sources_list - assert "something.random.invalid" in sources_list DISABLED_DATA = """\ #cloud-config +bootcmd: [mkdir -p /etc/apt/sources.new.d] apt: + conf: | + Dir::Etc::sourceparts "sources.new.d"; disable_suites: - $RELEASE - $RELEASE-updates @@ -373,22 +373,22 @@ def test_default_primary_with_uri(client: IntegrationInstance): @pytest.mark.user_data(DISABLED_DATA) class TestDisabled: def test_disable_suites(self, class_client: IntegrationInstance): - """Test disabling of apt suites. - - Ported from - tests/cloud_tests/testcases/modules/apt_configure_disable_suites.py - """ + """Test disabling of apt suites.""" + feature_deb822 = is_true( + get_feature_flag_value(class_client, "APT_DEB822_SOURCE_LIST_FILE") + ) + if feature_deb822: + # DISABLED_DATA changes Dir:Etc::sourceparts to sources.new.d + src_file = DEB822_SOURCES_FILE.replace("list", "new") + else: + src_file = ORIG_SOURCES_FILE sources_list = class_client.execute( - "cat /etc/apt/sources.list | grep -v '^#'" + f"cat {src_file} | grep -v '^#'" ).strip() assert "" == sources_list def test_disable_apt_pipelining(self, class_client: IntegrationInstance): - """Test disabling of apt pipelining. - - Ported from - tests/cloud_tests/testcases/modules/apt_pipelining_disable.py - """ + """Test disabling of apt pipelining.""" conf = class_client.read_from_file( "/etc/apt/apt.conf.d/90cloud-init-pipelining" ) diff --git a/tests/unittests/config/test_apt_configure_sources_list_v1.py b/tests/unittests/config/test_apt_configure_sources_list_v1.py index ed6acbb13f6..f8afe84557e 100644 --- a/tests/unittests/config/test_apt_configure_sources_list_v1.py +++ b/tests/unittests/config/test_apt_configure_sources_list_v1.py @@ -14,15 +14,6 @@ EXAMPLE_TMPL = """\ ## template:jinja ## Note, this file is written by cloud-init on first boot of an instance -## modifications made here will not survive a re-bundle. -## if you wish to make changes you can: -## a.) add 'apt_preserve_sources_list: true' to /etc/cloud/cloud.cfg -## or do the same in user-data -## b.) add sources in /etc/apt/sources.list.d -## c.) make changes to template file /etc/cloud/templates/sources.list.tmpl - -# See http://help.ubuntu.com/community/UpgradeNotes for how to upgrade to -# newer versions of the distribution. deb {{mirror}} {{codename}} main restricted deb-src {{mirror}} {{codename}} main restricted """ @@ -31,34 +22,60 @@ apt_mirror: http://archive.ubuntu.com/ubuntu/ apt_custom_sources_list: | ## Note, this file is written by cloud-init on first boot of an instance - ## modifications made here will not survive a re-bundle. - ## if you wish to make changes you can: - ## a.) add 'apt_preserve_sources_list: true' to /etc/cloud/cloud.cfg - ## or do the same in user-data - ## b.) add sources in /etc/apt/sources.list.d - ## c.) make changes to template file /etc/cloud/templates/sources.list.tmpl - - # See http://help.ubuntu.com/community/UpgradeNotes for how to upgrade to - # newer versions of the distribution. deb $MIRROR $RELEASE main restricted deb-src $MIRROR $RELEASE main restricted # FIND_SOMETHING_SPECIAL """ EXPECTED_CONVERTED_CONTENT = """## Note, this file is written by cloud-init on first boot of an instance -## modifications made here will not survive a re-bundle. -## if you wish to make changes you can: -## a.) add 'apt_preserve_sources_list: true' to /etc/cloud/cloud.cfg -## or do the same in user-data -## b.) add sources in /etc/apt/sources.list.d -## c.) make changes to template file /etc/cloud/templates/sources.list.tmpl - -# See http://help.ubuntu.com/community/UpgradeNotes for how to upgrade to -# newer versions of the distribution. deb http://archive.ubuntu.com/ubuntu/ fakerelease main restricted deb-src http://archive.ubuntu.com/ubuntu/ fakerelease main restricted """ # noqa: E501 +EXAMPLE_TMPL_DEB822 = """\ +## template:jinja +Types: deb deb-src +URIs: {{mirror}} +Suites: {{codename}} {{codename}}-updates +Components: main restricted + +# Security section +Types: deb deb-src +URIs: {{security}} +Suites: {{codename}}-security +Components: main restricted +""" + +YAML_TEXT_CUSTOM_SL_DEB822 = """ +apt_mirror: http://archive.ubuntu.com/ubuntu/ +apt_custom_sources_list: | + ## template:jinja + Types: deb deb-src + URIs: {{mirror}} + Suites: {{codename}} {{codename}}-updates + Components: main restricted + + # Security section + Types: deb deb-src + URIs: {{security}} + Suites: {{codename}}-security + Components: main restricted + # custom_sources_list +""" + +EXPECTED_CONVERTED_CONTENT_DEB822 = """\ +Types: deb deb-src +URIs: http://archive.ubuntu.com/ubuntu/ +Suites: fakerelease fakerelease-updates +Components: main restricted + +# Security section +Types: deb deb-src +URIs: http://archive.ubuntu.com/ubuntu/ +Suites: fakerelease-security +Components: main restricted +""" + @pytest.mark.usefixtures("fake_filesystem") class TestAptSourceConfigSourceList: @@ -67,7 +84,7 @@ class TestAptSourceConfigSourceList: """ @pytest.fixture(autouse=True) - def setup(self, mocker): + def common_mocks(self, mocker): self.subp = mocker.patch.object( subp, "subp", return_value=("PPID PID", "") ) @@ -76,6 +93,17 @@ def setup(self, mocker): lsb.return_value = {"codename": "fakerelease"} m_arch = mocker.patch("cloudinit.util.get_dpkg_architecture") m_arch.return_value = "amd64" + self.deb822 = mocker.patch.object( + cc_apt_configure.features, "APT_DEB822_SOURCE_LIST_FILE", True + ) + mocker.patch.object( + cc_apt_configure, + "get_apt_cfg", + return_value={ + "sourcelist": "/etc/apt/sources.list", + "sourceparts": "/etc/apt/sources.list.d/", + }, + ) def apt_source_list(self, distro, mirror, tmpdir, mirrorcheck=None): """apt_source_list @@ -120,12 +148,12 @@ def apt_source_list(self, distro, mirror, tmpdir, mirrorcheck=None): def test_apt_v1_source_list_by_distro(self, distro, mirror, tmpdir): """Test rendering of a source.list from template for each distro""" mycloud = get_cloud(distro) - tmpl_file = f"/etc/cloud/templates/sources.list.{distro}.tmpl" - util.write_file(tmpl_file, EXAMPLE_TMPL) + tmpl_file = f"/etc/cloud/templates/sources.list.{distro}.deb822.tmpl" + util.write_file(tmpl_file, EXAMPLE_TMPL_DEB822) cc_apt_configure.handle("test", {"apt_mirror": mirror}, mycloud, None) - sources_file = tmpdir.join("/etc/apt/sources.list") + sources_file = tmpdir.join(f"/etc/apt/sources.list.d/{distro}.sources") assert ( - EXPECTED_CONVERTED_CONTENT.replace( + EXPECTED_CONVERTED_CONTENT_DEB822.replace( "http://archive.ubuntu.com/ubuntu/", mirror ) == sources_file.read() @@ -139,18 +167,6 @@ def test_apt_v1_source_list_by_distro(self, distro, mirror, tmpdir): rcs=[0, 1], ) - def test_apt_v1_source_list_ubuntu(self, tmpdir): - """Test rendering of a source.list from template for ubuntu""" - self.apt_source_list( - "ubuntu", "http://archive.ubuntu.com/ubuntu/", tmpdir - ) - self.subp.assert_called_once_with( - ["ps", "-o", "ppid,pid", "-C", "dirmngr", "-C", "gpg-agent"], - capture=True, - target=None, - rcs=[0, 1], - ) - @staticmethod def myresolve(name): """Fake util.is_resolvable for mirrorfail tests""" @@ -182,10 +198,10 @@ def myresolve(name): def test_apt_v1_srcl_distro_mirrorfail( self, distro, mirrorlist, mirrorcheck, mocker, tmpdir ): - """Test rendering of a source.list from template for ubuntu""" + """Test rendering of a source.list from template for each distro""" mycloud = get_cloud(distro) - tmpl_file = f"/etc/cloud/templates/sources.list.{distro}.tmpl" - util.write_file(tmpl_file, EXAMPLE_TMPL) + tmpl_file = f"/etc/cloud/templates/sources.list.{distro}.deb822.tmpl" + util.write_file(tmpl_file, EXAMPLE_TMPL_DEB822) mockresolve = mocker.patch.object( util, "is_resolvable", side_effect=self.myresolve @@ -193,9 +209,9 @@ def test_apt_v1_srcl_distro_mirrorfail( cc_apt_configure.handle( "test", {"apt_mirror_search": mirrorlist}, mycloud, None ) - sources_file = tmpdir.join("/etc/apt/sources.list") + sources_file = tmpdir.join(f"/etc/apt/sources.list.d/{distro}.sources") assert ( - EXPECTED_CONVERTED_CONTENT.replace( + EXPECTED_CONVERTED_CONTENT_DEB822.replace( "http://archive.ubuntu.com/ubuntu/", mirrorcheck ) == sources_file.read() @@ -212,21 +228,55 @@ def test_apt_v1_srcl_distro_mirrorfail( ) @pytest.mark.parametrize( - "cfg,apt_file,expected", + "deb822,cfg,apt_file,expected", ( pytest.param( + True, + util.load_yaml(YAML_TEXT_CUSTOM_SL_DEB822), + "/etc/apt/sources.list.d/ubuntu.sources", + EXPECTED_CONVERTED_CONTENT_DEB822 + "# custom_sources_list\n", + id="deb822_and_deb822_sources_list_writes_deb822_source_file", + ), + pytest.param( + True, util.load_yaml(YAML_TEXT_CUSTOM_SL), "/etc/apt/sources.list", EXPECTED_CONVERTED_CONTENT + "# FIND_SOMETHING_SPECIAL\n", - id="sources_list_writes_list_file", + id="deb822_and_non_deb822_sources_list_writes_apt_list_file", + ), + pytest.param( + False, + util.load_yaml(YAML_TEXT_CUSTOM_SL), + "/etc/apt/sources.list", + EXPECTED_CONVERTED_CONTENT + "# FIND_SOMETHING_SPECIAL\n", + id="nodeb822_and_nondeb822_sources_list_writes_list_file", + ), + pytest.param( + True, + util.load_yaml(YAML_TEXT_CUSTOM_SL_DEB822), + "/etc/apt/sources.list.d/ubuntu.sources", + EXPECTED_CONVERTED_CONTENT_DEB822 + "# custom_sources_list\n", + id="nodeb822_and_deb822_sources_list_writes_sources_file", ), ), ) - def test_apt_v1_srcl_custom(self, cfg, apt_file, expected, tmpdir): + def test_apt_v1_srcl_custom( + self, deb822, cfg, apt_file, expected, mocker, tmpdir + ): """Test rendering from a custom source.list template""" + self.deb822 = mocker.patch.object( + cc_apt_configure.features, + "APT_DEB822_SOURCE_LIST_FILE", + deb822, + ) mycloud = get_cloud("ubuntu") - tmpl_file = "/etc/cloud/templates/sources.list.ubuntu.tmpl" - util.write_file(tmpl_file, EXAMPLE_TMPL) + if deb822: + tmpl_file = "/etc/cloud/templates/sources.list.ubuntu.deb822.tmpl" + tmpl_content = EXAMPLE_TMPL_DEB822 + else: + tmpl_content = EXAMPLE_TMPL + tmpl_file = "/etc/cloud/templates/sources.list.ubuntu.tmpl" + util.write_file(tmpl_file, tmpl_content) # the second mock restores the original subp cc_apt_configure.handle("notimportant", cfg, mycloud, None) diff --git a/tests/unittests/config/test_apt_configure_sources_list_v3.py b/tests/unittests/config/test_apt_configure_sources_list_v3.py index 0685154d0ed..c4f760636e2 100644 --- a/tests/unittests/config/test_apt_configure_sources_list_v3.py +++ b/tests/unittests/config/test_apt_configure_sources_list_v3.py @@ -23,6 +23,21 @@ deb {{security}} {{codename}}-security multiverse """ +EXAMPLE_TMPL_DEB822 = """\ +## template:jinja +# Generated by cloud-init +Types: deb deb-src +URIs: {{mirror}} +Suites: {{codename}} {{codename}}-updates +Components: main restricted + +# Security section +Types: deb deb-src +URIs: {{security}} +Suites: {{codename}}-security +Components: main restricted +""" + YAML_TEXT_CUSTOM_SL = """ apt: primary: @@ -49,6 +64,41 @@ deb http://testsec.ubuntu.com/ubuntu/ fakerel-security multiverse """ +YAML_TEXT_CUSTOM_SL_DEB822 = """ +apt: + primary: + - arches: [default] + uri: http://test.ubuntu.com/ubuntu/ + security: + - arches: [default] + uri: http://testsec.ubuntu.com/ubuntu/ + sources_list: | + + # custom deb822 sources_list deb822 + Types: deb deb-src + URIs: $PRIMARY + Suites: $RELEASE $RELEASE-updates + Components: main restricted + + Types: deb deb-src + URIs: $SECURITY + Suites: $RELEASE-security + Components: main restricted +""" + +EXPECTED_CONVERTED_CONTENT_DEB822 = """ +# custom deb822 sources_list deb822 +Types: deb deb-src +URIs: http://test.ubuntu.com/ubuntu/ +Suites: fakerel fakerel-updates +Components: main restricted + +Types: deb deb-src +URIs: http://testsec.ubuntu.com/ubuntu/ +Suites: fakerel-security +Components: main restricted +""" + # mocked to be independent to the unittest system EXPECTED_BASE_CONTENT = """\ @@ -65,6 +115,19 @@ deb http://test.ubuntu.com/ubuntu/ fakerel-security main restricted """ +EXPECTED_BASE_CONTENT_DEB822 = """\ +Types: deb deb-src +URIs: http://archive.ubuntu.com/ubuntu/ +Suites: fakerel fakerel-updates +Components: main restricted + +# Security section +Types: deb deb-src +URIs: http://security.ubuntu.com/ubuntu/ +Suites: fakerel-security +Components: main restricted +""" + EXPECTED_PRIMSEC_CONTENT = """\ deb http://test.ubuntu.com/ubuntu/ fakerel main restricted deb-src http://test.ubuntu.com/ubuntu/ fakerel main restricted @@ -72,13 +135,26 @@ deb http://testsec.ubuntu.com/ubuntu/ fakerel-security multiverse """ +EXPECTED_PRIMSEC_CONTENT_DEB822 = """# Generated by cloud-init +Types: deb deb-src +URIs: http://test.ubuntu.com/ubuntu/ +Suites: fakerel fakerel-updates +Components: main restricted + +# Security section +Types: deb deb-src +URIs: http://testsec.ubuntu.com/ubuntu/ +Suites: fakerel-security +Components: main restricted +""" + @pytest.mark.usefixtures("fake_filesystem") class TestAptSourceConfigSourceList: """TestAptSourceConfigSourceList - Class to test sources list rendering""" @pytest.fixture(autouse=True) - def setup(self, mocker): + def common_mocks(self, mocker): self.subp = mocker.patch.object( subp, "subp", @@ -89,6 +165,17 @@ def setup(self, mocker): m_arch = mocker.patch("cloudinit.util.get_dpkg_architecture") m_arch.return_value = "amd64" mocker.patch("cloudinit.config.cc_apt_configure._ensure_dependencies") + self.deb822 = mocker.patch.object( + cc_apt_configure.features, "APT_DEB822_SOURCE_LIST_FILE", True + ) + mocker.patch.object( + cc_apt_configure, + "get_apt_cfg", + return_value={ + "sourcelist": "/etc/apt/sources.list", + "sourceparts": "/etc/apt/sources.list.d/", + }, + ) @pytest.mark.parametrize( "distro,template_present", @@ -98,9 +185,9 @@ def test_apt_v3_empty_cfg_source_list_by_distro( self, distro, template_present, mocker, tmpdir ): """Template based on distro, empty config relies on mirror default.""" - template = f"/etc/cloud/templates/sources.list.{distro}.tmpl" + template = f"/etc/cloud/templates/sources.list.{distro}.deb822.tmpl" if template_present: - util.write_file(template, EXAMPLE_TMPL) + util.write_file(template, EXAMPLE_TMPL_DEB822) mycloud = get_cloud(distro) mock_shouldcfg = mocker.patch.object( @@ -110,9 +197,12 @@ def test_apt_v3_empty_cfg_source_list_by_distro( ) cc_apt_configure.handle("test", {"apt": {}}, mycloud, None) - sources_file = tmpdir.join("/etc/apt/sources.list") + sources_file = tmpdir.join(f"/etc/apt/sources.list.d/{distro}.sources") if template_present: - assert EXPECTED_BASE_CONTENT == sources_file.read() + assert ( + "# Generated by cloud-init\n" + EXPECTED_BASE_CONTENT_DEB822 + == sources_file.read() + ) assert 0o644 == stat.S_IMODE(sources_file.stat().mode) else: assert ( @@ -135,9 +225,17 @@ def test_apt_v3_source_list_ubuntu_snappy(self, mocker): assert 1 == mock_issnappy.call_count @pytest.mark.parametrize( - "tmpl_file,tmpl_content,apt_file,expected", + "deb822,tmpl_file,tmpl_content,apt_file,expected", ( ( + True, + "/etc/cloud/templates/sources.list.ubuntu.deb822.tmpl", + EXAMPLE_TMPL_DEB822, + "/etc/apt/sources.list.d/ubuntu.sources", + EXPECTED_PRIMSEC_CONTENT_DEB822, + ), + ( + False, "/etc/cloud/templates/sources.list.ubuntu.tmpl", EXAMPLE_TMPL, "/etc/apt/sources.list", @@ -145,10 +243,21 @@ def test_apt_v3_source_list_ubuntu_snappy(self, mocker): ), ), ) - def test_apt_v3_source_list_psm( - self, tmpl_file, tmpl_content, apt_file, expected, tmpdir + def test_apt_v3_source_list_psm_deb822_feature_aware( + self, + deb822, + tmpl_file, + tmpl_content, + apt_file, + expected, + tmpdir, + mocker, ): - """test_apt_v3_source_list_psm - Test specifying prim+sec mirrors""" + """test_apt_v3_source_list_psm - Test specifying prim+sec mirrors + + Assert APT_DEB822_SOURCE_LIST_FILE is taken into account when + determining which sources.list.tmpl source file to read. + """ pm = "http://test.ubuntu.com/ubuntu/" sm = "http://testsec.ubuntu.com/ubuntu/" cfg = { @@ -157,6 +266,10 @@ def test_apt_v3_source_list_psm( "security": [{"arches": ["default"], "uri": sm}], } + self.deb822 = mocker.patch.object( + cc_apt_configure.features, "APT_DEB822_SOURCE_LIST_FILE", deb822 + ) + util.write_file(tmpl_file, tmpl_content) mycloud = get_cloud("ubuntu") cc_apt_configure.handle("test", {"apt": cfg}, mycloud, None) @@ -166,20 +279,52 @@ def test_apt_v3_source_list_psm( assert 0o644 == stat.S_IMODE(sources_file.stat().mode) @pytest.mark.parametrize( - "cfg,apt_file,expected", + "deb822,cfg,apt_file,expected", ( pytest.param( + True, + util.load_yaml(YAML_TEXT_CUSTOM_SL_DEB822), + "/etc/apt/sources.list.d/debian.sources", + EXPECTED_CONVERTED_CONTENT_DEB822, + id="deb822_feature_deb822_sources_list_writes_deb822_source", + ), + pytest.param( + True, + util.load_yaml(YAML_TEXT_CUSTOM_SL), + "/etc/apt/sources.list", + EXPECTED_CONVERTED_CONTENT + "# FIND_SOMETHING_SPECIAL\n", + id="deb822_feature_nondeb822_sources_list_writes_list_file", + ), + pytest.param( + False, util.load_yaml(YAML_TEXT_CUSTOM_SL), "/etc/apt/sources.list", EXPECTED_CONVERTED_CONTENT + "# FIND_SOMETHING_SPECIAL\n", - id="sources_list_writes_list_file", + id="nodeb822_feature_nondeb822_sources_list_writes_list_file", + ), + pytest.param( + False, + util.load_yaml(YAML_TEXT_CUSTOM_SL_DEB822), + "/etc/apt/sources.list.d/debian.sources", + EXPECTED_CONVERTED_CONTENT_DEB822, + id="nodeb822_feature_deb822_sources_list_writes_deb822_source", ), ), ) - def test_apt_v3_srcl_custom(self, cfg, apt_file, expected, mocker, tmpdir): - """test_apt_v3_srcl_custom - Test rendering a custom source template""" + def test_apt_v3_srcl_custom_deb822_feature_aware( + self, deb822, cfg, apt_file, expected, mocker, tmpdir + ): + """test_apt_v3_srcl_custom - Test rendering a custom source template + + Also take into account deb822 feature flag to assert writing the + appropriate deb822 /etc/apt/sources.list.d/*list or *source file based + on content and deb822 feature flag. + """ mycloud = get_cloud("debian") + self.deb822 = mocker.patch.object( + cc_apt_configure.features, "APT_DEB822_SOURCE_LIST_FILE", deb822 + ) mocker.patch.object(Distro, "get_primary_arch", return_value="amd64") cc_apt_configure.handle("notimportant", cfg, mycloud, None) sources_file = tmpdir.join(apt_file) diff --git a/tests/unittests/config/test_apt_source_v3.py b/tests/unittests/config/test_apt_source_v3.py index ed9eab0b731..c8bb8815765 100644 --- a/tests/unittests/config/test_apt_source_v3.py +++ b/tests/unittests/config/test_apt_source_v3.py @@ -5,6 +5,7 @@ This tries to call all in the new v3 format and cares about new features """ import glob +import logging import os import pathlib import re @@ -16,6 +17,7 @@ from cloudinit import gpg, subp, util from cloudinit.config import cc_apt_configure +from tests.unittests.helpers import skipIfAptPkg from tests.unittests.util import get_cloud EXPECTEDKEY = """-----BEGIN PGP PUBLIC KEY BLOCK----- @@ -1439,3 +1441,228 @@ def test_dpkg_reconfigure_not_done_on_no_data(self, m_subp): def test_dpkg_reconfigure_not_done_if_no_cleaners(self, m_subp): cc_apt_configure.dpkg_reconfigure(["pkgfoo", "pkgbar"]) m_subp.assert_not_called() + + +DEB822_SINGLE_SUITE = """\ +Types: deb +URIs: https://ppa.launchpadcontent.net/cloud-init-dev/daily/ubuntu/ +Suites: mantic # Some comment +Components: main +""" + +DEB822_DISABLED_SINGLE_SUITE = """\ +## Entry disabled by cloud-init, due to disable_suites +# disabled by cloud-init: Types: deb +# disabled by cloud-init: URIs: https://ppa.launchpadcontent.net/cloud-init-dev/daily/ubuntu/ +# disabled by cloud-init: Suites: mantic # Some comment +# disabled by cloud-init: Components: main +""" + +DEB822_SINGLE_SECTION_TWO_SUITES = """\ +Types: deb +URIs: https://ppa.launchpadcontent.net/cloud-init-dev/daily/ubuntu/ +Suites: mantic mantic-updates +Components: main +""" + +DEB822_SINGLE_SECTION_TWO_SUITES_DISABLE_ONE = """\ +Types: deb +URIs: https://ppa.launchpadcontent.net/cloud-init-dev/daily/ubuntu/ +# cloud-init disable_suites redacted: Suites: mantic mantic-updates +Suites: mantic-updates +Components: main +""" + +DEB822_SUITE_2 = """ +# APT Suite 2 +Types: deb +URIs: https://ppa.launchpadcontent.net/cloud-init-dev/daily/ubuntu/ +Suites: mantic-backports +Components: main +""" + + +DEB822_DISABLED_SINGLE_SUITE = """\ +## Entry disabled by cloud-init, due to disable_suites +# disabled by cloud-init: Types: deb +# disabled by cloud-init: URIs: https://ppa.launchpadcontent.net/cloud-init-dev/daily/ubuntu/ +# disabled by cloud-init: Suites: mantic # Some comment +# disabled by cloud-init: Components: main +""" + +DEB822_DISABLED_MULTIPLE_SUITES = """\ +## Entry disabled by cloud-init, due to disable_suites +# disabled by cloud-init: Types: deb +# disabled by cloud-init: URIs: https://ppa.launchpadcontent.net/cloud-init-dev/daily/ubuntu/ +# disabled by cloud-init: Suites: mantic mantic-updates +# disabled by cloud-init: Components: main +""" + + +class TestDisableSuitesDeb822: + @pytest.mark.parametrize( + "disabled_suites,src,expected", + ( + pytest.param( + [], + DEB822_SINGLE_SUITE, + DEB822_SINGLE_SUITE, + id="empty_suites_nochange", + ), + pytest.param( + ["$RELEASE-updates"], + DEB822_SINGLE_SUITE, + DEB822_SINGLE_SUITE, + id="no_matching_suites_nochange", + ), + pytest.param( + ["$RELEASE"], + DEB822_SINGLE_SUITE, + DEB822_DISABLED_SINGLE_SUITE, + id="matching_all_suites_disables_whole_section", + ), + pytest.param( + ["$RELEASE"], + DEB822_SINGLE_SECTION_TWO_SUITES + DEB822_SUITE_2, + DEB822_SINGLE_SECTION_TWO_SUITES_DISABLE_ONE + + "\n" + + DEB822_SUITE_2, + id="matching_some_suites_redacts_matches_and_comments_orig", + ), + pytest.param( + ["$RELEASE", "$RELEASE-updates"], + DEB822_SINGLE_SECTION_TWO_SUITES + DEB822_SUITE_2, + DEB822_DISABLED_MULTIPLE_SUITES + "\n" + DEB822_SUITE_2, + id="matching_all_suites_disables_specific_section", + ), + ), + ) + def test_disable_deb822_suites_disables_proper_suites( + self, disabled_suites, src, expected + ): + assert expected == cc_apt_configure.disable_suites_deb822( + disabled_suites, src, "mantic" + ) + + +APT_CONFIG_DUMP = """ +APT ""; +Dir "/"; +Dir::Etc "etc/myapt"; +Dir::Etc::sourcelist "sources.my.list"; +Dir::Etc::sourceparts "sources.my.list.d"; +Dir::Etc::main "apt.conf"; +""" + + +class TestGetAptCfg: + @skipIfAptPkg() + @pytest.mark.parametrize( + "subp_side_effect,expected", + ( + pytest.param( + [(APT_CONFIG_DUMP, "")], + { + "sourcelist": "/etc/myapt/sources.my.list", + "sourceparts": "/etc/myapt/sources.my.list.d/", + }, + id="no_aptpkg_use_apt_config_cmd", + ), + pytest.param( + [("", "")], + { + "sourcelist": "/etc/apt/sources.list", + "sourceparts": "/etc/apt/sources.list.d/", + }, + id="no_aptpkg_unparsable_apt_config_cmd_defaults", + ), + pytest.param( + [ + subp.ProcessExecutionError( + "No such file or directory 'apt-config'" + ) + ], + { + "sourcelist": "/etc/apt/sources.list", + "sourceparts": "/etc/apt/sources.list.d/", + }, + id="no_aptpkg_no_apt_config_cmd_defaults", + ), + ), + ) + def test_use_defaults_or_apt_config_dump( + self, subp_side_effect, expected, mocker + ): + subp = mocker.patch("cloudinit.config.cc_apt_configure.subp.subp") + subp.side_effect = subp_side_effect + assert expected == cc_apt_configure.get_apt_cfg() + subp.assert_called_once_with(["apt-config", "dump"]) + + +class TestIsDeb822SourcesFormat: + @pytest.mark.parametrize( + "content,is_deb822,warnings", + ( + pytest.param( + "#Something\ndeb-src http://url lunar multiverse\n", + False, + [], + id="any_deb_src_is_not_deb822", + ), + pytest.param( + "#Something\ndeb http://url lunar multiverse\n", + False, + [], + id="any_deb_url_is_not_deb822", + ), + pytest.param( + "#Something\ndeb http://url lunar multiverse\nTypes: deb\n", + False, + [], + id="even_some_deb822_fields_not_deb822_if_any_deb_line", + ), + pytest.param( + "#Something\nTypes: deb\n", + True, + [], + id="types_deb822_keys_and_no_deb_or_deb_src_is_deb822", + ), + pytest.param( + "#Something\nURIs: http://url\n", + True, + [], + id="uris_deb822_keys_and_no_deb_or_deb_src_is_deb822", + ), + pytest.param( + "#Something\nSuites: http://url\n", + True, + [], + id="suites_deb822_keys_and_no_deb_deb_src_is_deb822", + ), + pytest.param( + "#Something\nComponents: http://url\n", + True, + [], + id="components_deb822_keys_and_no_deb_deb_src_is_deb822", + ), + pytest.param( + "#Something neither deb/deb-src nor deb822\n", + False, + [ + "apt.sources_list value does not match either deb822" + " source keys or deb/deb-src list keys. Assuming APT" + " deb/deb-src list format." + ], + id="neither_deb822_keys_nor_deb_deb_src_warn_and_not_deb822", + ), + ), + ) + def test_is_deb822_format_prefers_non_deb822( + self, content, is_deb822, warnings, caplog + ): + with caplog.at_level(logging.WARNING): + assert is_deb822 is cc_apt_configure.is_deb822_sources_format( + content + ) + for warning in warnings: + assert warning in caplog.text diff --git a/tests/unittests/helpers.py b/tests/unittests/helpers.py index 628c88da9b0..8bd7fd03d70 100644 --- a/tests/unittests/helpers.py +++ b/tests/unittests/helpers.py @@ -39,6 +39,14 @@ skipIf = unittest.skipIf +try: + import apt_pkg # type: ignore # noqa: F401 + + HAS_APT_PKG = True +except ImportError: + HAS_APT_PKG = False + + # Makes the old path start # with new base instead of whatever # it previously had @@ -519,6 +527,13 @@ def readResource(name, mode="r"): return fh.read() +def skipIfAptPkg(): + return skipIf( + HAS_APT_PKG, + "No python-apt dependency present.", + ) + + try: import jsonschema diff --git a/tests/unittests/test_data.py b/tests/unittests/test_data.py index dc1c7793ff5..464c0c12073 100644 --- a/tests/unittests/test_data.py +++ b/tests/unittests/test_data.py @@ -475,6 +475,7 @@ def test_mime_text_plain(self, init_tmp, caplog): EXPIRE_APPLIES_TO_HASHED_USERS=False, NETPLAN_CONFIG_ROOT_READ_ONLY=True, NOCLOUD_SEED_URL_APPEND_FORWARD_SLASH=False, + APT_DEB822_SOURCE_LIST_FILE=True, ) def test_shellscript(self, init_tmp, tmpdir, caplog): """Raw text starting #!/bin/sh is treated as script.""" @@ -505,6 +506,7 @@ def test_shellscript(self, init_tmp, tmpdir, caplog): "EXPIRE_APPLIES_TO_HASHED_USERS": False, "NETPLAN_CONFIG_ROOT_READ_ONLY": True, "NOCLOUD_SEED_URL_APPEND_FORWARD_SLASH": False, + "APT_DEB822_SOURCE_LIST_FILE": True, }, "system_info": { "default_user": {"name": "ubuntu"}, diff --git a/tests/unittests/test_features.py b/tests/unittests/test_features.py index 1be712b0d9c..c9eff407064 100644 --- a/tests/unittests/test_features.py +++ b/tests/unittests/test_features.py @@ -20,6 +20,7 @@ def test_feature_without_override(self): EXPIRE_APPLIES_TO_HASHED_USERS=False, NETPLAN_CONFIG_ROOT_READ_ONLY=True, NOCLOUD_SEED_URL_APPEND_FORWARD_SLASH=False, + APT_DEB822_SOURCE_LIST_FILE=True, ): assert { "ERROR_ON_USER_DATA_FAILURE": True, @@ -27,4 +28,5 @@ def test_feature_without_override(self): "EXPIRE_APPLIES_TO_HASHED_USERS": False, "NETPLAN_CONFIG_ROOT_READ_ONLY": True, "NOCLOUD_SEED_URL_APPEND_FORWARD_SLASH": False, + "APT_DEB822_SOURCE_LIST_FILE": True, } == features.get_features()