Ubuntu/devel snapshot release #2

cloudinit/cmd/devel/render.py

@@ -12,6 +12,7 @@
 from cloudinit.cmd.devel import read_cfg_paths
 from cloudinit.handlers.jinja_template import (
     JinjaLoadError,
+    JinjaSyntaxParsingException,
     NotJinjaError,
     render_jinja_payload_from_file,
 )
@@ -99,6 +100,13 @@ def render_template(user_data_path, instance_data_path=None, debug=False):
             "Cannot render from instance data due to exception: %s", repr(e)
         )
         return 1
+    except JinjaSyntaxParsingException as e:
+        LOG.error(
+            "Failed to render templated user-data file '%s'. %s",
+            user_data_path,
+            str(e),
+        )
+        return 1
     if not rendered_payload:
         LOG.error("Unable to render user-data file: %s", user_data_path)
         return 1

cloudinit/cmd/query.py

@@ -29,6 +29,7 @@
     render_jinja_payload,
 )
 from cloudinit.sources import REDACT_SENSITIVE_VALUE
+from cloudinit.templater import JinjaSyntaxParsingException
 
 NAME = "query"
 LOG = logging.getLogger(__name__)
@@ -277,12 +278,19 @@ def handle_args(name, args):
         return 1
     if args.format:
         payload = "## template: jinja\n{fmt}".format(fmt=args.format)
-        rendered_payload = render_jinja_payload(
-            payload=payload,
-            payload_fn="query commandline",
-            instance_data=instance_data,
-            debug=True if args.debug else False,
-        )
+        try:
+            rendered_payload = render_jinja_payload(
+                payload=payload,
+                payload_fn="query commandline",
+                instance_data=instance_data,
+                debug=True if args.debug else False,
+            )
+        except JinjaSyntaxParsingException as e:
+            LOG.error(
+                "Failed to render templated data. %s",
+                str(e),
+            )
+            return 1
         if rendered_payload:
             print(rendered_payload)
             return 0

cloudinit/config/cc_apt_configure.py

@@ -604,7 +604,6 @@ def get_apt_cfg() -> Dict[str, str]:
             "Dir::Etc::sourceparts", DEFAULT_APT_CFG["Dir::Etc::sourceparts"]
         )
     except ImportError:
-
         try:
             apt_dump, _ = subp.subp(["apt-config", "dump"])
         except subp.ProcessExecutionError:
@@ -680,7 +679,20 @@ def generate_sources_list(cfg, release, mirrors, cloud):
             )
             aptsrc_file = apt_sources_list
     disabled = disable_suites(cfg.get("disable_suites"), rendered, release)
+    disable_apt_sources_list = False
+    if aptsrc_file == apt_sources_deb822 and os.path.exists(apt_sources_list):
+        disable_apt_sources_list = True
     util.write_file(aptsrc_file, disabled, mode=0o644)
+    if disable_apt_sources_list:
+        LOG.warning(
+            "Disabling %s to favor deb822 source format", apt_sources_list
+        )
+        content = pathlib.Path(apt_sources_list).read_bytes()
+        content = b"# disabled by cloud-init\n" + content
+        util.rename(apt_sources_list, f"{apt_sources_list}.disabled")
+        util.write_file(
+            f"{apt_sources_list}.disabled", content, preserve_mode=True
+        )
 
 
 def add_apt_key_raw(key, file_name, hardened=False):

cloudinit/config/cc_final_message.py

@@ -96,6 +96,10 @@ def handle(name: str, cfg: Config, cloud: Cloud, args: list) -> None:
             stderr=True,
             log=LOG,
         )
+    except templater.JinjaSyntaxParsingException as e:
+        util.logexc(
+            LOG, "Failed to render templated final message: %s", str(e)
+        )
     except Exception:
         util.logexc(LOG, "Failed to render final message template")
 

cloudinit/config/schema.py

@@ -816,9 +816,12 @@ def _get_config_type_and_rendered_userdata(
     :return: UserDataTypeAndDecodedContent
     :raises: SchemaValidationError when non-jinja content found but
         header declared ## template: jinja.
+    :raises JinjaSyntaxParsingException when jinja syntax error found.
+    :raises JinjaLoadError when jinja template fails to load.
     """
     from cloudinit.handlers.jinja_template import (
         JinjaLoadError,
+        JinjaSyntaxParsingException,
         NotJinjaError,
         render_jinja_payload_from_file,
     )
@@ -840,6 +843,11 @@ def _get_config_type_and_rendered_userdata(
                     )
                 ]
             ) from e
+        except JinjaSyntaxParsingException as e:
+            error(
+                "Failed to render templated user-data. " + str(e),
+                sys_exit=True,
+            )
         except JinjaLoadError as e:
             error(str(e), sys_exit=True)
         schema_position = "format-l2.c1"

cloudinit/distros/alpine.py

@@ -86,6 +86,9 @@ def _write_hostname(self, hostname, filename):
             if create_hostname_file:
                 pass
             else:
+                LOG.info(
+                    "create_hostname_file is False; hostname file not created"
+                )
                 return
         if not conf:
             conf = HostnameConf("")

cloudinit/distros/arch.py

@@ -124,6 +124,9 @@ def _write_hostname(self, hostname, filename):
             if create_hostname_file:
                 pass
             else:
+                LOG.info(
+                    "create_hostname_file is False; hostname file not created"
+                )
                 return
         if not conf:
             conf = HostnameConf("")

cloudinit/distros/debian.py

@@ -136,6 +136,9 @@ def _write_hostname(self, hostname, filename):
             if create_hostname_file:
                 pass
             else:
+                LOG.info(
+                    "create_hostname_file is False; hostname file not created"
+                )
                 return
         if not conf:
             conf = HostnameConf("")

cloudinit/distros/gentoo.py

@@ -189,6 +189,9 @@ def _write_hostname(self, hostname, filename):
             if create_hostname_file:
                 pass
             else:
+                LOG.info(
+                    "create_hostname_file is False; hostname file not created"
+                )
                 return
         if not conf:
             conf = HostnameConf("")

cloudinit/distros/opensuse.py

@@ -248,6 +248,10 @@ def _write_hostname(self, hostname, filename):
                 if create_hostname_file:
                     pass
                 else:
+                    LOG.info(
+                        "create_hostname_file is False; hostname file not"
+                        "created"
+                    )
                     return
             if not conf:
                 conf = HostnameConf("")

cloudinit/distros/photon.py

@@ -108,6 +108,9 @@ def _write_hostname(self, hostname, filename):
                         str(hostname),
                     ]
                 )
+                LOG.info(
+                    "create_hostname_file is False; hostname set transiently"
+                )
             if ret:
                 LOG.warning(
                     (

cloudinit/distros/rhel.py

@@ -121,6 +121,9 @@ def _write_hostname(self, hostname, filename):
                         str(hostname),
                     ]
                 )
+                LOG.info(
+                    "create_hostname_file is False; hostname set transiently"
+                )
         else:
             host_cfg = {
                 "HOSTNAME": hostname,

cloudinit/handlers/jinja_template.py

@@ -13,6 +13,7 @@
 from cloudinit.settings import PER_ALWAYS
 from cloudinit.templater import (
     MISSING_JINJA_PREFIX,
+    JinjaSyntaxParsingException,
     detect_template,
     render_string,
 )
@@ -54,9 +55,19 @@ def handle_part(self, data, ctype, filename, payload, frequency, headers):
         if ctype in handlers.CONTENT_SIGNALS:
             return
         jinja_json_file = self.paths.get_runpath("instance_data_sensitive")
-        rendered_payload = render_jinja_payload_from_file(
-            payload, filename, jinja_json_file
-        )
+        try:
+            rendered_payload = render_jinja_payload_from_file(
+                payload, filename, jinja_json_file
+            )
+        except JinjaSyntaxParsingException as e:
+            LOG.warning(
+                "Ignoring jinja template for %s. "
+                "Failed to render template. %s",
+                filename,
+                str(e),
+            )
+            return
+
         if not rendered_payload:
             return
         subtype = handlers.type_from_starts_with(rendered_payload)

cloudinit/net/__init__.py

@@ -1287,6 +1287,8 @@ def subnet_is_ipv6(subnet) -> bool:
     """Common helper for checking network_state subnets for ipv6."""
     # 'static6', 'dhcp6', 'ipv6_dhcpv6-stateful', 'ipv6_dhcpv6-stateless' or
     # 'ipv6_slaac'
+    # This function is inappropriate for v2-based routes as routes defined
+    # under v2 subnets can contain ipv4 and ipv6 simultaneously
     if subnet["type"].endswith("6") or subnet["type"] in IPV6_DYNAMIC_TYPES:
         # This is a request either static6 type or DHCPv6.
         return True

cloudinit/net/network_manager.py

@@ -12,10 +12,15 @@
 import logging
 import os
 import uuid
-from typing import Optional
+from typing import List, Optional
 
 from cloudinit import subp, util
-from cloudinit.net import is_ipv6_address, renderer, subnet_is_ipv6
+from cloudinit.net import (
+    is_ipv6_address,
+    is_ipv6_network,
+    renderer,
+    subnet_is_ipv6,
+)
 from cloudinit.net.network_state import NetworkState
 from cloudinit.net.sysconfig import available_nm_ifcfg_rh
 
@@ -158,11 +163,11 @@ def _set_ip_method(self, family, subnet_type):
         if self.config[family]["method"] == "auto" and method == "manual":
             return
 
-        if (
-            subnet_type == "ipv6_dhcpv6-stateful"
-            or subnet_type == "ipv6_dhcpv6-stateless"
-            or subnet_type == "ipv6_slaac"
-        ):
+        if subnet_type in [
+            "ipv6_dhcpv6-stateful",
+            "ipv6_dhcpv6-stateless",
+            "ipv6_slaac",
+        ]:
             # set ipv4 method to 'disabled' to align with sysconfig renderer.
             self._set_default("ipv4", "method", "disabled")
 
@@ -174,7 +179,8 @@ def _add_numbered(self, section, key_prefix, value):
         Adds a numbered property, such as address<n> or route<n>, ensuring
         the appropriate value gets used for <n>.
         """
-
+        if not self.config.has_section(section):
+            self.config[section] = {}
         for index in itertools.count(1):
             key = f"{key_prefix}{index}"
             if not self.config.has_option(section, key):
@@ -189,40 +195,37 @@ def _add_address(self, family, subnet):
         value = subnet["address"] + "/" + str(subnet["prefix"])
         self._add_numbered(family, "address", value)
 
-    def _add_route(self, family, route):
-        """
-        Adds a ipv[46].route<n> property.
-        """
-
+    def _add_route(self, route):
+        """Adds a ipv[46].route<n> property."""
+        # Because network v2 route definitions can have mixed v4 and v6
+        # routes, determine the family per route based on the gateway
+        family = "ipv6" if is_ipv6_network(route["gateway"]) else "ipv4"
         value = route["network"] + "/" + str(route["prefix"])
         if "gateway" in route:
             value = value + "," + route["gateway"]
         self._add_numbered(family, "route", value)
 
-    def _add_nameserver(self, dns):
+    def _add_nameserver(self, dns: str) -> None:
         """
         Extends the ipv[46].dns property with a name server.
         """
-
-        # FIXME: the subnet contains IPv4 and IPv6 name server mixed
-        # together. We might be getting an IPv6 name server while
-        # we're dealing with an IPv4 subnet. Sort this out by figuring
-        # out the correct family and making sure a valid section exist.
         family = "ipv6" if is_ipv6_address(dns) else "ipv4"
-        self._set_default(family, "method", "disabled")
-
-        self._set_default(family, "dns", "")
-        self.config[family]["dns"] = self.config[family]["dns"] + dns + ";"
+        if self.config.has_section(family):
+            self._set_default(family, "dns", "")
+            self.config[family]["dns"] = self.config[family]["dns"] + dns + ";"
 
-    def _add_dns_search(self, family, dns_search):
+    def _add_dns_search(self, dns_search: List[str]) -> None:
         """
         Extends the ipv[46].dns-search property with a name server.
         """
-
-        self._set_default(family, "dns-search", "")
-        self.config[family]["dns-search"] = (
-            self.config[family]["dns-search"] + ";".join(dns_search) + ";"
-        )
+        for family in ["ipv4", "ipv6"]:
+            if self.config.has_section(family):
+                self._set_default(family, "dns-search", "")
+                self.config[family]["dns-search"] = (
+                    self.config[family]["dns-search"]
+                    + ";".join(dns_search)
+                    + ";"
+                )
 
     def con_uuid(self):
         """
@@ -304,8 +307,11 @@ def render_interface(self, iface, renderer):
 
         device_mtu = iface["mtu"]
         ipv4_mtu = None
+        found_nameservers = []
+        found_dns_search = []
 
         # Deal with Layer 3 configuration
+        use_top_level_dns = "dns" in iface
         for subnet in iface["subnets"]:
             family = "ipv6" if subnet_is_ipv6(subnet) else "ipv4"
 
@@ -315,15 +321,28 @@ def render_interface(self, iface, renderer):
             if "gateway" in subnet:
                 self.config[family]["gateway"] = subnet["gateway"]
             for route in subnet["routes"]:
-                self._add_route(family, route)
-            if "dns_nameservers" in subnet:
+                self._add_route(route)
+            if not use_top_level_dns and "dns_nameservers" in subnet:
                 for nameserver in subnet["dns_nameservers"]:
-                    self._add_nameserver(nameserver)
-            if "dns_search" in subnet:
-                self._add_dns_search(family, subnet["dns_search"])
+                    found_nameservers.append(nameserver)
+            if not use_top_level_dns and "dns_search" in subnet:
+                found_dns_search.append(subnet["dns_search"])
             if family == "ipv4" and "mtu" in subnet:
                 ipv4_mtu = subnet["mtu"]
 
+        # Now add our DNS search domains. We add them later because we
+        # only want them if an IP family has already been defined
+        if use_top_level_dns:
+            for nameserver in iface["dns"]["nameservers"]:
+                self._add_nameserver(nameserver)
+            if iface["dns"]["search"]:
+                self._add_dns_search(iface["dns"]["search"])
+        else:
+            for nameserver in found_nameservers:
+                self._add_nameserver(nameserver)
+            for dns_search in found_dns_search:
+                self._add_dns_search(dns_search)
+
         # we do not want to set may-fail to false for both ipv4 and ipv6 dhcp
         # at the at the same time. This will make the network configuration
         # work only when both ipv4 and ipv6 dhcp succeeds. This may not be

cloudinit/sources/azure/imds.py

@@ -104,7 +104,7 @@ def exception_callback(self, req_args, exception) -> bool:
             report_diagnostic_event(
                 "Polling IMDS failed attempt %d with exception: %r"
                 % (self._request_count, exception),
-                logger_func=LOG.info,
+                logger_func=LOG.warning,
             )
         return retry