Merge branch 'canonical:main' into cloudcix_ds

.gitignore

@@ -31,7 +31,7 @@ cloud-init_*.buildinfo
 cloud-init_*.changes
 cloud-init_*.deb
 cloud-init_*.dsc
-cloud-init_*.orig.tar.gz
+cloud-init_*.tar.gz
 cloud-init_*.tar.xz
 cloud-init_*.upload
 

.pylintrc

@@ -7,7 +7,6 @@ jobs=4
 [MESSAGES CONTROL]
 
 # Errors and warnings with some filtered:
-# W0201(attribute-defined-outside-init)
 # W0212(protected-access)
 # W0221(arguments-differ)
 # W0222(signature-differs)
@@ -27,7 +26,7 @@ jobs=4
 # W1514(unspecified-encoding)
 # E0012(bad-option-value)
 
-disable=C, F, I, R, W0201, W0212, W0221, W0222, W0223, W0231, W0311, W0511, W0602, W0603, W0611, W0613, W0621, W0622, W0631, W0703, W1401, W1514, E0012
+disable=C, F, I, R, W0212, W0221, W0222, W0223, W0231, W0311, W0511, W0602, W0603, W0611, W0613, W0621, W0622, W0631, W0703, W1401, W1514, E0012
 
 
 [REPORTS]
@@ -66,4 +65,3 @@ ignored-classes=argparse.Namespace,optparse.Values,thread._local,ImageManager,Co
 # system, and so shouldn't trigger E1101 when accessed. Python regular
 # expressions are accepted.
 generated-members=types,http.client,command_handlers,m_.*,enter_context
-

ChangeLog

@@ -1,3 +1,28 @@
+24.1.2
+ - test: Don't assume ordering of ThreadPoolExecutor submissions (#5052)
+ - refactor(ec2): simplify convert_ec2_metadata_network_config
+ - tests: drop CiTestCase and convert to pytest
+ - bug(tests): mock reads of host's /sys/class/net via get_sys_class_path
+ - fix: Fix breaking changes in package install (#5069)
+ - fix: Undeprecate 'network' in schema route definition (#5072)
+ - fix(ec2): fix ipv6 policy routing
+ - fix: document and add 'accept-ra' to network schema (#5060)
+ - bug(maas): register the correct DatasourceMAASLocal in init-local
+   (#5068) (LP: #2057763)
+
+24.1.1
+ - fix: Include DataSourceCloudStack attribute in unpickle test (#5039)
+ - bug(vmware): initialize new DataSourceVMware attributes at unpickle (#5021)
+ - fix(apt): Don't warn on apt 822 source format (#5028)
+ - fix: Add "broadcast" to network v1 schema (#5034)
+ - pro: honor but warn on custom ubuntu_advantage in /etc/cloud/cloud.cfg (#5030)
+ - net/dhcp: handle timeouts for dhcpcd (#5022)
+ - fix: Make wait_for_url respect explicit arguments
+ - bug(wait_for_url): when exceptions occur url is unset, use url_exc
+ - test: Fix scaleway retry assumptions
+ - fix: Make DataSourceOracle more resilient to early network issues (#5025)
+ - tests: Fix wsl test (#5008)
+
 24.1
  - fix: Don't warn on vendor directory (#4986)
  - apt: kill spawned keyboxd after gpg cmd interaction

cloudinit/atomic_helper.py

@@ -7,6 +7,8 @@
 import tempfile
 from base64 import b64decode, b64encode
 
+from cloudinit import util
+
 _DEF_PERMS = 0o644
 LOG = logging.getLogger(__name__)
 
@@ -43,9 +45,9 @@ def write_file(
 
     tf = None
     try:
-        tf = tempfile.NamedTemporaryFile(
-            dir=os.path.dirname(filename), delete=False, mode=omode
-        )
+        dirname = os.path.dirname(filename)
+        util.ensure_dir(dirname)
+        tf = tempfile.NamedTemporaryFile(dir=dirname, delete=False, mode=omode)
         LOG.debug(
             "Atomically writing to file %s (via temporary file %s) - %s: [%o]"
             " %d bytes/chars",

cloudinit/cmd/main.py

@@ -35,14 +35,7 @@
 from cloudinit.config import cc_set_hostname
 from cloudinit.config.modules import Modules
 from cloudinit.config.schema import validate_cloudconfig_schema
-from cloudinit.log import (
-    LogExporter,
-    setup_basic_logging,
-    setup_logging,
-    reset_logging,
-    configure_root_logger,
-    DEPRECATED,
-)
+from cloudinit import log
 from cloudinit.reporting import events
 from cloudinit.safeyaml import load
 from cloudinit.settings import PER_INSTANCE, PER_ALWAYS, PER_ONCE, CLOUD_CONFIG
@@ -223,7 +216,7 @@ def attempt_cmdline_url(path, network=True, cmdline=None) -> Tuple[int, str]:
             if is_cloud_cfg:
                 if cmdline_name == "url":
                     return (
-                        DEPRECATED,
+                        log.DEPRECATED,
                         str(
                             util.deprecate(
                                 deprecated="The kernel command line key `url`",
@@ -348,8 +341,8 @@ def main_init(name, args):
         LOG.debug(
             "Logging being reset, this logger may no longer be active shortly"
         )
-        reset_logging()
-    setup_logging(init.cfg)
+        log.reset_logging()
+    log.setup_logging(init.cfg)
     apply_reporting_cfg(init.cfg)
 
     # Any log usage prior to setup_logging above did not have local user log
@@ -510,7 +503,7 @@ def main_init(name, args):
             (outfmt, errfmt) = util.fixup_output(mods.cfg, name)
     except Exception:
         util.logexc(LOG, "Failed to re-adjust output redirection!")
-    setup_logging(mods.cfg)
+    log.setup_logging(mods.cfg)
 
     # give the activated datasource a chance to adjust
     init.activate_datasource()
@@ -615,13 +608,20 @@ def main_modules(action_name, args):
         LOG.debug(
             "Logging being reset, this logger may no longer be active shortly"
         )
-        reset_logging()
-    setup_logging(mods.cfg)
+        log.reset_logging()
+    log.setup_logging(mods.cfg)
     apply_reporting_cfg(init.cfg)
 
     # now that logging is setup and stdout redirected, send welcome
     welcome(name, msg=w_msg)
 
+    if name == "init":
+        util.deprecate(
+            deprecated="`--mode init`",
+            deprecated_version="24.1",
+            extra_message="Use `cloud-init init` instead.",
+        )
+
     # Stage 5
     return run_module_section(mods, name, name)
 
@@ -677,8 +677,8 @@ def main_single(name, args):
         LOG.debug(
             "Logging being reset, this logger may no longer be active shortly"
         )
-        reset_logging()
-    setup_logging(mods.cfg)
+        log.reset_logging()
+    log.setup_logging(mods.cfg)
     apply_reporting_cfg(init.cfg)
 
     # now that logging is setup and stdout redirected, send welcome
@@ -768,7 +768,7 @@ def status_wrapper(name, args, data_d=None, link_d=None):
     v1["stage"] = mode
     v1[mode]["start"] = time.time()
     v1[mode]["recoverable_errors"] = next(
-        filter(lambda h: isinstance(h, LogExporter), root_logger.handlers)
+        filter(lambda h: isinstance(h, log.LogExporter), root_logger.handlers)
     ).export_logs()
 
     # Write status.json prior to running init / module code
@@ -798,7 +798,7 @@ def status_wrapper(name, args, data_d=None, link_d=None):
 
     # Write status.json after running init / module code
     v1[mode]["recoverable_errors"] = next(
-        filter(lambda h: isinstance(h, LogExporter), root_logger.handlers)
+        filter(lambda h: isinstance(h, log.LogExporter), root_logger.handlers)
     ).export_logs()
     atomic_helper.write_json(status_path, status)
 
@@ -856,7 +856,7 @@ def main_features(name, args):
 
 
 def main(sysv_args=None):
-    configure_root_logger()
+    log.configure_root_logger()
     if not sysv_args:
         sysv_args = sys.argv
     parser = argparse.ArgumentParser(prog=sysv_args.pop(0))
@@ -918,11 +918,20 @@ def main(sysv_args=None):
     parser_mod = subparsers.add_parser(
         "modules", help="Activate modules using a given configuration key."
     )
+    extra_help = util.deprecate(
+        deprecated="`init`",
+        deprecated_version="24.1",
+        extra_message="Use `cloud-init init` instead.",
+        return_log=True,
+    )
     parser_mod.add_argument(
         "--mode",
         "-m",
         action="store",
-        help="Module configuration name to use (default: %(default)s).",
+        help=(
+            f"Module configuration name to use (default: %(default)s)."
+            f" {extra_help}"
+        ),
         default="config",
         choices=("init", "config", "final"),
     )
@@ -1080,9 +1089,11 @@ def main(sysv_args=None):
     #   - if --debug is passed, logging.DEBUG
     #   - if --debug is not passed, logging.WARNING
     if name not in ("init", "modules"):
-        setup_basic_logging(logging.DEBUG if args.debug else logging.WARNING)
+        log.setup_basic_logging(
+            logging.DEBUG if args.debug else logging.WARNING
+        )
     elif args.debug:
-        setup_basic_logging()
+        log.setup_basic_logging()
 
     # Setup signal handlers before running
     signal_handler.attach_handlers()
@@ -1132,6 +1143,12 @@ def main(sysv_args=None):
             args=(name, args),
         )
     reporting.flush_events()
+
+    # handle return code for main_modules, as it is not wrapped by
+    # status_wrapped when mode == init
+    if "modules" == name and "init" == args.mode:
+        retval = len(retval)
+
     return retval
 
 

cloudinit/config/cc_ansible.py

@@ -9,12 +9,12 @@
 from textwrap import dedent
 from typing import Optional
 
+from cloudinit import subp
 from cloudinit.cloud import Cloud
 from cloudinit.config import Config
 from cloudinit.config.schema import MetaSchema, get_meta_doc
 from cloudinit.distros import ALL_DISTROS, Distro
 from cloudinit.settings import PER_INSTANCE
-from cloudinit.subp import subp, which
 from cloudinit.util import Version, get_cfg_by_path
 
 meta: MetaSchema = {
@@ -100,7 +100,7 @@ def do_as(self, command: list, **kwargs):
         return self.distro.do_as(command, self.run_user, **kwargs)
 
     def subp(self, command, **kwargs):
-        return subp(command, update_env=self.env, **kwargs)
+        return subp.subp(command, update_env=self.env, **kwargs)
 
     @abc.abstractmethod
     def is_installed(self):
@@ -165,7 +165,7 @@ def install(self, pkg_name: str):
             self.distro.install_packages([pkg_name])
 
     def is_installed(self) -> bool:
-        return bool(which("ansible"))
+        return bool(subp.which("ansible"))
 
 
 def handle(name: str, cfg: Config, cloud: Cloud, args: list) -> None: