diff --git a/cloudinit/sources/DataSourceAliYun.py b/cloudinit/sources/DataSourceAliYun.py index 8db89e467cdd..727477df4621 100644 --- a/cloudinit/sources/DataSourceAliYun.py +++ b/cloudinit/sources/DataSourceAliYun.py @@ -7,7 +7,7 @@ from cloudinit import dmi, sources from cloudinit.event import EventScope, EventType from cloudinit.sources import DataSourceEc2 as EC2 -from cloudinit.sources import DataSourceHostname +from cloudinit.sources import DataSourceHostname, NicOrder LOG = logging.getLogger(__name__) @@ -32,6 +32,11 @@ def __init__(self, sys_cfg, distro, paths): super(DataSourceAliYun, self).__init__(sys_cfg, distro, paths) self.default_update_events = copy.deepcopy(self.default_update_events) self.default_update_events[EventScope.NETWORK].add(EventType.BOOT) + self._fallback_nic_order = NicOrder.NIC_NAME + + def _unpickle(self, ci_pkl_version: int) -> None: + super()._unpickle(ci_pkl_version) + self._fallback_nic_order = NicOrder.NIC_NAME def get_hostname(self, fqdn=False, resolve_ip=False, metadata_only=False): hostname = self.metadata.get("hostname") diff --git a/cloudinit/sources/DataSourceEc2.py b/cloudinit/sources/DataSourceEc2.py index 06918fff0dd9..49434c67464d 100644 --- a/cloudinit/sources/DataSourceEc2.py +++ b/cloudinit/sources/DataSourceEc2.py @@ -21,6 +21,7 @@ from cloudinit.net import activators from cloudinit.net.dhcp import NoDHCPLeaseError from cloudinit.net.ephemeral import EphemeralIPNetwork +from cloudinit.sources import NicOrder from cloudinit.sources.helpers import ec2 LOG = logging.getLogger(__name__) @@ -117,10 +118,12 @@ def __init__(self, sys_cfg, distro, paths): super(DataSourceEc2, self).__init__(sys_cfg, distro, paths) self.metadata_address = None self.identity = None + self._fallback_nic_order = NicOrder.MAC def _unpickle(self, ci_pkl_version: int) -> None: super()._unpickle(ci_pkl_version) self.extra_hotplug_udev_rules = _EXTRA_HOTPLUG_UDEV_RULES + self._fallback_nic_order = NicOrder.MAC def _get_cloud_name(self): """Return the cloud name as identified during _get_data.""" @@ -531,6 +534,7 @@ def network_config(self): full_network_config=util.get_cfg_option_bool( self.ds_cfg, "apply_full_imds_network_config", True ), + fallback_nic_order=self._fallback_nic_order, ) # Non-VPC (aka Classic) Ec2 instances need to rewrite the @@ -891,10 +895,12 @@ def _collect_platform_data(): def _build_nic_order( - macs_metadata: Dict[str, Dict], macs: List[str] + macs_metadata: Dict[str, Dict], + macs_to_nics: Dict[str, str], + fallback_nic_order: NicOrder = NicOrder.MAC, ) -> Dict[str, int]: """ - Builds a dictionary containing macs as keys nad nic orders as values, + Builds a dictionary containing macs as keys and nic orders as values, taking into account `network-card` and `device-number` if present. Note that the first NIC will be the primary NIC as it will be the one with @@ -902,19 +908,22 @@ def _build_nic_order( @param macs_metadata: dictionary with mac address as key and contents like: {"device-number": "0", "interface-id": "...", "local-ipv4s": ...} - @macs: list of macs to consider + @macs_to_nics: dictionary with mac address as key and nic name as value @return: Dictionary with macs as keys and nic orders as values. """ nic_order: Dict[str, int] = {} - if len(macs) == 0 or len(macs_metadata) == 0: + if len(macs_to_nics) == 0 or len(macs_metadata) == 0: return nic_order valid_macs_metadata = filter( # filter out nics without metadata (not a physical nic) lambda mmd: mmd[1] is not None, # filter by macs - map(lambda mac: (mac, macs_metadata.get(mac)), macs), + map( + lambda mac: (mac, macs_metadata.get(mac), macs_to_nics[mac]), + macs_to_nics.keys(), + ), ) def _get_key_as_int_or(dikt, key, alt_value): @@ -931,7 +940,7 @@ def _get_key_as_int_or(dikt, key, alt_value): # function. return { mac: i - for i, (mac, _mac_metadata) in enumerate( + for i, (mac, _mac_metadata, _nic_name) in enumerate( sorted( valid_macs_metadata, key=lambda mmd: ( @@ -941,6 +950,9 @@ def _get_key_as_int_or(dikt, key, alt_value): _get_key_as_int_or( mmd[1], "device-number", float("infinity") ), + mmd[2] + if fallback_nic_order == NicOrder.NIC_NAME + else mmd[0], ), ) ) @@ -953,6 +965,7 @@ def convert_ec2_metadata_network_config( macs_to_nics=None, fallback_nic=None, full_network_config=True, + fallback_nic_order=NicOrder.MAC, ): """Convert ec2 metadata to network config version 2 data dict. @@ -995,8 +1008,10 @@ def convert_ec2_metadata_network_config( return netcfg # Apply network config for all nics and any secondary IPv4/v6 addresses is_netplan = distro.network_activator == activators.NetplanActivator + nic_order = _build_nic_order( + macs_metadata, macs_to_nics, fallback_nic_order + ) macs = sorted(macs_to_nics.keys()) - nic_order = _build_nic_order(macs_metadata, macs) for mac in macs: nic_name = macs_to_nics[mac] nic_metadata = macs_metadata.get(mac) diff --git a/cloudinit/sources/__init__.py b/cloudinit/sources/__init__.py index 65222b29b372..a6d46c6c4b1d 100644 --- a/cloudinit/sources/__init__.py +++ b/cloudinit/sources/__init__.py @@ -83,6 +83,16 @@ def __str__(self) -> str: return self.value +class NicOrder(Enum): + """Represents ways to sort NICs""" + + MAC = "mac" + NIC_NAME = "nic_name" + + def __str__(self) -> str: + return self.value + + class DatasourceUnpickleUserDataError(Exception): """Raised when userdata is unable to be unpickled due to python upgrades""" diff --git a/tests/unittests/sources/test_ec2.py b/tests/unittests/sources/test_ec2.py index d7d5581ead7c..fa5736742678 100644 --- a/tests/unittests/sources/test_ec2.py +++ b/tests/unittests/sources/test_ec2.py @@ -13,6 +13,7 @@ from cloudinit import helpers from cloudinit.net import activators from cloudinit.sources import DataSourceEc2 as ec2 +from cloudinit.sources import NicOrder from tests.unittests import helpers as test_helpers from tests.unittests.util import MockDistro @@ -945,11 +946,18 @@ def test_invalid_ipv4_ipv6_cidr_metadata_logged_with_defaults(self): class TestBuildNicOrder: @pytest.mark.parametrize( - ["macs_metadata", "macs", "expected"], + ["macs_metadata", "macs_to_nics", "default_nic_order", "expected"], [ - pytest.param({}, [], {}, id="all_empty"), + pytest.param({}, {}, NicOrder.MAC, {}, id="all_empty"), pytest.param( - {}, ["0a:f7:8d:96:f2:a1"], {}, id="empty_macs_metadata" + {}, {}, NicOrder.NIC_NAME, {}, id="all_empty_sort_by_nic_name" + ), + pytest.param( + {}, + {"0a:f7:8d:96:f2:a1": "eth0"}, + NicOrder.MAC, + {}, + id="empty_macs_metadata", ), pytest.param( { @@ -958,7 +966,8 @@ class TestBuildNicOrder: "mac": "0a:0d:dd:44:cd:7b", } }, - [], + {}, + NicOrder.MAC, {}, id="empty_macs", ), @@ -971,8 +980,9 @@ class TestBuildNicOrder: "mac": "0a:f7:8d:96:f2:a1", }, }, - ["0a:f7:8d:96:f2:a1", "0a:0d:dd:44:cd:7b"], - {"0a:f7:8d:96:f2:a1": 0, "0a:0d:dd:44:cd:7b": 1}, + {"0a:f7:8d:96:f2:a1": "eth0", "0a:0d:dd:44:cd:7b": "eth1"}, + NicOrder.MAC, + {"0a:0d:dd:44:cd:7b": 0, "0a:f7:8d:96:f2:a1": 1}, id="no-device-number-info", ), pytest.param( @@ -984,7 +994,8 @@ class TestBuildNicOrder: "mac": "0a:f7:8d:96:f2:a1", }, }, - ["0a:f7:8d:96:f2:a1"], + {"0a:f7:8d:96:f2:a1": "eth0"}, + NicOrder.MAC, {"0a:f7:8d:96:f2:a1": 0}, id="no-device-number-info-subset", ), @@ -999,7 +1010,8 @@ class TestBuildNicOrder: "mac": "0a:f7:8d:96:f2:a1", }, }, - ["0a:f7:8d:96:f2:a1", "0a:0d:dd:44:cd:7b"], + {"0a:0d:dd:44:cd:7b": "eth0", "0a:f7:8d:96:f2:a1": "eth1"}, + NicOrder.MAC, {"0a:0d:dd:44:cd:7b": 0, "0a:f7:8d:96:f2:a1": 1}, id="device-numbers", ), @@ -1021,11 +1033,12 @@ class TestBuildNicOrder: "mac": "0a:f7:8d:96:f2:a1", }, }, - [ - "0a:f7:8d:96:f2:a1", - "0a:0d:dd:44:cd:7b", - "0a:f7:8d:96:f2:a2", - ], + { + "0a:0d:dd:44:cd:7b": "eth0", + "0a:f7:8d:96:f2:a1": "eth1", + "0a:f7:8d:96:f2:a2": "eth2", + }, + NicOrder.MAC, { "0a:0d:dd:44:cd:7b": 0, "0a:f7:8d:96:f2:a1": 1, @@ -1047,14 +1060,15 @@ class TestBuildNicOrder: }, "0a:f7:8d:96:f2:a2": { "device-number": "1", - "mac": "0a:f7:8d:96:f2:a1", + "mac": "0a:f7:8d:96:f2:a2", }, }, - [ - "0a:f7:8d:96:f2:a1", - "0a:0d:dd:44:cd:7b", - "0a:f7:8d:96:f2:a2", - ], + { + "0a:0d:dd:44:cd:7b": "eth0", + "0a:f7:8d:96:f2:a1": "eth1", + "0a:f7:8d:96:f2:a2": "eth2", + }, + NicOrder.MAC, { "0a:0d:dd:44:cd:7b": 0, "0a:f7:8d:96:f2:a1": 1, @@ -1071,14 +1085,182 @@ class TestBuildNicOrder: "mac": "0a:f7:8d:96:f2:a1", }, }, - ["0a:f7:8d:96:f2:a9"], + {"0a:f7:8d:96:f2:a9": "eth0"}, + NicOrder.MAC, {}, id="macs-not-in-md", ), + pytest.param( + {}, + {"0a:f7:8d:96:f2:a1": "eth0"}, + NicOrder.NIC_NAME, + {}, + id="empty_macs_metadata_sort_by_nic_name", + ), + pytest.param( + { + "0a:0d:dd:44:cd:7b": { + "device-number": "0", + "mac": "0a:0d:dd:44:cd:7b", + } + }, + {}, + NicOrder.NIC_NAME, + {}, + id="empty_macs_sort_by_nic_name", + ), + pytest.param( + { + "0a:0d:dd:44:cd:7b": { + "mac": "0a:0d:dd:44:cd:7b", + }, + "0a:f7:8d:96:f2:a1": { + "mac": "0a:f7:8d:96:f2:a1", + }, + }, + {"0a:f7:8d:96:f2:a1": "eth0", "0a:0d:dd:44:cd:7b": "eth1"}, + NicOrder.NIC_NAME, + {"0a:f7:8d:96:f2:a1": 0, "0a:0d:dd:44:cd:7b": 1}, + id="no-device-number-info-sort-by-nic-name", + ), + pytest.param( + { + "0a:0d:dd:44:cd:7b": { + "mac": "0a:0d:dd:44:cd:7b", + }, + "0a:f7:8d:96:f2:a1": { + "mac": "0a:f7:8d:96:f2:a1", + }, + }, + {"0a:f7:8d:96:f2:a1": "eth0"}, + NicOrder.NIC_NAME, + {"0a:f7:8d:96:f2:a1": 0}, + id="no-device-number-info-subset-sort-by-nic-name", + ), + pytest.param( + { + "0a:0d:dd:44:cd:7b": { + "device-number": "0", + "mac": "0a:0d:dd:44:cd:7b", + }, + "0a:f7:8d:96:f2:a1": { + "device-number": "1", + "mac": "0a:f7:8d:96:f2:a1", + }, + }, + {"0a:0d:dd:44:cd:7b": "eth0", "0a:f7:8d:96:f2:a1": "eth1"}, + NicOrder.NIC_NAME, + {"0a:0d:dd:44:cd:7b": 0, "0a:f7:8d:96:f2:a1": 1}, + id="device-numbers-sort-by-nic-name", + ), + pytest.param( + { + "0a:0d:dd:44:cd:7b": { + "network-card": "1", + "device-number": "1", + "mac": "0a:0d:dd:44:cd:7b", + }, + "0a:f7:8d:96:f2:a1": { + "network-card": "0", + "device-number": "0", + "mac": "0a:f7:8d:96:f2:a1", + }, + "0a:f7:8d:96:f2:a2": { + "network-card": "2", + "device-number": "1", + "mac": "0a:f7:8d:96:f2:a1", + }, + }, + { + "0a:f7:8d:96:f2:a1": "eth0", + "0a:0d:dd:44:cd:7b": "eth1", + "0a:f7:8d:96:f2:a2": "eth2", + }, + NicOrder.MAC, + { + "0a:f7:8d:96:f2:a1": 0, + "0a:0d:dd:44:cd:7b": 1, + "0a:f7:8d:96:f2:a2": 2, + }, + id="network-cardes-sort-by-nic-name", + ), + pytest.param( + { + "0a:0d:dd:44:cd:7b": { + "network-card": "0", + "device-number": "0", + "mac": "0a:0d:dd:44:cd:7b", + }, + "0a:f7:8d:96:f2:a1": { + "network-card": "1", + "device-number": "1", + "mac": "0a:f7:8d:96:f2:a1", + }, + "0a:f7:8d:96:f2:a2": { + "device-number": "1", + "mac": "0a:f7:8d:96:f2:a2", + }, + }, + { + "0a:0d:dd:44:cd:7b": "eth0", + "0a:f7:8d:96:f2:a1": "eth1", + "0a:f7:8d:96:f2:a2": "eth2", + }, + NicOrder.NIC_NAME, + { + "0a:0d:dd:44:cd:7b": 0, + "0a:f7:8d:96:f2:a1": 1, + "0a:f7:8d:96:f2:a2": 2, + }, + id="network-card-partially-missing-sort-by-nic-name", + ), + pytest.param( + { + "0a:0d:dd:44:cd:7b": { + "mac": "0a:0d:dd:44:cd:7b", + }, + "0a:f7:8d:96:f2:a1": { + "mac": "0a:f7:8d:96:f2:a1", + }, + }, + {"0a:f7:8d:96:f2:a9": "eth0"}, + NicOrder.NIC_NAME, + {}, + id="macs-not-in-md-sort-by-nic-name", + ), + pytest.param( + { + "0a:0d:dd:44:cd:7b": { + "mac": "0a:0d:dd:44:cd:7b", + }, + "0a:f7:8d:96:f2:a1": { + "mac": "0a:f7:8d:96:f2:a1", + }, + "0a:f7:8d:96:f2:a2": { + "mac": "0a:f7:8d:96:f2:a1", + }, + }, + { + "0a:f7:8d:96:f2:a1": "eth0", + "0a:0d:dd:44:cd:7b": "eth1", + "0a:f7:8d:96:f2:a2": "eth2", + }, + NicOrder.NIC_NAME, + { + "0a:f7:8d:96:f2:a1": 0, + "0a:0d:dd:44:cd:7b": 1, + "0a:f7:8d:96:f2:a2": 2, + }, + id="no-device-number-info-subset-sort-by-nic-name", + ), ], ) - def test_build_nic_order(self, macs_metadata, macs, expected): - assert expected == ec2._build_nic_order(macs_metadata, macs) + def test_build_nic_order( + self, macs_metadata, macs_to_nics, default_nic_order, expected + ): + assert expected == ec2._build_nic_order( + macs_metadata, macs_to_nics, default_nic_order + ) class TestConvertEc2MetadataNetworkConfig(test_helpers.CiTestCase): diff --git a/tests/unittests/test_upgrade.py b/tests/unittests/test_upgrade.py index 0aca3683b868..9c3975defd62 100644 --- a/tests/unittests/test_upgrade.py +++ b/tests/unittests/test_upgrade.py @@ -34,7 +34,11 @@ class TestUpgrade: # The presence of these attributes existed in 20.1. ds_expected_unpickle_attrs = { "AltCloud": {"seed", "supported_seed_starts"}, - "AliYun": {"identity", "metadata_address", "default_update_events"}, + "AliYun": { + "identity", + "metadata_address", + "default_update_events", + }, "Azure": { "_ephemeral_dhcp_ctx", "_iso_dev", @@ -75,7 +79,10 @@ class TestUpgrade: "use_ip4LL", "wait_retry", }, - "Ec2": {"identity", "metadata_address"}, + "Ec2": { + "identity", + "metadata_address", + }, "Exoscale": { "api_version", "extra_config",