diff options
| author | Chad Smith <chad.smith@canonical.com> | 2022-08-30 13:29:28 -0600 |
|---|---|---|
| committer | git-ubuntu importer <ubuntu-devel-discuss@lists.ubuntu.com> | 2022-08-30 21:29:10 +0000 |
| commit | bb2229803764a77f8b45a533f71f8e9742585bf2 (patch) | |
| tree | 7aa5daa7dd9a45fef4489a9873368de154f1648a | |
| parent | 611f032b07fa6fdd0e59b165a12d02201e61ba35 (diff) | |
| download | cloud-init-git-bb2229803764a77f8b45a533f71f8e9742585bf2.tar.gz | |
22.3-13-g70ce6442-0ubuntu1~22.10.1 (patches unapplied)
Imported using git-ubuntu import.
35 files changed, 550 insertions, 96 deletions
@@ -5,7 +5,6 @@ YAML_FILES=$(shell find cloudinit tests tools -name "*.yaml" -type f ) YAML_FILES+=$(shell find doc/examples -name "cloud-config*.txt" -type f ) PYTHON = python3 -PIP_INSTALL := pip3 install NUM_ITER ?= 100 @@ -56,14 +55,6 @@ ci-deps-ubuntu: ci-deps-centos: @$(PYTHON) $(CWD)/tools/read-dependencies --distro centos --test-distro -pip-requirements: - @echo "Installing cloud-init dependencies..." - $(PIP_INSTALL) -r "$@.txt" -q - -pip-test-requirements: - @echo "Installing cloud-init test dependencies..." - $(PIP_INSTALL) -r "$@.txt" -q - test: unittest check_version: @@ -174,6 +165,6 @@ fix_spelling: sh .PHONY: all check test flake8 clean rpm srpm deb deb-src yaml -.PHONY: check_version pip-test-requirements pip-requirements clean_pyc +.PHONY: check_version clean_pyc .PHONY: unittest style-check fix_spelling render-template benchmark-generator .PHONY: clean_pytest clean_packaging check_spelling clean_release doc diff --git a/cloudinit/cmd/devel/net_convert.py b/cloudinit/cmd/devel/net_convert.py index 3e6cdd95..269d72cd 100755 --- a/cloudinit/cmd/devel/net_convert.py +++ b/cloudinit/cmd/devel/net_convert.py @@ -133,19 +133,14 @@ def handle_args(name, args): config = ovf.Config(ovf.ConfigFile(args.network_data.name)) pre_ns = ovf.get_network_config_from_conf(config, False) - ns = network_state.parse_net_config_data(pre_ns) - - if args.debug: - sys.stderr.write("\n".join(["", "Internal State", yaml.dump(ns), ""])) distro_cls = distros.fetch(args.distro) distro = distro_cls(args.distro, {}, None) - config = {} if args.output_kind == "eni": r_cls = eni.Renderer config = distro.renderer_configs.get("eni") elif args.output_kind == "netplan": r_cls = netplan.Renderer - config = distro.renderer_configs.get("netplan") + config = distro.renderer_configs.get("netplan", {}) # don't run netplan generate/apply config["postcmds"] = False # trim leading slash @@ -165,6 +160,11 @@ def handle_args(name, args): raise RuntimeError("Invalid output_kind") r = r_cls(config=config) + ns = network_state.parse_net_config_data(pre_ns, renderer=r) + + if args.debug: + sys.stderr.write("\n".join(["", "Internal State", yaml.dump(ns), ""])) + sys.stderr.write( "".join( [ diff --git a/cloudinit/config/cc_ubuntu_autoinstall.py b/cloudinit/config/cc_ubuntu_autoinstall.py index a6180fe6..3d79c9ea 100644 --- a/cloudinit/config/cc_ubuntu_autoinstall.py +++ b/cloudinit/config/cc_ubuntu_autoinstall.py @@ -21,20 +21,21 @@ distros = ["ubuntu"] meta: MetaSchema = { "id": "cc_ubuntu_autoinstall", - "name": "Autoinstall", + "name": "Ubuntu Autoinstall", "title": "Support Ubuntu live-server install syntax", "description": dedent( """\ - Ubuntu's autoinstall syntax supports single-system automated installs - in either the live-server or live-desktop installers. + Ubuntu's autoinstall YAML supports single-system automated installs + in either the live-server install, via the ``subiquity`` snap, or the + next generation desktop installer, via `ubuntu-desktop-install` snap. When "autoinstall" directives are provided in either - #cloud-config user-data or ``/etc/cloud/cloud.cfg.d`` validate + ``#cloud-config`` user-data or ``/etc/cloud/cloud.cfg.d`` validate minimal autoinstall schema adherance and emit a warning if the live-installer is not present. The live-installer will use autoinstall directives to seed answers to configuration prompts during system install to allow for a - "touchless" Ubuntu system install. + "touchless" or non-interactive Ubuntu system install. For more details on Ubuntu's autoinstaller: https://ubuntu.com/server/docs/install/autoinstall diff --git a/cloudinit/config/cc_wireguard.py b/cloudinit/config/cc_wireguard.py index 8cfbf6f1..366aff40 100644 --- a/cloudinit/config/cc_wireguard.py +++ b/cloudinit/config/cc_wireguard.py @@ -264,7 +264,7 @@ def handle(name: str, cfg: dict, cloud: Cloud, log, args: list): wg_section = cfg["wireguard"] else: LOG.debug( - "Skipping module named %s," " no 'wireguard' configuration found", + "Skipping module named %s, no 'wireguard' configuration found", name, ) return diff --git a/cloudinit/distros/__init__.py b/cloudinit/distros/__init__.py index 7aafaa78..4a468cf8 100644 --- a/cloudinit/distros/__init__.py +++ b/cloudinit/distros/__init__.py @@ -16,7 +16,7 @@ import stat import string import urllib.parse from io import StringIO -from typing import Any, Mapping, Optional, Type +from typing import Any, Mapping, MutableMapping, Optional, Type from cloudinit import importer from cloudinit import log as logging @@ -25,6 +25,7 @@ from cloudinit.distros.parsers import hosts from cloudinit.features import ALLOW_EC2_MIRRORS_ON_NON_AWS_INSTANCE_TYPES from cloudinit.net import activators, eni, network_state, renderers from cloudinit.net.network_state import parse_net_config_data +from cloudinit.net.renderer import Renderer from .networking import LinuxNetworking, Networking @@ -78,7 +79,7 @@ class Distro(persistence.CloudInitPickleMixin, metaclass=abc.ABCMeta): tz_zone_dir = "/usr/share/zoneinfo" default_owner = "root:root" init_cmd = ["service"] # systemctl, service etc - renderer_configs: Mapping[str, Mapping[str, Any]] = {} + renderer_configs: Mapping[str, MutableMapping[str, Any]] = {} _preferred_ntp_clients = None networking_cls: Type[Networking] = LinuxNetworking # This is used by self.shutdown_command(), and can be overridden in @@ -129,7 +130,7 @@ class Distro(persistence.CloudInitPickleMixin, metaclass=abc.ABCMeta): except activators.NoActivatorException: return None - def _write_network_state(self, network_state): + def _get_renderer(self) -> Renderer: priority = util.get_cfg_by_path( self._cfg, ("network", "renderers"), None ) @@ -139,6 +140,9 @@ class Distro(persistence.CloudInitPickleMixin, metaclass=abc.ABCMeta): "Selected renderer '%s' from priority list: %s", name, priority ) renderer = render_cls(config=self.renderer_configs.get(name)) + return renderer + + def _write_network_state(self, network_state, renderer: Renderer): renderer.render_network_state(network_state) def _find_tz_file(self, tz): @@ -241,15 +245,17 @@ class Distro(persistence.CloudInitPickleMixin, metaclass=abc.ABCMeta): """ # This method is preferred to apply_network which only takes # a much less complete network config format (interfaces(5)). - network_state = parse_net_config_data(netconfig) try: - self._write_network_state(network_state) + renderer = self._get_renderer() except NotImplementedError: # backwards compat until all distros have apply_network_config return self._apply_network_from_network_config( netconfig, bring_up=bring_up ) + network_state = parse_net_config_data(netconfig, renderer=renderer) + self._write_network_state(network_state, renderer) + # Now try to bring them up if bring_up: LOG.debug("Bringing up newly configured network interfaces") diff --git a/cloudinit/distros/arch.py b/cloudinit/distros/arch.py index 0bdfef83..2d5cfbf6 100644 --- a/cloudinit/distros/arch.py +++ b/cloudinit/distros/arch.py @@ -11,6 +11,7 @@ from cloudinit import log as logging from cloudinit import subp, util from cloudinit.distros import net_util from cloudinit.distros.parsers.hostname import HostnameConf +from cloudinit.net.renderer import Renderer from cloudinit.net.renderers import RendererNotFoundError from cloudinit.settings import PER_INSTANCE @@ -61,9 +62,9 @@ class Distro(distros.Distro): self.update_package_sources() self.package_command("", pkgs=pkglist) - def _write_network_state(self, network_state): + def _get_renderer(self) -> Renderer: try: - super()._write_network_state(network_state) + return super()._get_renderer() except RendererNotFoundError as e: # Fall back to old _write_network raise NotImplementedError from e diff --git a/cloudinit/distros/debian.py b/cloudinit/distros/debian.py index 6dc1ad40..87f4cc9f 100644 --- a/cloudinit/distros/debian.py +++ b/cloudinit/distros/debian.py @@ -137,9 +137,9 @@ class Distro(distros.Distro): self.update_package_sources() self.package_command("install", pkgs=pkglist) - def _write_network_state(self, network_state): + def _write_network_state(self, *args, **kwargs): _maybe_remove_legacy_eth0() - return super()._write_network_state(network_state) + return super()._write_network_state(*args, **kwargs) def _write_hostname(self, hostname, filename): conf = None diff --git a/cloudinit/net/bsd.py b/cloudinit/net/bsd.py index ff5c7413..e0f18366 100644 --- a/cloudinit/net/bsd.py +++ b/cloudinit/net/bsd.py @@ -1,11 +1,13 @@ # This file is part of cloud-init. See LICENSE file for license information. import re +from typing import Optional from cloudinit import log as logging from cloudinit import net, subp, util from cloudinit.distros import bsd_utils from cloudinit.distros.parsers.resolv_conf import ResolvConf +from cloudinit.net.network_state import NetworkState from . import renderer @@ -156,7 +158,12 @@ class BSDRenderer(renderer.Renderer): 0o644, ) - def render_network_state(self, network_state, templates=None, target=None): + def render_network_state( + self, + network_state: NetworkState, + templates: Optional[dict] = None, + target=None, + ) -> None: if target: self.target = target self._ifconfig_entries(settings=network_state) diff --git a/cloudinit/net/eni.py b/cloudinit/net/eni.py index b0ec67bd..ea0b8e4a 100644 --- a/cloudinit/net/eni.py +++ b/cloudinit/net/eni.py @@ -4,10 +4,12 @@ import copy import glob import os import re +from typing import Optional from cloudinit import log as logging from cloudinit import subp, util from cloudinit.net import subnet_is_ipv6 +from cloudinit.net.network_state import NetworkState from . import ParserError, renderer @@ -561,7 +563,12 @@ class Renderer(renderer.Renderer): return "\n\n".join(["\n".join(s) for s in sections]) + "\n" - def render_network_state(self, network_state, templates=None, target=None): + def render_network_state( + self, + network_state: NetworkState, + templates: Optional[dict] = None, + target=None, + ) -> None: fpeni = subp.target_path(target, self.eni_path) util.ensure_dir(os.path.dirname(fpeni)) header = self.eni_header if self.eni_header else "" diff --git a/cloudinit/net/ephemeral.py b/cloudinit/net/ephemeral.py index c0d83d29..81f7079f 100644 --- a/cloudinit/net/ephemeral.py +++ b/cloudinit/net/ephemeral.py @@ -429,9 +429,9 @@ class EphemeralIPNetwork: # therefore catch exception unless only v4 is used try: if self.ipv4: - self.stack.enter_context(EphemeralIPv6Network(self.interface)) - if self.ipv6: self.stack.enter_context(EphemeralDHCPv4(self.interface)) + if self.ipv6: + self.stack.enter_context(EphemeralIPv6Network(self.interface)) # v6 link local might be usable # caller may want to log network state except NoDHCPLeaseError as e: diff --git a/cloudinit/net/netplan.py b/cloudinit/net/netplan.py index d63d86d8..7b91077d 100644 --- a/cloudinit/net/netplan.py +++ b/cloudinit/net/netplan.py @@ -3,7 +3,7 @@ import copy import os import textwrap -from typing import cast +from typing import Optional, cast from cloudinit import log as logging from cloudinit import safeyaml, subp, util @@ -240,7 +240,12 @@ class Renderer(renderer.Renderer): LOG.debug("Failed to list features from netplan info: %s", e) return self._features - def render_network_state(self, network_state, templates=None, target=None): + def render_network_state( + self, + network_state: NetworkState, + templates: Optional[dict] = None, + target=None, + ) -> None: # check network state for version # if v2, then extract network_state.config # else render_v2_from_state diff --git a/cloudinit/net/network_manager.py b/cloudinit/net/network_manager.py index 8fd15575..8053511c 100644 --- a/cloudinit/net/network_manager.py +++ b/cloudinit/net/network_manager.py @@ -11,10 +11,12 @@ import io import itertools import os import uuid +from typing import Optional from cloudinit import log as logging from cloudinit import subp, util from cloudinit.net import is_ipv6_address, subnet_is_ipv6 +from cloudinit.net.network_state import NetworkState from . import renderer @@ -69,7 +71,7 @@ class NMConnection: method_map = { "static": "manual", - "dhcp6": "dhcp", + "dhcp6": "auto", "ipv6_slaac": "auto", "ipv6_dhcpv6-stateless": "auto", "ipv6_dhcpv6-stateful": "auto", @@ -96,8 +98,6 @@ class NMConnection: self.config[family]["method"] = method self._set_default(family, "may-fail", "false") - if family == "ipv6": - self._set_default(family, "addr-gen-mode", "stable-privacy") def _add_numbered(self, section, key_prefix, value): """ @@ -344,7 +344,12 @@ class Renderer(renderer.Renderer): # Well, what can we do... return con_id - def render_network_state(self, network_state, templates=None, target=None): + def render_network_state( + self, + network_state: NetworkState, + templates: Optional[dict] = None, + target=None, + ) -> None: # First pass makes sure there's NMConnections for all known # interfaces that have UUIDs that can be linked to from related # interfaces diff --git a/cloudinit/net/network_state.py b/cloudinit/net/network_state.py index 80f2b108..e4f7a7fd 100644 --- a/cloudinit/net/network_state.py +++ b/cloudinit/net/network_state.py @@ -7,7 +7,7 @@ import copy import functools import logging -from typing import Any, Dict +from typing import TYPE_CHECKING, Any, Dict, Optional from cloudinit import safeyaml, util from cloudinit.net import ( @@ -22,6 +22,9 @@ from cloudinit.net import ( net_prefix_to_ipv4_mask, ) +if TYPE_CHECKING: + from cloudinit.net.renderer import Renderer + LOG = logging.getLogger(__name__) NETWORK_STATE_VERSION = 1 @@ -136,14 +139,16 @@ class CommandHandlerMeta(type): class NetworkState(object): - def __init__(self, network_state, version=NETWORK_STATE_VERSION): + def __init__( + self, network_state: dict, version: int = NETWORK_STATE_VERSION + ): self._network_state = copy.deepcopy(network_state) self._version = version self.use_ipv6 = network_state.get("use_ipv6", False) self._has_default_route = None @property - def config(self): + def config(self) -> dict: return self._network_state["config"] @property @@ -204,6 +209,20 @@ class NetworkState(object): route.get("prefix") == 0 and route.get("network") in default_nets ) + @classmethod + def to_passthrough(cls, network_state: dict) -> "NetworkState": + """Instantiates a `NetworkState` without interpreting its data. + + That means only `config` and `version` are copied. + + :param network_state: Network state data. + :return: Instance of `NetworkState`. + """ + kwargs = {} + if "version" in network_state: + kwargs["version"] = network_state["version"] + return cls({"config": network_state}, **kwargs) + class NetworkStateInterpreter(metaclass=CommandHandlerMeta): @@ -218,16 +237,27 @@ class NetworkStateInterpreter(metaclass=CommandHandlerMeta): "config": None, } - def __init__(self, version=NETWORK_STATE_VERSION, config=None): + def __init__( + self, + version=NETWORK_STATE_VERSION, + config=None, + renderer=None, # type: Optional[Renderer] + ): self._version = version self._config = config self._network_state = copy.deepcopy(self.initial_network_state) self._network_state["config"] = config self._parsed = False - self._interface_dns_map = {} + self._interface_dns_map: dict = {} + self._renderer = renderer @property - def network_state(self): + def network_state(self) -> NetworkState: + from cloudinit.net.netplan import Renderer as NetplanRenderer + + if self._version == 2 and isinstance(self._renderer, NetplanRenderer): + LOG.debug("Passthrough netplan v2 config") + return NetworkState.to_passthrough(self._config) return NetworkState(self._network_state, version=self._version) @property @@ -268,10 +298,6 @@ class NetworkStateInterpreter(metaclass=CommandHandlerMeta): def as_dict(self): return {"version": self._version, "config": self._config} - def get_network_state(self): - ns = self.network_state - return ns - def parse_config(self, skip_broken=True): if self._version == 1: self.parse_config_v1(skip_broken=skip_broken) @@ -316,6 +342,12 @@ class NetworkStateInterpreter(metaclass=CommandHandlerMeta): } def parse_config_v2(self, skip_broken=True): + from cloudinit.net.netplan import Renderer as NetplanRenderer + + if isinstance(self._renderer, NetplanRenderer): + # Nothing to parse as we are going to perform a Netplan passthrough + return + for command_type, command in self._config.items(): if command_type in ["version", "renderer"]: continue @@ -764,7 +796,7 @@ class NetworkStateInterpreter(metaclass=CommandHandlerMeta): " netplan rendering support." ) - def _v2_common(self, cfg): + def _v2_common(self, cfg) -> None: LOG.debug("v2_common: handling config:\n%s", cfg) for iface, dev_cfg in cfg.items(): if "set-name" in dev_cfg: @@ -781,10 +813,13 @@ class NetworkStateInterpreter(metaclass=CommandHandlerMeta): name_cmd.update({"address": dns}) self.handle_nameserver(name_cmd) - mac_address = dev_cfg.get("match", {}).get("macaddress") - real_if_name = find_interface_name_from_mac(mac_address) - if real_if_name: - iface = real_if_name + mac_address: Optional[str] = dev_cfg.get("match", {}).get( + "macaddress" + ) + if mac_address: + real_if_name = find_interface_name_from_mac(mac_address) + if real_if_name: + iface = real_if_name self._handle_individual_nameserver(name_cmd, iface) @@ -1044,7 +1079,11 @@ def _normalize_subnets(subnets): return [_normalize_subnet(s) for s in subnets] -def parse_net_config_data(net_config, skip_broken=True) -> NetworkState: +def parse_net_config_data( + net_config: dict, + skip_broken: bool = True, + renderer=None, # type: Optional[Renderer] +) -> NetworkState: """Parses the config, returns NetworkState object :param net_config: curtin network config dict @@ -1058,9 +1097,11 @@ def parse_net_config_data(net_config, skip_broken=True) -> NetworkState: config = net_config if version and config is not None: - nsi = NetworkStateInterpreter(version=version, config=config) + nsi = NetworkStateInterpreter( + version=version, config=config, renderer=renderer + ) nsi.parse_config(skip_broken=skip_broken) - state = nsi.get_network_state() + state = nsi.network_state if not state: raise RuntimeError( diff --git a/cloudinit/net/networkd.py b/cloudinit/net/networkd.py index 7d7d82c2..e0a5d848 100644 --- a/cloudinit/net/networkd.py +++ b/cloudinit/net/networkd.py @@ -8,9 +8,11 @@ # This file is part of cloud-init. See LICENSE file for license information. from collections import OrderedDict +from typing import Optional from cloudinit import log as logging from cloudinit import subp, util +from cloudinit.net.network_state import NetworkState from . import renderer @@ -44,10 +46,16 @@ class CfgParser: for k, v in sorted(self.conf_dict.items()): if not v: continue - contents += "[" + k + "]\n" - for e in sorted(v): - contents += e + "\n" - contents += "\n" + if k == "Address": + for e in sorted(v): + contents += "[" + k + "]\n" + contents += e + "\n" + contents += "\n" + else: + contents += "[" + k + "]\n" + for e in sorted(v): + contents += e + "\n" + contents += "\n" return contents @@ -217,7 +225,12 @@ class Renderer(renderer.Renderer): util.write_file(net_fn, conf) util.chownbyname(net_fn, net_fn_owner, net_fn_owner) - def render_network_state(self, network_state, templates=None, target=None): + def render_network_state( + self, + network_state: NetworkState, + templates: Optional[dict] = None, + target=None, + ) -> None: network_dir = self.network_conf_dir if target: network_dir = subp.target_path(target) + network_dir @@ -242,7 +255,7 @@ class Renderer(renderer.Renderer): self.parse_routes(route, cfg) if ns.version == 2: - name = iface["name"] + name: Optional[str] = iface["name"] # network state doesn't give dhcp domain info # using ns.config as a workaround here @@ -257,8 +270,8 @@ class Renderer(renderer.Renderer): if dev_cfg.get("set-name") == name: name = dev_name break - - self.dhcp_domain(ns.config["ethernets"][name], cfg) + if name in ns.config["ethernets"]: + self.dhcp_domain(ns.config["ethernets"][name], cfg) ret_dict.update({link: cfg.get_final_conf()}) diff --git a/cloudinit/net/renderer.py b/cloudinit/net/renderer.py index da154731..d7bc19b1 100644 --- a/cloudinit/net/renderer.py +++ b/cloudinit/net/renderer.py @@ -7,6 +7,7 @@ import abc import io +from typing import Optional from cloudinit.net.network_state import NetworkState, parse_net_config_data from cloudinit.net.udev import generate_udev_rule @@ -49,11 +50,19 @@ class Renderer(object): return content.getvalue() @abc.abstractmethod - def render_network_state(self, network_state, templates=None, target=None): + def render_network_state( + self, + network_state: NetworkState, + templates: Optional[dict] = None, + target=None, + ) -> None: """Render network state.""" def render_network_config( - self, network_config, templates=None, target=None + self, + network_config: dict, + templates: Optional[dict] = None, + target=None, ): return self.render_network_state( network_state=parse_net_config_data(network_config), diff --git a/cloudinit/net/sysconfig.py b/cloudinit/net/sysconfig.py index 698724ab..d5789fb0 100644 --- a/cloudinit/net/sysconfig.py +++ b/cloudinit/net/sysconfig.py @@ -4,7 +4,7 @@ import copy import io import os import re -from typing import Mapping +from typing import Mapping, Optional from cloudinit import log as logging from cloudinit import subp, util @@ -980,8 +980,11 @@ class Renderer(renderer.Renderer): return contents def render_network_state( - self, network_state: NetworkState, templates=None, target=None - ): + self, + network_state: NetworkState, + templates: Optional[dict] = None, + target=None, + ) -> None: if not templates: templates = self.templates file_mode = 0o644 diff --git a/debian/changelog b/debian/changelog index 37334586..711fa812 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,21 @@ +cloud-init (22.3-13-g70ce6442-0ubuntu1~22.10.1) kinetic; urgency=medium + + * New upstream snapshot. + + Fix v2 interface matching when no MAC (LP: #1986551) + + test: reduce number of network dependencies in flaky test (#1702) + + docs: publish cc_ubuntu_autoinstall docs to rtd (#1696) + + net: Fix EphemeraIPNetwork (#1697) + + test: make ansible test work across older versions (#1691) + + Networkd multi-address support/fix (#1685) [Teodor Garzdin] + + make: drop broken targets (#1688) + + net: Passthough v2 netconfigs in netplan systems (#1650) + (LP: #1978543) + + NM ipv6 connection does not work on Azure and Openstack (#1616) + [Emanuele Giuseppe Esposito] + + Fix check_format_tip (#1679) + + -- Chad Smith <chad.smith@canonical.com> Tue, 30 Aug 2022 13:29:28 -0600 + cloud-init (22.3-3-g9f0efc47-0ubuntu1~22.10.1) kinetic; urgency=medium * New upstream snapshot. diff --git a/doc/rtd/topics/modules.rst b/doc/rtd/topics/modules.rst index 8ffb984d..cbe0f5d7 100644 --- a/doc/rtd/topics/modules.rst +++ b/doc/rtd/topics/modules.rst @@ -55,6 +55,7 @@ Module Reference .. automodule:: cloudinit.config.cc_ssh_import_id .. automodule:: cloudinit.config.cc_timezone .. automodule:: cloudinit.config.cc_ubuntu_advantage +.. automodule:: cloudinit.config.cc_ubuntu_autoinstall .. automodule:: cloudinit.config.cc_ubuntu_drivers .. automodule:: cloudinit.config.cc_update_etc_hosts .. automodule:: cloudinit.config.cc_update_hostname diff --git a/tests/integration_tests/modules/test_ansible.py b/tests/integration_tests/modules/test_ansible.py index eebc7be9..0d979d40 100644 --- a/tests/integration_tests/modules/test_ansible.py +++ b/tests/integration_tests/modules/test_ansible.py @@ -31,8 +31,9 @@ write_files: WantedBy=cloud-init-local.service [Service] - ExecStart=/usr/bin/env python3 -m http.server \ - --directory /root/playbooks/.git + WorkingDirectory=/root/playbooks/.git + ExecStart=/usr/bin/env python3 -m http.server --bind 0.0.0.0 8000 + - path: /etc/systemd/system/repo_waiter.service content: | @@ -49,7 +50,7 @@ write_files: # running and continue once it is up, but this is simple and works [Service] Type=oneshot - ExecStart=sh -c "while \ + ExecStart=/bin/sh -c "while \ ! git clone http://0.0.0.0:8000/ $(mktemp -d); do sleep 0.1; done" - path: /root/playbooks/ubuntu.yml @@ -94,6 +95,9 @@ ansible: """ SETUP_REPO = f"cd {REPO_D} &&\ +git config --global user.name auto &&\ +git config --global user.email autom@tic.io &&\ +git config --global init.defaultBranch main &&\ git init {REPO_D} &&\ git add {REPO_D}/roles/apt/tasks/main.yml {REPO_D}/ubuntu.yml &&\ git commit -m auto &&\ @@ -101,8 +105,9 @@ git commit -m auto &&\ def _test_ansible_pull_from_local_server(my_client): - - assert my_client.execute(SETUP_REPO).ok + setup = my_client.execute(SETUP_REPO) + assert not setup.stderr + assert not setup.return_code my_client.execute("cloud-init clean --logs") my_client.restart() log = my_client.read_from_file("/var/log/cloud-init.log") diff --git a/tests/integration_tests/modules/test_combined.py b/tests/integration_tests/modules/test_combined.py index 7e84626f..93523bfc 100644 --- a/tests/integration_tests/modules/test_combined.py +++ b/tests/integration_tests/modules/test_combined.py @@ -67,8 +67,8 @@ snap: commands: - snap install hello-world ssh_import_id: - - gh:powersj - lp:smoser + timezone: US/Aleutian """ diff --git a/tests/unittests/cmd/devel/test_net_convert.py b/tests/unittests/cmd/devel/test_net_convert.py index 60acb1a6..100aa8de 100644 --- a/tests/unittests/cmd/devel/test_net_convert.py +++ b/tests/unittests/cmd/devel/test_net_convert.py @@ -4,6 +4,7 @@ import itertools import pytest +from cloudinit import safeyaml as yaml from cloudinit.cmd.devel import net_convert from cloudinit.distros.debian import NETWORK_FILE_HEADER from tests.unittests.helpers import mock @@ -183,5 +184,46 @@ class TestNetConvert: ) ] == chown.call_args_list + @pytest.mark.parametrize("debug", (False, True)) + def test_convert_netplan_passthrough(self, debug, tmpdir): + """Assert that if the network config's version is 2 and the renderer is + Netplan, then the config is passed through as-is. + """ + network_data = tmpdir.join("network_data") + # `default` as a route supported by Netplan but not by cloud-init + content = """\ + network: + version: 2 + ethernets: + enp0s3: + dhcp4: false + addresses: [10.0.4.10/24] + nameservers: + addresses: [10.0.4.1] + routes: + - to: default + via: 10.0.4.1 + metric: 100 + """ + network_data.write(content) + args = [ + "-m", + "enp0s3,AA", + f"--directory={tmpdir.strpath}", + f"--network-data={network_data.strpath}", + "--distro=ubuntu", + "--kind=yaml", + "--output-kind=netplan", + ] + if debug: + args.append("--debug") + params = self._replace_path_args(args, tmpdir) + with mock.patch("sys.argv", ["net-convert"] + params): + args = net_convert.get_parser().parse_args() + with mock.patch("cloudinit.util.chownbyname"): + net_convert.handle_args("somename", args) + outfile = tmpdir.join("etc/netplan/50-cloud-init.yaml") + assert yaml.load(content) == yaml.load(outfile.read()) + # vi: ts=4 expandtab diff --git a/tests/unittests/conftest.py b/tests/unittests/conftest.py index e265a285..1ab17e8b 100644 --- a/tests/unittests/conftest.py +++ b/tests/unittests/conftest.py @@ -1,6 +1,7 @@ import builtins import glob import os +from pathlib import Path import pytest @@ -55,3 +56,12 @@ def fake_filesystem(mocker, tmpdir): func = getattr(mod, f) trap_func = retarget_many_wrapper(str(tmpdir), nargs, func) mocker.patch.object(mod, f, trap_func) + + +PYTEST_VERSION_TUPLE = tuple(map(int, pytest.__version__.split("."))) + +if PYTEST_VERSION_TUPLE < (3, 9, 0): + + @pytest.fixture + def tmp_path(tmpdir): + return Path(tmpdir) diff --git a/tests/unittests/distros/test_netconfig.py b/tests/unittests/distros/test_netconfig.py index 38e92f0e..6509f1de 100644 --- a/tests/unittests/distros/test_netconfig.py +++ b/tests/unittests/distros/test_netconfig.py @@ -235,6 +235,38 @@ network: """ +V2_PASSTHROUGH_NET_CFG = { + "ethernets": { + "eth7": { + "addresses": ["192.168.1.5/24"], + "gateway4": "192.168.1.254", + "routes": [{"to": "default", "via": "10.0.4.1", "metric": 100}], + }, + }, + "version": 2, +} + + +V2_PASSTHROUGH_NET_CFG_OUTPUT = """\ +# This file is generated from information provided by the datasource. Changes +# to it will not persist across an instance reboot. To disable cloud-init's +# network configuration capabilities, write a file +# /etc/cloud/cloud.cfg.d/99-disable-network-config.cfg with the following: +# network: {config: disabled} +network: + ethernets: + eth7: + addresses: + - 192.168.1.5/24 + gateway4: 192.168.1.254 + routes: + - metric: 100 + to: default + via: 10.0.4.1 + version: 2 +""" + + class WriteBuffer(object): def __init__(self): self.buffer = StringIO() @@ -472,6 +504,9 @@ class TestNetCfgDistroUbuntuEni(TestNetCfgDistroBase): class TestNetCfgDistroUbuntuNetplan(TestNetCfgDistroBase): + + with_logs = True + def setUp(self): super(TestNetCfgDistroUbuntuNetplan, self).setUp() self.distro = self._get_distro("ubuntu", renderers=["netplan"]) @@ -540,6 +575,22 @@ class TestNetCfgDistroUbuntuNetplan(TestNetCfgDistroBase): expected_cfgs=expected_cfgs.copy(), ) + def test_apply_network_config_v2_full_passthrough_ub(self): + expected_cfgs = { + self.netplan_path(): V2_PASSTHROUGH_NET_CFG_OUTPUT, + } + # ub_distro.apply_network_config(V2_PASSTHROUGH_NET_CFG, False) + self._apply_and_verify_netplan( + self.distro.apply_network_config, + V2_PASSTHROUGH_NET_CFG, + expected_cfgs=expected_cfgs.copy(), + ) + self.assertIn("Passthrough netplan v2 config", self.logs.getvalue()) + self.assertIn( + "Selected renderer 'netplan' from priority list: ['netplan']", + self.logs.getvalue(), + ) + class TestNetCfgDistroRedhat(TestNetCfgDistroBase): def setUp(self): diff --git a/tests/unittests/net/artifacts/no_matching_mac/etc/NetworkManager/system-connections/cloud-init-encc000.2653.nmconnection b/tests/unittests/net/artifacts/no_matching_mac/etc/NetworkManager/system-connections/cloud-init-encc000.2653.nmconnection new file mode 100644 index 00000000..80483d4f --- /dev/null +++ b/tests/unittests/net/artifacts/no_matching_mac/etc/NetworkManager/system-connections/cloud-init-encc000.2653.nmconnection @@ -0,0 +1,21 @@ +# Generated by cloud-init. Changes will be lost. + +[connection] +id=cloud-init encc000.2653 +uuid=116aaf19-aabc-50ea-b480-e9aee18bda59 +type=vlan +interface-name=encc000.2653 + +[user] +org.freedesktop.NetworkManager.origin=cloud-init + +[vlan] +id=2653 +parent=f869ebd3-f175-5747-bf02-d0d44d687248 + +[ipv4] +method=manual +may-fail=false +address1=10.245.236.14/24 +gateway=10.245.236.1 +dns=10.245.236.1; diff --git a/tests/unittests/net/artifacts/no_matching_mac/etc/NetworkManager/system-connections/cloud-init-encc000.nmconnection b/tests/unittests/net/artifacts/no_matching_mac/etc/NetworkManager/system-connections/cloud-init-encc000.nmconnection new file mode 100644 index 00000000..3368388d --- /dev/null +++ b/tests/unittests/net/artifacts/no_matching_mac/etc/NetworkManager/system-connections/cloud-init-encc000.nmconnection @@ -0,0 +1,12 @@ +# Generated by cloud-init. Changes will be lost. + +[connection] +id=cloud-init encc000 +uuid=f869ebd3-f175-5747-bf02-d0d44d687248 +type=ethernet +interface-name=encc000 + +[user] +org.freedesktop.NetworkManager.origin=cloud-init + +[ethernet] diff --git a/tests/unittests/net/artifacts/no_matching_mac/etc/NetworkManager/system-connections/cloud-init-zz-all-en.nmconnection b/tests/unittests/net/artifacts/no_matching_mac/etc/NetworkManager/system-connections/cloud-init-zz-all-en.nmconnection new file mode 100644 index 00000000..16120bc1 --- /dev/null +++ b/tests/unittests/net/artifacts/no_matching_mac/etc/NetworkManager/system-connections/cloud-init-zz-all-en.nmconnection @@ -0,0 +1,16 @@ +# Generated by cloud-init. Changes will be lost. + +[connection] +id=cloud-init zz-all-en +uuid=159daec9-cba3-5101-85e7-46d831857f43 +type=ethernet +interface-name=zz-all-en + +[user] +org.freedesktop.NetworkManager.origin=cloud-init + +[ethernet] + +[ipv4] +method=auto +may-fail=false diff --git a/tests/unittests/net/artifacts/no_matching_mac/etc/NetworkManager/system-connections/cloud-init-zz-all-eth.nmconnection b/tests/unittests/net/artifacts/no_matching_mac/etc/NetworkManager/system-connections/cloud-init-zz-all-eth.nmconnection new file mode 100644 index 00000000..df44d546 --- /dev/null +++ b/tests/unittests/net/artifacts/no_matching_mac/etc/NetworkManager/system-connections/cloud-init-zz-all-eth.nmconnection @@ -0,0 +1,16 @@ +# Generated by cloud-init. Changes will be lost. + +[connection] +id=cloud-init zz-all-eth +uuid=23a83d8a-d7db-5133-a77b-e68a6ac61ec9 +type=ethernet +interface-name=zz-all-eth + +[user] +org.freedesktop.NetworkManager.origin=cloud-init + +[ethernet] + +[ipv4] +method=auto +may-fail=false diff --git a/tests/unittests/net/artifacts/no_matching_mac_v2.yaml b/tests/unittests/net/artifacts/no_matching_mac_v2.yaml new file mode 100644 index 00000000..f5fc5ef1 --- /dev/null +++ b/tests/unittests/net/artifacts/no_matching_mac_v2.yaml @@ -0,0 +1,22 @@ +network: + version: 2 + ethernets: + encc000: {} + zz-all-en: + match: + name: "en*" + dhcp4: true + zz-all-eth: + match: + name: "eth*" + dhcp4: true + vlans: + encc000.2653: + id: 2653 + link: "encc000" + addresses: + - "10.245.236.14/24" + gateway4: "10.245.236.1" + nameservers: + addresses: + - "10.245.236.1" diff --git a/tests/unittests/net/test_ephemeral.py b/tests/unittests/net/test_ephemeral.py new file mode 100644 index 00000000..d2237faf --- /dev/null +++ b/tests/unittests/net/test_ephemeral.py @@ -0,0 +1,49 @@ +# This file is part of cloud-init. See LICENSE file for license information. + +from unittest import mock + +import pytest + +from cloudinit.net.ephemeral import EphemeralIPNetwork + +M_PATH = "cloudinit.net.ephemeral." + + +class TestEphemeralIPNetwork: + @pytest.mark.parametrize("ipv6", [False, True]) + @pytest.mark.parametrize("ipv4", [False, True]) + @mock.patch(M_PATH + "contextlib.ExitStack") + @mock.patch(M_PATH + "EphemeralIPv6Network") + @mock.patch(M_PATH + "EphemeralDHCPv4") + def test_stack_order( + self, + m_ephemeral_dhcp_v4, + m_ephemeral_ip_v6_network, + m_exit_stack, + ipv4, + ipv6, + ): + interface = object() + with EphemeralIPNetwork(interface, ipv4=ipv4, ipv6=ipv6): + pass + expected_call_args_list = [] + if ipv4: + expected_call_args_list.append( + mock.call(m_ephemeral_dhcp_v4.return_value) + ) + assert [mock.call(interface)] == m_ephemeral_dhcp_v4.call_args_list + else: + assert [] == m_ephemeral_dhcp_v4.call_args_list + if ipv6: + expected_call_args_list.append( + mock.call(m_ephemeral_ip_v6_network.return_value) + ) + assert [ + mock.call(interface) + ] == m_ephemeral_ip_v6_network.call_args_list + else: + assert [] == m_ephemeral_ip_v6_network.call_args_list + assert ( + expected_call_args_list + == m_exit_stack.return_value.enter_context.call_args_list + ) diff --git a/tests/unittests/net/test_net_rendering.py b/tests/unittests/net/test_net_rendering.py new file mode 100644 index 00000000..06feab89 --- /dev/null +++ b/tests/unittests/net/test_net_rendering.py @@ -0,0 +1,101 @@ +"""Home of the tests for end-to-end net rendering + +Tests defined here should take a v1 or v2 yaml config as input, and verify +that the rendered network config is as expected. Input files are defined +under `tests/unittests/net/artifacts` with the format of + +<test_name><format>.yaml + +For example, if my test name is "test_all_the_things" and I'm testing a +v2 format, I should have a file named test_all_the_things_v2.yaml. + +If a renderer outputs multiple files, the expected files should live in +the artifacts directory under the given test name. For example, if I'm +expecting NetworkManager to output a file named eth0.nmconnection as +part of my "test_all_the_things" test, then in the artifacts directory +there should be a +`test_all_the_things/etc/NetworkManager/system-connections/eth0.nmconnection` +file. + +To add a new nominal test, create the input and output files, then add the test +name to the `test_convert` test along with it's supported renderers. + +Before adding a test here, check that it is not already represented +in `unittests/test_net.py`. While that file contains similar tests, it has +become too large to be maintainable. +""" +import glob +from enum import Flag, auto +from pathlib import Path + +import pytest + +from cloudinit import safeyaml +from cloudinit.net.netplan import Renderer as NetplanRenderer +from cloudinit.net.network_manager import Renderer as NetworkManagerRenderer +from cloudinit.net.network_state import NetworkState, parse_net_config_data + +ARTIFACT_DIR = Path(__file__).parent.absolute() / "artifacts" + + +class Renderer(Flag): + Netplan = auto() + NetworkManager = auto() + Networkd = auto() + + +@pytest.fixture(autouse=True) +def setup(mocker): + mocker.patch("cloudinit.net.network_state.get_interfaces_by_mac") + + +def _check_netplan( + network_state: NetworkState, netplan_path: Path, expected_config +): + if network_state.version == 2: + renderer = NetplanRenderer(config={"netplan_path": netplan_path}) + renderer.render_network_state(network_state) + assert safeyaml.load(netplan_path.read_text()) == expected_config, ( + f"Netplan config generated at {netplan_path} does not match v2 " + "config defined for this test." + ) + else: + raise NotImplementedError + + +def _check_network_manager(network_state: NetworkState, tmp_path: Path): + renderer = NetworkManagerRenderer() + renderer.render_network_state( + network_state, target=str(tmp_path / "no_matching_mac") + ) + expected_paths = glob.glob( + str(ARTIFACT_DIR / "no_matching_mac" / "**/*.nmconnection"), + recursive=True, + ) + for expected_path in expected_paths: + expected_contents = Path(expected_path).read_text() + actual_path = tmp_path / expected_path.split( + str(ARTIFACT_DIR), maxsplit=1 + )[1].lstrip("/") + assert ( + actual_path.exists() + ), f"Expected {actual_path} to exist, but it does not" + actual_contents = actual_path.read_text() + assert expected_contents.strip() == actual_contents.strip() + + +@pytest.mark.parametrize( + "test_name, renderers", + [("no_matching_mac_v2", Renderer.Netplan | Renderer.NetworkManager)], +) +def test_convert(test_name, renderers, tmp_path): + network_config = safeyaml.load( + Path(ARTIFACT_DIR, f"{test_name}.yaml").read_text() + ) + network_state = parse_net_config_data(network_config["network"]) + if Renderer.Netplan in renderers: + _check_netplan( + network_state, tmp_path / "netplan.yaml", network_config + ) + if Renderer.NetworkManager in renderers: + _check_network_manager(network_state, tmp_path) diff --git a/tests/unittests/net/test_network_state.py b/tests/unittests/net/test_network_state.py index b76b5dd7..75d033dc 100644 --- a/tests/unittests/net/test_network_state.py +++ b/tests/unittests/net/test_network_state.py @@ -79,7 +79,8 @@ class TestNetworkStateParseConfig(CiTestCase): ncfg = {"version": 2, "otherconfig": {}, "somemore": [1, 2, 3]} network_state.parse_net_config_data(ncfg) self.assertEqual( - [mock.call(version=2, config=ncfg)], self.m_nsi.call_args_list + [mock.call(version=2, config=ncfg, renderer=None)], + self.m_nsi.call_args_list, ) def test_valid_config_gets_network_state(self): diff --git a/tests/unittests/net/test_networkd.py b/tests/unittests/net/test_networkd.py index ee50e640..a22c5092 100644 --- a/tests/unittests/net/test_networkd.py +++ b/tests/unittests/net/test_networkd.py @@ -12,6 +12,7 @@ network: eth0: match: macaddress: '00:11:22:33:44:55' + addresses: [172.16.10.2/12, 172.16.10.3/12] nameservers: search: [spam.local, eggs.local] addresses: [8.8.8.8] @@ -24,7 +25,13 @@ network: addresses: [4.4.4.4] """ -V2_CONFIG_SET_NAME_RENDERED_ETH0 = """[Match] +V2_CONFIG_SET_NAME_RENDERED_ETH0 = """[Address] +Address=172.16.10.2/12 + +[Address] +Address=172.16.10.3/12 + +[Match] MACAddress=00:11:22:33:44:55 Name=eth0 diff --git a/tests/unittests/test_net.py b/tests/unittests/test_net.py index bfc13734..525706d1 100644 --- a/tests/unittests/test_net.py +++ b/tests/unittests/test_net.py @@ -1248,9 +1248,8 @@ NETWORK_CONFIGS = { may-fail=false [ipv6] - method=dhcp + method=auto may-fail=false - addr-gen-mode=stable-privacy """ ), @@ -1278,6 +1277,7 @@ NETWORK_CONFIGS = { DHCP=no [Address] Address=192.168.14.2/24 + [Address] Address=2001:1::1/64 """ ).rstrip(" "), @@ -1383,7 +1383,6 @@ NETWORK_CONFIGS = { [ipv6] method=manual may-fail=false - addr-gen-mode=stable-privacy address1=2001:1::1/64 """ @@ -1416,9 +1415,8 @@ NETWORK_CONFIGS = { [ethernet] [ipv6] - method=dhcp + method=auto may-fail=false - addr-gen-mode=stable-privacy [ipv4] method=auto @@ -1517,9 +1515,8 @@ NETWORK_CONFIGS = { [ethernet] [ipv6] - method=dhcp + method=auto may-fail=false - addr-gen-mode=stable-privacy """ ), @@ -1750,7 +1747,6 @@ NETWORK_CONFIGS = { [ipv6] method=auto may-fail=false - addr-gen-mode=stable-privacy """ ), @@ -1862,7 +1858,6 @@ NETWORK_CONFIGS = { [ipv6] method=auto may-fail=false - addr-gen-mode=stable-privacy """ ), @@ -2683,7 +2678,6 @@ pre-down route del -net 10.0.0.0/8 gw 11.0.0.1 metric 3 || true [ipv6] method=manual may-fail=false - addr-gen-mode=stable-privacy address1=2001:1::1/64 route1=::/0,2001:4800:78ff:1b::1 @@ -2736,9 +2730,8 @@ pre-down route del -net 10.0.0.0/8 gw 11.0.0.1 metric 3 || true xmit_hash_policy=layer3+4 [ipv6] - method=dhcp + method=auto may-fail=false - addr-gen-mode=stable-privacy """ ), @@ -3342,7 +3335,6 @@ iface bond0 inet6 static [ipv6] method=manual may-fail=false - addr-gen-mode=stable-privacy address1=2001:1::1/92 route1=2001:67c::/32,2001:67c:1562::1 route2=3001:67c::/32,3001:67c:15::1 @@ -3463,7 +3455,6 @@ iface bond0 inet6 static [ipv6] method=manual may-fail=false - addr-gen-mode=stable-privacy address1=2001:1::bbbb/96 route1=::/0,2001:1::1 @@ -3641,7 +3632,6 @@ iface bond0 inet6 static [ipv6] method=manual may-fail=false - addr-gen-mode=stable-privacy address1=2001:1::100/96 """ @@ -3666,7 +3656,6 @@ iface bond0 inet6 static [ipv6] method=manual may-fail=false - addr-gen-mode=stable-privacy address1=2001:1::101/96 """ diff --git a/tools/.github-cla-signers b/tools/.github-cla-signers index 2a69fd57..271a4710 100644 --- a/tools/.github-cla-signers +++ b/tools/.github-cla-signers @@ -38,6 +38,7 @@ emmanuelthome eslerm esposem GabrielNagy +garzdin giggsoff hamalq holmanb @@ -99,6 +99,8 @@ commands = deps = black flake8 + hypothesis + hypothesis_jsonschema isort mypy pylint @@ -108,6 +110,7 @@ deps = types-pyyaml types-requests types-setuptools + typing-extensions -r{toxinidir}/test-requirements.txt -r{toxinidir}/integration-requirements.txt commands = |