diff options
Diffstat (limited to 'cloudinit')
| -rw-r--r-- | cloudinit/cmd/tests/test_query.py | 2 | ||||
| -rw-r--r-- | cloudinit/config/cc_apt_configure.py | 2 | ||||
| -rw-r--r-- | cloudinit/config/cc_chef.py | 2 | ||||
| -rw-r--r-- | cloudinit/config/cc_grub_dpkg.py | 95 | ||||
| -rw-r--r-- | cloudinit/config/tests/test_grub_dpkg.py | 176 | ||||
| -rwxr-xr-x | cloudinit/distros/__init__.py | 2 | ||||
| -rw-r--r-- | cloudinit/net/__init__.py | 8 | ||||
| -rw-r--r-- | cloudinit/net/tests/test_dhcp.py | 2 | ||||
| -rw-r--r-- | cloudinit/sources/DataSourceGCE.py | 2 | ||||
| -rw-r--r-- | cloudinit/sources/DataSourceOpenNebula.py | 6 | ||||
| -rw-r--r-- | cloudinit/sources/helpers/openstack.py | 17 | ||||
| -rw-r--r-- | cloudinit/ssh_util.py | 4 | ||||
| -rw-r--r-- | cloudinit/tests/test_url_helper.py | 2 |
13 files changed, 275 insertions, 45 deletions
diff --git a/cloudinit/cmd/tests/test_query.py b/cloudinit/cmd/tests/test_query.py index 6d36a4ea..cb15b2d2 100644 --- a/cloudinit/cmd/tests/test_query.py +++ b/cloudinit/cmd/tests/test_query.py @@ -261,7 +261,7 @@ class TestQuery(CiTestCase): args = self.args( debug=False, dump_all=False, format=None, instance_data=self.instance_data, list_keys=True, user_data='ud', - vendor_data='vd', varname='top') + vendor_data='vd', varname='top') with mock.patch('sys.stderr', new_callable=StringIO) as m_stderr: with mock.patch('sys.stdout', new_callable=StringIO) as m_stdout: with mock.patch('os.getuid') as m_getuid: diff --git a/cloudinit/config/cc_apt_configure.py b/cloudinit/config/cc_apt_configure.py index 9a33451d..b1c7b471 100644 --- a/cloudinit/config/cc_apt_configure.py +++ b/cloudinit/config/cc_apt_configure.py @@ -297,7 +297,7 @@ schema = { }, 'conf': { 'type': 'string', - 'description': dedent("""\ + 'description': dedent("""\ Specify configuration for apt, such as proxy configuration. This configuration is specified as a string. For multiline apt configuration, make sure diff --git a/cloudinit/config/cc_chef.py b/cloudinit/config/cc_chef.py index 01d61fa1..03285ef0 100644 --- a/cloudinit/config/cc_chef.py +++ b/cloudinit/config/cc_chef.py @@ -50,6 +50,7 @@ file). written to /etc/chef/client.rb) chef: + chef_license: client_key: encrypted_data_bag_secret: environment: @@ -125,6 +126,7 @@ CHEF_RB_TPL_PATH_KEYS = frozenset([ 'file_cache_path', 'pid_file', 'encrypted_data_bag_secret', + 'chef_license', ]) CHEF_RB_TPL_KEYS = list(CHEF_RB_TPL_DEFAULTS.keys()) CHEF_RB_TPL_KEYS.extend(CHEF_RB_TPL_BOOL_KEYS) diff --git a/cloudinit/config/cc_grub_dpkg.py b/cloudinit/config/cc_grub_dpkg.py index a323edfa..7888464e 100644 --- a/cloudinit/config/cc_grub_dpkg.py +++ b/cloudinit/config/cc_grub_dpkg.py @@ -1,8 +1,9 @@ -# Copyright (C) 2009-2010 Canonical Ltd. +# Copyright (C) 2009-2010, 2020 Canonical Ltd. # Copyright (C) 2012 Hewlett-Packard Development Company, L.P. # # Author: Scott Moser <scott.moser@canonical.com> # Author: Juerg Haefliger <juerg.haefliger@hp.com> +# Author: Matthew Ruffell <matthew.ruffell@canonical.com> # # This file is part of cloud-init. See LICENSE file for license information. @@ -15,15 +16,15 @@ Configure which device is used as the target for grub installation. This module should work correctly by default without any user configuration. It can be enabled/disabled using the ``enabled`` config key in the ``grub_dpkg`` config dict. The global config key ``grub-dpkg`` is an alias for ``grub_dpkg``. If no -installation device is specified this module will look for the first existing -device in: +installation device is specified this module will execute grub-probe to +determine which disk the /boot directory is associated with. - - ``/dev/sda`` - - ``/dev/vda`` - - ``/dev/xvda`` - - ``/dev/sda1`` - - ``/dev/vda1`` - - ``/dev/xvda1`` +The value which is placed into the debconf database is in the format which the +grub postinstall script expects. Normally, this is a /dev/disk/by-id/ value, +but we do fallback to the plain disk name if a by-id name is not present. + +If this module is executed inside a container, then the debconf database is +seeded with empty values, and install_devices_empty is set to true. **Internal name:** ``cc_grub_dpkg`` @@ -43,10 +44,66 @@ device in: import os from cloudinit import util +from cloudinit.util import ProcessExecutionError distros = ['ubuntu', 'debian'] +def fetch_idevs(log): + """ + Fetches the /dev/disk/by-id device grub is installed to. + Falls back to plain disk name if no by-id entry is present. + """ + disk = "" + devices = [] + + try: + # get the root disk where the /boot directory resides. + disk = util.subp(['grub-probe', '-t', 'disk', '/boot'], + capture=True)[0].strip() + except ProcessExecutionError as e: + # grub-common may not be installed, especially on containers + # FileNotFoundError is a nested exception of ProcessExecutionError + if isinstance(e.reason, FileNotFoundError): + log.debug("'grub-probe' not found in $PATH") + # disks from the container host are present in /proc and /sys + # which is where grub-probe determines where /boot is. + # it then checks for existence in /dev, which fails as host disks + # are not exposed to the container. + elif "failed to get canonical path" in e.stderr: + log.debug("grub-probe 'failed to get canonical path'") + else: + # something bad has happened, continue to log the error + raise + except Exception: + util.logexc(log, "grub-probe failed to execute for grub-dpkg") + + if not disk or not os.path.exists(disk): + # If we failed to detect a disk, we can return early + return '' + + try: + # check if disk exists and use udevadm to fetch symlinks + devices = util.subp( + ['udevadm', 'info', '--root', '--query=symlink', disk], + capture=True + )[0].strip().split() + except Exception: + util.logexc( + log, "udevadm DEVLINKS symlink query failed for disk='%s'", disk + ) + + log.debug('considering these device symlinks: %s', ','.join(devices)) + # filter symlinks for /dev/disk/by-id entries + devices = [dev for dev in devices if 'disk/by-id' in dev] + log.debug('filtered to these disk/by-id symlinks: %s', ','.join(devices)) + # select first device if there is one, else fall back to plain name + idevs = sorted(devices)[0] if devices else disk + log.debug('selected %s', idevs) + + return idevs + + def handle(name, cfg, _cloud, log, _args): mycfg = cfg.get("grub_dpkg", cfg.get("grub-dpkg", {})) @@ -62,22 +119,10 @@ def handle(name, cfg, _cloud, log, _args): idevs_empty = util.get_cfg_option_str( mycfg, "grub-pc/install_devices_empty", None) - if ((os.path.exists("/dev/sda1") and not os.path.exists("/dev/sda")) or - (os.path.exists("/dev/xvda1") and not os.path.exists("/dev/xvda"))): - if idevs is None: - idevs = "" - if idevs_empty is None: - idevs_empty = "true" - else: - if idevs_empty is None: - idevs_empty = "false" - if idevs is None: - idevs = "/dev/sda" - for dev in ("/dev/sda", "/dev/vda", "/dev/xvda", - "/dev/sda1", "/dev/vda1", "/dev/xvda1"): - if os.path.exists(dev): - idevs = dev - break + if idevs is None: + idevs = fetch_idevs(log) + if idevs_empty is None: + idevs_empty = "false" if idevs else "true" # now idevs and idevs_empty are set to determined values # or, those set by user diff --git a/cloudinit/config/tests/test_grub_dpkg.py b/cloudinit/config/tests/test_grub_dpkg.py new file mode 100644 index 00000000..01efa330 --- /dev/null +++ b/cloudinit/config/tests/test_grub_dpkg.py @@ -0,0 +1,176 @@ +# This file is part of cloud-init. See LICENSE file for license information. + +import pytest + +from unittest import mock +from logging import Logger +from cloudinit.util import ProcessExecutionError +from cloudinit.config.cc_grub_dpkg import fetch_idevs, handle + + +class TestFetchIdevs: + """Tests cc_grub_dpkg.fetch_idevs()""" + + # Note: udevadm info returns devices in a large single line string + @pytest.mark.parametrize( + "grub_output,path_exists,expected_log_call,udevadm_output" + ",expected_idevs", + [ + # Inside a container, grub not installed + ( + ProcessExecutionError(reason=FileNotFoundError()), + False, + mock.call("'grub-probe' not found in $PATH"), + '', + '', + ), + # Inside a container, grub installed + ( + ProcessExecutionError(stderr="failed to get canonical path"), + False, + mock.call("grub-probe 'failed to get canonical path'"), + '', + '', + ), + # KVM Instance + ( + ['/dev/vda'], + True, + None, + ( + '/dev/disk/by-path/pci-0000:00:00.0 ', + '/dev/disk/by-path/virtio-pci-0000:00:00.0 ' + ), + '/dev/vda', + ), + # Xen Instance + ( + ['/dev/xvda'], + True, + None, + '', + '/dev/xvda', + ), + # NVMe Hardware Instance + ( + ['/dev/nvme1n1'], + True, + None, + ( + '/dev/disk/by-id/nvme-Company_hash000 ', + '/dev/disk/by-id/nvme-nvme.000-000-000-000-000 ', + '/dev/disk/by-path/pci-0000:00:00.0-nvme-0 ' + ), + '/dev/disk/by-id/nvme-Company_hash000', + ), + # SCSI Hardware Instance + ( + ['/dev/sda'], + True, + None, + ( + '/dev/disk/by-id/company-user-1 ', + '/dev/disk/by-id/scsi-0Company_user-1 ', + '/dev/disk/by-path/pci-0000:00:00.0-scsi-0:0:0:0 ' + ), + '/dev/disk/by-id/company-user-1', + ), + ], + ) + @mock.patch("cloudinit.config.cc_grub_dpkg.util.logexc") + @mock.patch("cloudinit.config.cc_grub_dpkg.os.path.exists") + @mock.patch("cloudinit.config.cc_grub_dpkg.util.subp") + def test_fetch_idevs(self, m_subp, m_exists, m_logexc, grub_output, + path_exists, expected_log_call, udevadm_output, + expected_idevs): + """Tests outputs from grub-probe and udevadm info against grub-dpkg""" + m_subp.side_effect = [ + grub_output, + ["".join(udevadm_output)] + ] + m_exists.return_value = path_exists + log = mock.Mock(spec=Logger) + idevs = fetch_idevs(log) + assert expected_idevs == idevs + if expected_log_call is not None: + assert expected_log_call in log.debug.call_args_list + + +class TestHandle: + """Tests cc_grub_dpkg.handle()""" + + @pytest.mark.parametrize( + "cfg_idevs,cfg_idevs_empty,fetch_idevs_output,expected_log_output", + [ + ( + # No configuration + None, + None, + '/dev/disk/by-id/nvme-Company_hash000', + ( + "Setting grub debconf-set-selections with ", + "'/dev/disk/by-id/nvme-Company_hash000','false'" + ), + ), + ( + # idevs set, idevs_empty unset + '/dev/sda', + None, + '/dev/sda', + ( + "Setting grub debconf-set-selections with ", + "'/dev/sda','false'" + ), + ), + ( + # idevs unset, idevs_empty set + None, + 'true', + '/dev/xvda', + ( + "Setting grub debconf-set-selections with ", + "'/dev/xvda','true'" + ), + ), + ( + # idevs set, idevs_empty set + '/dev/vda', + 'false', + '/dev/disk/by-id/company-user-1', + ( + "Setting grub debconf-set-selections with ", + "'/dev/vda','false'" + ), + ), + ( + # idevs set, idevs_empty set + # Respect what the user defines, even if its logically wrong + '/dev/nvme0n1', + 'true', + '', + ( + "Setting grub debconf-set-selections with ", + "'/dev/nvme0n1','true'" + ), + ) + ], + ) + @mock.patch("cloudinit.config.cc_grub_dpkg.fetch_idevs") + @mock.patch("cloudinit.config.cc_grub_dpkg.util.get_cfg_option_str") + @mock.patch("cloudinit.config.cc_grub_dpkg.util.logexc") + @mock.patch("cloudinit.config.cc_grub_dpkg.util.subp") + def test_handle(self, m_subp, m_logexc, m_get_cfg_str, m_fetch_idevs, + cfg_idevs, cfg_idevs_empty, fetch_idevs_output, + expected_log_output): + """Test setting of correct debconf database entries""" + m_get_cfg_str.side_effect = [ + cfg_idevs, + cfg_idevs_empty + ] + m_fetch_idevs.return_value = fetch_idevs_output + log = mock.Mock(spec=Logger) + handle(mock.Mock(), mock.Mock(), mock.Mock(), log, mock.Mock()) + log.debug.assert_called_with("".join(expected_log_output)) + + +# vi: ts=4 expandtab diff --git a/cloudinit/distros/__init__.py b/cloudinit/distros/__init__.py index e99529df..35a10590 100755 --- a/cloudinit/distros/__init__.py +++ b/cloudinit/distros/__init__.py @@ -582,7 +582,7 @@ class Distro(metaclass=abc.ABCMeta): # passwd must use short '-l' due to SLES11 lacking long form '--lock' lock_tools = (['passwd', '-l', name], ['usermod', '--lock', name]) try: - cmd = next(l for l in lock_tools if util.which(l[0])) + cmd = next(tool for tool in lock_tools if util.which(tool[0])) except StopIteration: raise RuntimeError(( "Unable to lock user account '%s'. No tools available. " diff --git a/cloudinit/net/__init__.py b/cloudinit/net/__init__.py index cb8c1601..8af24fa9 100644 --- a/cloudinit/net/__init__.py +++ b/cloudinit/net/__init__.py @@ -824,13 +824,13 @@ def get_interfaces_by_mac_on_freebsd(): # flatten each interface block in a single line def flatten(out): curr_block = '' - for l in out.split('\n'): - if l.startswith('\t'): - curr_block += l + for line in out.split('\n'): + if line.startswith('\t'): + curr_block += line else: if curr_block: yield curr_block - curr_block = l + curr_block = line yield curr_block # looks for interface and mac in a list of flatten block diff --git a/cloudinit/net/tests/test_dhcp.py b/cloudinit/net/tests/test_dhcp.py index bc7bef45..7768da7c 100644 --- a/cloudinit/net/tests/test_dhcp.py +++ b/cloudinit/net/tests/test_dhcp.py @@ -211,7 +211,7 @@ class TestDHCPParseStaticRoutes(CiTestCase): "class_b": "16,172,16,10", "class_a": "8,10,10", "gateway": "0,0", - "netlen": "33,0", + "netlen": "33,0", } for rfc3442 in bad_rfc3442.values(): self.assertEqual([], parse_static_routes(rfc3442)) diff --git a/cloudinit/sources/DataSourceGCE.py b/cloudinit/sources/DataSourceGCE.py index 6cbfbbac..0ec5f6ec 100644 --- a/cloudinit/sources/DataSourceGCE.py +++ b/cloudinit/sources/DataSourceGCE.py @@ -116,7 +116,7 @@ def _write_host_key_to_guest_attributes(key_type, key_value): resp = url_helper.readurl(url=url, data=key_value, headers=HEADERS, request_method='PUT', check_status=False) if resp.ok(): - LOG.debug('Wrote %s host key to guest attributes.', key_type) + LOG.debug('Wrote %s host key to guest attributes.', key_type) else: LOG.debug('Unable to write %s host key to guest attributes.', key_type) diff --git a/cloudinit/sources/DataSourceOpenNebula.py b/cloudinit/sources/DataSourceOpenNebula.py index 02c9a7b8..a08ab404 100644 --- a/cloudinit/sources/DataSourceOpenNebula.py +++ b/cloudinit/sources/DataSourceOpenNebula.py @@ -417,9 +417,9 @@ def read_context_disk_dir(source_dir, asuser=None): if ssh_key_var: lines = context.get(ssh_key_var).splitlines() - results['metadata']['public-keys'] = [l for l in lines - if len(l) and not - l.startswith("#")] + results['metadata']['public-keys'] = [ + line for line in lines if len(line) and not line.startswith("#") + ] # custom hostname -- try hostname or leave cloud-init # itself create hostname from IP address later diff --git a/cloudinit/sources/helpers/openstack.py b/cloudinit/sources/helpers/openstack.py index e91398ea..a4373f24 100644 --- a/cloudinit/sources/helpers/openstack.py +++ b/cloudinit/sources/helpers/openstack.py @@ -411,8 +411,11 @@ class ConfigDriveReader(BaseReader): keydata = meta_js.get('public-keys', keydata) if keydata: lines = keydata.splitlines() - md['public-keys'] = [l for l in lines - if len(l) and not l.startswith("#")] + md['public-keys'] = [ + line + for line in lines + if len(line) and not line.startswith("#") + ] # config-drive-v1 has no way for openstack to provide the instance-id # so we copy that into metadata from the user input @@ -674,11 +677,13 @@ def convert_net_json(network_json=None, known_macs=None): raise ValueError("Unable to find a system nic for %s" % d) d['name'] = known_macs[mac] - for cfg, key, fmt, target in link_updates: - if isinstance(target, (list, tuple)): - cfg[key] = [fmt % link_id_info[l]['name'] for l in target] + for cfg, key, fmt, targets in link_updates: + if isinstance(targets, (list, tuple)): + cfg[key] = [ + fmt % link_id_info[target]['name'] for target in targets + ] else: - cfg[key] = fmt % link_id_info[target]['name'] + cfg[key] = fmt % link_id_info[targets]['name'] # Infiniband interfaces may be referenced in network_data.json by a 6 byte # Ethernet MAC-style address, and we use that address to look up the diff --git a/cloudinit/ssh_util.py b/cloudinit/ssh_util.py index c3a9b5b7..918c4aec 100644 --- a/cloudinit/ssh_util.py +++ b/cloudinit/ssh_util.py @@ -344,7 +344,9 @@ def update_ssh_config(updates, fname=DEF_SSHD_CFG): changed = update_ssh_config_lines(lines=lines, updates=updates) if changed: util.write_file( - fname, "\n".join([str(l) for l in lines]) + "\n", copy_mode=True) + fname, "\n".join( + [str(line) for line in lines] + ) + "\n", copy_mode=True) return len(changed) != 0 diff --git a/cloudinit/tests/test_url_helper.py b/cloudinit/tests/test_url_helper.py index 29b39374..364ec822 100644 --- a/cloudinit/tests/test_url_helper.py +++ b/cloudinit/tests/test_url_helper.py @@ -85,7 +85,7 @@ class TestReadFileOrUrl(CiTestCase): read_file_or_url(url, headers=headers, headers_redact=['sensitive']) logs = self.logs.getvalue() for k in headers.keys(): - self.assertEqual(headers[k], httpretty.last_request().headers[k]) + self.assertEqual(headers[k], httpretty.last_request().headers[k]) self.assertIn(REDACTED, logs) self.assertNotIn('sekret', logs) |