From 1aa70cf25156c5d403f0a62997aa6b0e56c42e93 Mon Sep 17 00:00:00 2001 From: Daniel Watkins Date: Wed, 20 Jan 2021 13:15:55 -0500 Subject: 20.4-66-gad23441d-0ubuntu1 (patches unapplied) Imported using git-ubuntu import. --- .travis.yml | 80 ------ cloudinit/config/cc_seed_random.py | 12 +- cloudinit/distros/arch.py | 2 + cloudinit/net/eni.py | 2 + cloudinit/sources/DataSourceOVF.py | 162 +++++++++++- cloudinit/sources/helpers/vmware/imc/config.py | 12 + .../sources/helpers/vmware/imc/guestcust_error.py | 1 + cloudinit/ssh_util.py | 6 +- debian/changelog | 49 ++-- tests/integration_tests/bugs/test_gh632.py | 3 + tests/integration_tests/bugs/test_gh668.py | 37 +++ tests/integration_tests/bugs/test_lp1910835.py | 66 +++++ tests/integration_tests/clouds.py | 3 + .../modules/test_seed_random_data.py | 2 +- tests/integration_tests/test_upgrade.py | 4 +- tests/unittests/test_datasource/test_ovf.py | 291 ++++++++++++++++++++- tests/unittests/test_net.py | 7 + tests/unittests/test_sshutil.py | 6 +- tests/unittests/test_vmware_config_file.py | 16 ++ tools/.github-cla-signers | 3 + tox.ini | 1 + 21 files changed, 641 insertions(+), 124 deletions(-) create mode 100644 tests/integration_tests/bugs/test_gh668.py create mode 100644 tests/integration_tests/bugs/test_lp1910835.py diff --git a/.travis.yml b/.travis.yml index 8ead3468..f15752bc 100644 --- a/.travis.yml +++ b/.travis.yml @@ -41,86 +41,6 @@ matrix: fast_finish: true include: - python: 3.6 - - if: NOT branch =~ /^ubuntu\// - env: {} - cache: - - directories: - - lxd_images - - chroots - before_cache: - - | - # Find the most recent image file - latest_file="$(sudo ls -Art /var/snap/lxd/common/lxd/images/ | tail -n 1)" - # This might be .rootfs or , normalise - latest_file="$(basename $latest_file .rootfs)" - # Find all files with that prefix and copy them to our cache dir - sudo find /var/snap/lxd/common/lxd/images/ -name $latest_file* -print -exec cp {} "$TRAVIS_BUILD_DIR/lxd_images/" \; - install: - - git fetch --unshallow - - sudo apt-get install -y --install-recommends sbuild ubuntu-dev-tools fakeroot tox debhelper - - pip install . - - pip install tox - # bionic has lxd from deb installed, remove it first to ensure - # pylxd talks only to the lxd from snap - - sudo apt remove --purge lxd lxd-client - - sudo rm -Rf /var/lib/lxd - - sudo snap install lxd - - sudo lxd init --auto - - sudo mkdir --mode=1777 -p /var/snap/lxd/common/consoles - # Move any cached lxd images into lxd's image dir - - sudo find "$TRAVIS_BUILD_DIR/lxd_images/" -type f -print -exec mv {} /var/snap/lxd/common/lxd/images/ \; - - sudo usermod -a -G lxd $USER - - sudo sbuild-adduser $USER - - cp /usr/share/doc/sbuild/examples/example.sbuildrc /home/$USER/.sbuildrc - script: - # Ubuntu LTS: Build - - ./packages/bddeb -S -d --release xenial - - | - needs_caching=false - if [ -e "$TRAVIS_BUILD_DIR/chroots/xenial-amd64.tar" ]; then - # If we have a cached chroot, move it into place - sudo mkdir -p /var/lib/schroot/chroots/xenial-amd64 - sudo tar --sparse --xattrs --preserve-permissions --numeric-owner -xf "$TRAVIS_BUILD_DIR/chroots/xenial-amd64.tar" -C /var/lib/schroot/chroots/xenial-amd64 - # Write its configuration - cat > sbuild-xenial-amd64 << EOM - [xenial-amd64] - description=xenial-amd64 - groups=sbuild,root,admin - root-groups=sbuild,root,admin - # Uncomment these lines to allow members of these groups to access - # the -source chroots directly (useful for automated updates, etc). - #source-root-users=sbuild,root,admin - #source-root-groups=sbuild,root,admin - type=directory - profile=sbuild - union-type=overlay - directory=/var/lib/schroot/chroots/xenial-amd64 - EOM - sudo mv sbuild-xenial-amd64 /etc/schroot/chroot.d/ - sudo chown root /etc/schroot/chroot.d/sbuild-xenial-amd64 - # And ensure it's up-to-date. - before_pkgs="$(sudo schroot -c source:xenial-amd64 -d / dpkg -l | sha256sum)" - sudo schroot -c source:xenial-amd64 -d / -- sh -c "apt-get update && apt-get -qqy upgrade" - after_pkgs=$(sudo schroot -c source:xenial-amd64 -d / dpkg -l | sha256sum) - if [ "$before_pkgs" != "$after_pkgs" ]; then - needs_caching=true - fi - else - # Otherwise, create the chroot - sudo -E su $USER -c 'mk-sbuild xenial' - needs_caching=true - fi - # If there are changes to the schroot (or it's entirely new), - # tar up the schroot (to preserve ownership/permissions) and - # move it into the cached dir; no need to compress it because - # Travis will do that anyway - if [ "$needs_caching" = "true" ]; then - sudo tar --sparse --xattrs --xattrs-include=* -cf "$TRAVIS_BUILD_DIR/chroots/xenial-amd64.tar" -C /var/lib/schroot/chroots/xenial-amd64 . - fi - # Use sudo to get a new shell where we're in the sbuild group - - sudo -E su $USER -c 'sbuild --nolog --no-run-lintian --verbose --dist=xenial cloud-init_*.dsc' - # Ubuntu LTS: Integration - - sg lxd -c 'tox -e citest -- run --verbose --preserve-data --data-dir results --os-name xenial --test modules/apt_configure_sources_list.yaml --test modules/ntp_servers --test modules/set_password_list --test modules/user_groups --deb cloud-init_*_all.deb' - name: "Integration Tests" if: NOT branch =~ /^ubuntu\// env: {} diff --git a/cloudinit/config/cc_seed_random.py b/cloudinit/config/cc_seed_random.py index 4fb9b44e..911789c7 100644 --- a/cloudinit/config/cc_seed_random.py +++ b/cloudinit/config/cc_seed_random.py @@ -24,15 +24,19 @@ Configuration for this module is under the ``random_seed`` config key. The optionally be specified in encoded form, with the encoding specified in ``encoding``. +If the cloud provides its own random seed data, it will be appended to ``data`` +before it is written to ``file``. + .. note:: when using a multiline value for ``data`` or specifying binary data, be sure to follow yaml syntax and use the ``|`` and ``!binary`` yaml format specifiers when appropriate -Instead of specifying a data string, a command can be run to generate/collect -the data to be written. The command should be specified as a list of args in -the ``command`` key. If a command is specified that cannot be run, no error -will be reported unless ``command_required`` is set to true. +If the ``command`` key is specified, the given command will be executed. This +will happen after ``file`` has been populated. That command's environment will +contain the value of the ``file`` key as ``RANDOM_SEED_FILE``. If a command is +specified that cannot be run, no error will be reported unless +``command_required`` is set to true. For example, to use ``pollinate`` to gather data from a remote entropy server and write it to ``/dev/urandom``, the following could be diff --git a/cloudinit/distros/arch.py b/cloudinit/distros/arch.py index 967be168..378a6daa 100644 --- a/cloudinit/distros/arch.py +++ b/cloudinit/distros/arch.py @@ -152,6 +152,8 @@ class Distro(distros.Distro): elif args and isinstance(args, list): cmd.extend(args) + if command == "upgrade": + command = "-u" if command: cmd.append(command) diff --git a/cloudinit/net/eni.py b/cloudinit/net/eni.py index 0074691b..a89e5ad2 100644 --- a/cloudinit/net/eni.py +++ b/cloudinit/net/eni.py @@ -387,6 +387,8 @@ class Renderer(renderer.Renderer): if k == 'network': if ':' in route[k]: route_line += ' -A inet6' + elif route.get('prefix') == 32: + route_line += ' -host' else: route_line += ' -net' if 'prefix' in route: diff --git a/cloudinit/sources/DataSourceOVF.py b/cloudinit/sources/DataSourceOVF.py index 741c140a..94d9f1b9 100644 --- a/cloudinit/sources/DataSourceOVF.py +++ b/cloudinit/sources/DataSourceOVF.py @@ -16,6 +16,7 @@ from xml.dom import minidom from cloudinit import dmi from cloudinit import log as logging +from cloudinit import safeyaml from cloudinit import sources from cloudinit import subp from cloudinit import util @@ -47,6 +48,7 @@ LOG = logging.getLogger(__name__) CONFGROUPNAME_GUESTCUSTOMIZATION = "deployPkg" GUESTCUSTOMIZATION_ENABLE_CUST_SCRIPTS = "enable-custom-scripts" +VMWARE_IMC_DIR = "/var/run/vmware-imc" class DataSourceOVF(sources.DataSource): @@ -99,9 +101,7 @@ class DataSourceOVF(sources.DataSource): if not self.vmware_customization_supported: LOG.debug("Skipping the check for " "VMware Customization support") - elif not util.get_cfg_option_bool( - self.sys_cfg, "disable_vmware_customization", True): - + else: search_paths = ( "/usr/lib/vmware-tools", "/usr/lib64/vmware-tools", "/usr/lib/open-vm-tools", "/usr/lib64/open-vm-tools") @@ -119,7 +119,9 @@ class DataSourceOVF(sources.DataSource): # When the VM is powered on, the "VMware Tools" daemon # copies the customization specification file to # /var/run/vmware-imc directory. cloud-init code needs - # to search for the file in that directory. + # to search for the file in that directory which indicates + # that required metadata and userdata files are now + # present. max_wait = get_max_wait_from_cfg(self.ds_cfg) vmwareImcConfigFilePath = util.log_time( logfunc=LOG.debug, @@ -129,26 +131,83 @@ class DataSourceOVF(sources.DataSource): else: LOG.debug("Did not find the customization plugin.") + md_path = None if vmwareImcConfigFilePath: + imcdirpath = os.path.dirname(vmwareImcConfigFilePath) + cf = ConfigFile(vmwareImcConfigFilePath) + self._vmware_cust_conf = Config(cf) LOG.debug("Found VMware Customization Config File at %s", vmwareImcConfigFilePath) - nicspath = wait_for_imc_cfg_file( - filename="nics.txt", maxwait=10, naplen=5) + try: + (md_path, ud_path, nicspath) = collect_imc_file_paths( + self._vmware_cust_conf) + except FileNotFoundError as e: + _raise_error_status( + "File(s) missing in directory", + e, + GuestCustEvent.GUESTCUST_EVENT_CUSTOMIZE_FAILED, + vmwareImcConfigFilePath, + self._vmware_cust_conf) else: LOG.debug("Did not find VMware Customization Config File") - else: - LOG.debug("Customization for VMware platform is disabled.") - if vmwareImcConfigFilePath: + # Honor disable_vmware_customization setting on metadata absent + if not md_path: + if util.get_cfg_option_bool(self.sys_cfg, + "disable_vmware_customization", + True): + LOG.debug( + "Customization for VMware platform is disabled.") + # reset vmwareImcConfigFilePath to None to avoid + # customization for VMware platform + vmwareImcConfigFilePath = None + + use_raw_data = bool(vmwareImcConfigFilePath and md_path) + if use_raw_data: + set_gc_status(self._vmware_cust_conf, "Started") + LOG.debug("Start to load cloud-init meta data and user data") + try: + (md, ud, cfg, network) = load_cloudinit_data(md_path, ud_path) + + if network: + self._network_config = network + else: + self._network_config = ( + self.distro.generate_fallback_config() + ) + + except safeyaml.YAMLError as e: + _raise_error_status( + "Error parsing the cloud-init meta data", + e, + GuestCustErrorEnum.GUESTCUST_ERROR_WRONG_META_FORMAT, + vmwareImcConfigFilePath, + self._vmware_cust_conf) + except Exception as e: + _raise_error_status( + "Error loading cloud-init configuration", + e, + GuestCustEvent.GUESTCUST_EVENT_CUSTOMIZE_FAILED, + vmwareImcConfigFilePath, + self._vmware_cust_conf) + + self._vmware_cust_found = True + found.append('vmware-tools') + + util.del_dir(imcdirpath) + set_customization_status( + GuestCustStateEnum.GUESTCUST_STATE_DONE, + GuestCustErrorEnum.GUESTCUST_ERROR_SUCCESS) + set_gc_status(self._vmware_cust_conf, "Successful") + + elif vmwareImcConfigFilePath: + # Load configuration from vmware_imc self._vmware_nics_to_enable = "" try: - cf = ConfigFile(vmwareImcConfigFilePath) - self._vmware_cust_conf = Config(cf) set_gc_status(self._vmware_cust_conf, "Started") (md, ud, cfg) = read_vmware_imc(self._vmware_cust_conf) self._vmware_nics_to_enable = get_nics_to_enable(nicspath) - imcdirpath = os.path.dirname(vmwareImcConfigFilePath) product_marker = self._vmware_cust_conf.marker_id hasmarkerfile = check_marker_exists( product_marker, os.path.join(self.paths.cloud_dir, 'data')) @@ -684,4 +743,83 @@ def _raise_error_status(prefix, error, event, config_file, conf): util.del_dir(os.path.dirname(config_file)) raise error + +def load_cloudinit_data(md_path, ud_path): + """ + Load the cloud-init meta data, user data, cfg and network from the + given files + + @return: 4-tuple of configuration + metadata, userdata, cfg={}, network + + @raises: FileNotFoundError if md_path or ud_path are absent + """ + LOG.debug('load meta data from: %s: user data from: %s', + md_path, ud_path) + md = {} + ud = None + network = None + + md = safeload_yaml_or_dict(util.load_file(md_path)) + + if 'network' in md: + network = md['network'] + + if ud_path: + ud = util.load_file(ud_path).replace("\r", "") + return md, ud, {}, network + + +def safeload_yaml_or_dict(data): + ''' + The meta data could be JSON or YAML. Since YAML is a strict superset of + JSON, we will unmarshal the data as YAML. If data is None then a new + dictionary is returned. + ''' + if not data: + return {} + return safeyaml.load(data) + + +def collect_imc_file_paths(cust_conf): + ''' + collect all the other imc files. + + metadata is preferred to nics.txt configuration data. + + If metadata file exists because it is specified in customization + configuration, then metadata is required and userdata is optional. + + @return a 3-tuple containing desired configuration file paths if present + Expected returns: + 1. user provided metadata and userdata (md_path, ud_path, None) + 2. user provided metadata (md_path, None, None) + 3. user-provided network config (None, None, nics_path) + 4. No config found (None, None, None) + ''' + md_path = None + ud_path = None + nics_path = None + md_file = cust_conf.meta_data_name + if md_file: + md_path = os.path.join(VMWARE_IMC_DIR, md_file) + if not os.path.exists(md_path): + raise FileNotFoundError("meta data file is not found: %s" + % md_path) + + ud_file = cust_conf.user_data_name + if ud_file: + ud_path = os.path.join(VMWARE_IMC_DIR, ud_file) + if not os.path.exists(ud_path): + raise FileNotFoundError("user data file is not found: %s" + % ud_path) + else: + nics_path = os.path.join(VMWARE_IMC_DIR, "nics.txt") + if not os.path.exists(nics_path): + LOG.debug('%s does not exist.', nics_path) + nics_path = None + + return md_path, ud_path, nics_path + + # vi: ts=4 expandtab diff --git a/cloudinit/sources/helpers/vmware/imc/config.py b/cloudinit/sources/helpers/vmware/imc/config.py index 7109aef3..bdfab5a0 100644 --- a/cloudinit/sources/helpers/vmware/imc/config.py +++ b/cloudinit/sources/helpers/vmware/imc/config.py @@ -27,6 +27,8 @@ class Config(object): UTC = 'DATETIME|UTC' POST_GC_STATUS = 'MISC|POST-GC-STATUS' DEFAULT_RUN_POST_SCRIPT = 'MISC|DEFAULT-RUN-POST-CUST-SCRIPT' + CLOUDINIT_META_DATA = 'CLOUDINIT|METADATA' + CLOUDINIT_USER_DATA = 'CLOUDINIT|USERDATA' def __init__(self, configFile): self._configFile = configFile @@ -130,4 +132,14 @@ class Config(object): raise ValueError('defaultRunPostScript value should be yes/no') return defaultRunPostScript == 'yes' + @property + def meta_data_name(self): + """Return the name of cloud-init meta data.""" + return self._configFile.get(Config.CLOUDINIT_META_DATA, None) + + @property + def user_data_name(self): + """Return the name of cloud-init user data.""" + return self._configFile.get(Config.CLOUDINIT_USER_DATA, None) + # vi: ts=4 expandtab diff --git a/cloudinit/sources/helpers/vmware/imc/guestcust_error.py b/cloudinit/sources/helpers/vmware/imc/guestcust_error.py index 65ae7390..96d839b8 100644 --- a/cloudinit/sources/helpers/vmware/imc/guestcust_error.py +++ b/cloudinit/sources/helpers/vmware/imc/guestcust_error.py @@ -11,5 +11,6 @@ class GuestCustErrorEnum(object): GUESTCUST_ERROR_SUCCESS = 0 GUESTCUST_ERROR_SCRIPT_DISABLED = 6 + GUESTCUST_ERROR_WRONG_META_FORMAT = 9 # vi: ts=4 expandtab diff --git a/cloudinit/ssh_util.py b/cloudinit/ssh_util.py index d5113996..c08042d6 100644 --- a/cloudinit/ssh_util.py +++ b/cloudinit/ssh_util.py @@ -262,13 +262,13 @@ def extract_authorized_keys(username, sshd_cfg_file=DEF_SSHD_CFG): except (IOError, OSError): # Give up and use a default key filename - auth_key_fns.append(default_authorizedkeys_file) + auth_key_fns[0] = default_authorizedkeys_file util.logexc(LOG, "Failed extracting 'AuthorizedKeysFile' in SSH " "config from %r, using 'AuthorizedKeysFile' file " "%r instead", DEF_SSHD_CFG, auth_key_fns[0]) - # always store all the keys in the first file configured on sshd_config - return (auth_key_fns[0], parse_authorized_keys(auth_key_fns)) + # always store all the keys in the user's private file + return (default_authorizedkeys_file, parse_authorized_keys(auth_key_fns)) def setup_user_keys(keys, username, options=None): diff --git a/debian/changelog b/debian/changelog index d6e39d98..4f1d5b9e 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,22 @@ +cloud-init (20.4-66-gad23441d-0ubuntu1) hirsute; urgency=medium + + * New upstream snapshot. + - Add antonyc to .github-cla-signers (#747) [Anton Chaporgin] + - integration_tests: log image serial if available (#772) + - Revert "ssh_util: handle non-default AuthorizedKeysFile config (#586)" + (#775) (LP: #1911680) + - [VMware] Support cloudinit raw data feature (#691) [xiaofengw-vmware] + - net: Fix static routes to host in eni renderer (#668) [Pavel Abalikhin] + - .travis.yml: don't run cloud_tests in CI (#756) + - test_upgrade: add some missing commas (#769) + - cc_seed_random: update documentation and fix integration test (#771) + (LP: #1911227) + - Fix test gh-632 test to only run on NoCloud (#770) (LP: #1911230) + - archlinux: fix package upgrade command handling (#768) [Bao Trinh] + - integration_tests: add integration test for LP: #1910835 (#761) + + -- Daniel Watkins Wed, 20 Jan 2021 13:15:55 -0500 + cloud-init (20.4-55-g4f62ae8d-0ubuntu1) hirsute; urgency=medium * New upstream snapshot. @@ -16,7 +35,7 @@ cloud-init (20.4-55-g4f62ae8d-0ubuntu1) hirsute; urgency=medium - integration_tests: port ca_certs tests from cloud_tests (#732) - Azure: Add telemetry for poll IMDS (#741) [Johnson Shi] - doc: move testing section from HACKING to its own doc (#739) - - No longer allow integration test failures on travis (#738) [James Falcon] + - No longer allow integration test failures on travis (#738) - stale: fix error in definition (#740) - integration_tests: set log-cli-level to INFO by default (#737) - PULL_REQUEST_TEMPLATE.md: use backticks around commit message (#736) @@ -29,44 +48,40 @@ cloud-init (20.4-55-g4f62ae8d-0ubuntu1) hirsute; urgency=medium [Thomas Stringer] - doc: fix CloudStack configuration example (#707) [Olivier Lemasle] - integration_tests: restrict test_lxd_bridge appropriately (#730) - - Add integration tests for CLI functionality (#729) [James Falcon] - - Integration test for gh-626 (#728) [James Falcon] - - Some test_upgrade fixes (#726) [James Falcon] + - Add integration tests for CLI functionality (#729) + - Integration test for gh-626 (#728) + - Some test_upgrade fixes (#726) - Ensure overriding test vars with env vars works for booleans (#727) - [James Falcon] - integration_tests: port lxd_bridge test from cloud_tests (#718) - - Integration test for gh-632. (#725) [James Falcon] - - Integration test for gh-671 (#724) [James Falcon] + - Integration test for gh-632. (#725) + - Integration test for gh-671 (#724) - integration-requirements.txt: bump pycloudlib commit (#723) - Drop unnecessary shebang from cmd/main.py (#722) [Eduardo Otubo] - - Integration test for LP: #1813396 and #669 (#719) [James Falcon] + - Integration test for LP: #1813396 and #669 (#719) - integration_tests: include timestamp in log output (#720) - integration_tests: add test for LP: #1898997 (#713) - - Add integration test for power_state_change module (#717) [James Falcon] + - Add integration test for power_state_change module (#717) - Update documentation for network-config-format-v2 (#701) [ggiesen] - sandbox CA Cert tests to not require ca-certificates (#715) [Eduardo Otubo] - - Add upgrade integration test (#693) [James Falcon] - - Integration test for 570 (#712) [James Falcon] + - Add upgrade integration test (#693) + - Integration test for 570 (#712) - Add ability to keep snapshotted images in integration tests (#711) - [James Falcon] - - Integration test for pull #586 (#706) [James Falcon] + - Integration test for pull #586 (#706) - integration_tests: introduce skipping of tests by OS (#702) - integration_tests: introduce IntegrationInstance.restart (#708) - Add lxd-vm to list of valid integration test platforms (#705) - [James Falcon] - Adding BOOTPROTO = dhcp to render sysconfig dhcp6 stateful on RHEL (#685) [Eduardo Otubo] - Delete image snapshots created for integration tests (#682) - [James Falcon] - Parametrize ssh_keys_provided integration test (#700) [lucasmoura] - Drop use_sudo attribute on IntegrationInstance (#694) [lucasmoura] - cc_apt_configure: add riscv64 as a ports arch (#687) [Dimitri John Ledkov] - cla: add xnox (#692) [Dimitri John Ledkov] - - Collect logs from integration test runs (#675) [James Falcon] + - Collect logs from integration test runs (#675) - -- Daniel Watkins Mon, 11 Jan 2021 16:50:12 -0500 + -- Daniel Watkins Mon, 11 Jan 2021 16:57:30 -0500 cloud-init (20.4-0ubuntu2) hirsute; urgency=medium diff --git a/tests/integration_tests/bugs/test_gh632.py b/tests/integration_tests/bugs/test_gh632.py index cf166356..3c1f9347 100644 --- a/tests/integration_tests/bugs/test_gh632.py +++ b/tests/integration_tests/bugs/test_gh632.py @@ -9,6 +9,9 @@ import pytest from tests.integration_tests.instances import IntegrationInstance +# With some datasource hacking, we can run this on a NoCloud instance +@pytest.mark.lxd_container +@pytest.mark.lxd_vm @pytest.mark.sru_2020_11 def test_datasource_rbx_no_stacktrace(client: IntegrationInstance): client.write_to_file( diff --git a/tests/integration_tests/bugs/test_gh668.py b/tests/integration_tests/bugs/test_gh668.py new file mode 100644 index 00000000..a3a0c374 --- /dev/null +++ b/tests/integration_tests/bugs/test_gh668.py @@ -0,0 +1,37 @@ +"""Integration test for gh-668. + +Ensure that static route to host is working correctly. +The original problem is specific to the ENI renderer but that test is suitable +for all network configuration outputs. +""" + +import pytest + +from tests.integration_tests.instances import IntegrationInstance + + +DESTINATION_IP = "172.16.0.10" +GATEWAY_IP = "10.0.0.100" + +NETWORK_CONFIG = """\ +version: 2 +ethernets: + eth0: + addresses: [10.0.0.10/8] + dhcp4: false + routes: + - to: {}/32 + via: {} +""".format(DESTINATION_IP, GATEWAY_IP) + +EXPECTED_ROUTE = "{} via {}".format(DESTINATION_IP, GATEWAY_IP) + + +@pytest.mark.lxd_container +@pytest.mark.lxd_vm +@pytest.mark.lxd_config_dict({ + "user.network-config": NETWORK_CONFIG, +}) +def test_static_route_to_host(client: IntegrationInstance): + route = client.execute("ip route | grep {}".format(DESTINATION_IP)) + assert route.startswith(EXPECTED_ROUTE) diff --git a/tests/integration_tests/bugs/test_lp1910835.py b/tests/integration_tests/bugs/test_lp1910835.py new file mode 100644 index 00000000..87f92d5e --- /dev/null +++ b/tests/integration_tests/bugs/test_lp1910835.py @@ -0,0 +1,66 @@ +"""Integration test for LP: #1910835. + +If users do not provide an SSH key and instead ask Azure to generate a key for +them, the key material available in the IMDS may include CRLF sequences. Prior +to e56b55452549cb037da0a4165154ffa494e9678a, the Azure datasource handled keys +via a certificate, the tooling for which removed these sequences. This test +ensures that cloud-init does not regress support for this Azure behaviour. + +This test provides the SSH key configured for tests to the instance in two +ways: firstly, with CRLFs to mimic the generated keys, via the Azure API; +secondly, as user-data in unmodified form. This means that even on systems +which exhibit the bug fetching the platform's metadata, we can SSH into the SUT +to confirm this (instead of having to assert SSH failure; there are lots of +reasons SSH might fail). + +Once SSH'd in, we check that the two keys in .ssh/authorized_keys have the same +material: if the Azure datasource has removed the CRLFs correctly, then they +will match. +""" +import pytest + + +USER_DATA_TMPL = """\ +#cloud-config +ssh_authorized_keys: + - {}""" + + +@pytest.mark.sru_2021_01 +@pytest.mark.azure +def test_crlf_in_azure_metadata_ssh_keys(session_cloud, setup_image): + authorized_keys_path = "/home/{}/.ssh/authorized_keys".format( + session_cloud.cloud_instance.username + ) + # Pass in user-data to allow us to access the instance when the normal + # path fails + key_data = session_cloud.cloud_instance.key_pair.public_key_content + user_data = USER_DATA_TMPL.format(key_data) + # Throw a CRLF into the otherwise good key data, to emulate Azure's + # behaviour for generated keys + key_data = key_data[:20] + "\r\n" + key_data[20:] + vm_params = { + "os_profile": { + "linux_configuration": { + "ssh": { + "public_keys": [ + {"path": authorized_keys_path, "key_data": key_data} + ] + } + } + } + } + with session_cloud.launch( + launch_kwargs={"vm_params": vm_params, "user_data": user_data} + ) as client: + authorized_keys = ( + client.read_from_file(authorized_keys_path).strip().splitlines() + ) + # We expect one key from the cloud, one from user-data + assert 2 == len(authorized_keys) + # And those two keys should be the same, except for a possible key + # comment, which Azure strips out + assert ( + authorized_keys[0].rsplit(" ")[:2] + == authorized_keys[1].split(" ")[:2] + ) diff --git a/tests/integration_tests/clouds.py b/tests/integration_tests/clouds.py index 72d77058..fc0a61d5 100644 --- a/tests/integration_tests/clouds.py +++ b/tests/integration_tests/clouds.py @@ -173,6 +173,9 @@ class IntegrationCloud(ABC): 'cloud-init version: %s', instance.execute("cloud-init --version") ) + serial = instance.execute("grep serial /etc/cloud/build.info") + if serial: + log.info('image serial: %s', serial.split()[1]) return instance def get_instance(self, cloud_instance, settings=integration_settings): diff --git a/tests/integration_tests/modules/test_seed_random_data.py b/tests/integration_tests/modules/test_seed_random_data.py index b365fa98..f6a67c19 100644 --- a/tests/integration_tests/modules/test_seed_random_data.py +++ b/tests/integration_tests/modules/test_seed_random_data.py @@ -25,4 +25,4 @@ class TestSeedRandomData: @pytest.mark.user_data(USER_DATA) def test_seed_random_data(self, client): seed_output = client.read_from_file("/root/seed") - assert seed_output.strip() == "MYUb34023nD:LFDK10913jk;dfnk:Df" + assert seed_output.startswith("MYUb34023nD:LFDK10913jk;dfnk:Df") diff --git a/tests/integration_tests/test_upgrade.py b/tests/integration_tests/test_upgrade.py index a115bea7..660d363f 100644 --- a/tests/integration_tests/test_upgrade.py +++ b/tests/integration_tests/test_upgrade.py @@ -26,9 +26,9 @@ def _output_to_compare(instance, file_path, netcfg_path): # have broken across re-constitution of a cached datasource. Some # platforms invalidate their datasource cache on reboot, so we run # it here to ensure we get a dirty run. - 'cloud-init init' + 'cloud-init init', 'grep Trace /var/log/cloud-init.log', - 'cloud-id' + 'cloud-id', 'cat {}'.format(netcfg_path), 'systemd-analyze', 'systemd-analyze blame', diff --git a/tests/unittests/test_datasource/test_ovf.py b/tests/unittests/test_datasource/test_ovf.py index 16773de5..dce01f5d 100644 --- a/tests/unittests/test_datasource/test_ovf.py +++ b/tests/unittests/test_datasource/test_ovf.py @@ -17,6 +17,7 @@ from cloudinit.helpers import Paths from cloudinit.sources import DataSourceOVF as dsovf from cloudinit.sources.helpers.vmware.imc.config_custom_script import ( CustomScriptNotFound) +from cloudinit.safeyaml import YAMLError MPATH = 'cloudinit.sources.DataSourceOVF.' @@ -138,16 +139,29 @@ class TestDatasourceOVF(CiTestCase): 'DEBUG: No system-product-name found', self.logs.getvalue()) def test_get_data_no_vmware_customization_disabled(self): - """When vmware customization is disabled via sys_cfg log a message.""" + """When cloud-init workflow for vmware is disabled via sys_cfg and + no meta data provided, log a message. + """ paths = Paths({'cloud_dir': self.tdir}) ds = self.datasource( sys_cfg={'disable_vmware_customization': True}, distro={}, paths=paths) + conf_file = self.tmp_path('test-cust', self.tdir) + conf_content = dedent("""\ + [CUSTOM-SCRIPT] + SCRIPT-NAME = test-script + [MISC] + MARKER-ID = 12345345 + """) + util.write_file(conf_file, conf_content) retcode = wrap_and_call( 'cloudinit.sources.DataSourceOVF', {'dmi.read_dmi_data': 'vmware', 'transport_iso9660': NOT_FOUND, - 'transport_vmware_guestinfo': NOT_FOUND}, + 'transport_vmware_guestinfo': NOT_FOUND, + 'util.del_dir': True, + 'search_file': self.tdir, + 'wait_for_imc_cfg_file': conf_file}, ds.get_data) self.assertFalse(retcode, 'Expected False return from ds.get_data') self.assertIn( @@ -344,6 +358,279 @@ class TestDatasourceOVF(CiTestCase): 'vmware (%s/seed/ovf-env.xml)' % self.tdir, ds.subplatform) + def test_get_data_cloudinit_metadata_json(self): + """Test metadata can be loaded to cloud-init metadata and network. + The metadata format is json. + """ + paths = Paths({'cloud_dir': self.tdir}) + ds = self.datasource( + sys_cfg={'disable_vmware_customization': True}, distro={}, + paths=paths) + # Prepare the conf file + conf_file = self.tmp_path('test-cust', self.tdir) + conf_content = dedent("""\ + [CLOUDINIT] + METADATA = test-meta + """) + util.write_file(conf_file, conf_content) + # Prepare the meta data file + metadata_file = self.tmp_path('test-meta', self.tdir) + metadata_content = dedent("""\ + { + "instance-id": "cloud-vm", + "local-hostname": "my-host.domain.com", + "network": { + "version": 2, + "ethernets": { + "eths": { + "match": { + "name": "ens*" + }, + "dhcp4": true + } + } + } + } + """) + util.write_file(metadata_file, metadata_content) + + with mock.patch(MPATH + 'set_customization_status', + return_value=('msg', b'')): + result = wrap_and_call( + 'cloudinit.sources.DataSourceOVF', + {'dmi.read_dmi_data': 'vmware', + 'util.del_dir': True, + 'search_file': self.tdir, + 'wait_for_imc_cfg_file': conf_file, + 'collect_imc_file_paths': [self.tdir + '/test-meta', '', ''], + 'get_nics_to_enable': ''}, + ds._get_data) + + self.assertTrue(result) + self.assertEqual("cloud-vm", ds.metadata['instance-id']) + self.assertEqual("my-host.domain.com", ds.metadata['local-hostname']) + self.assertEqual(2, ds.network_config['version']) + self.assertTrue(ds.network_config['ethernets']['eths']['dhcp4']) + + def test_get_data_cloudinit_metadata_yaml(self): + """Test metadata can be loaded to cloud-init metadata and network. + The metadata format is yaml. + """ + paths = Paths({'cloud_dir': self.tdir}) + ds = self.datasource( + sys_cfg={'disable_vmware_customization': True}, distro={}, + paths=paths) + # Prepare the conf file + conf_file = self.tmp_path('test-cust', self.tdir) + conf_content = dedent("""\ + [CLOUDINIT] + METADATA = test-meta + """) + util.write_file(conf_file, conf_content) + # Prepare the meta data file + metadata_file = self.tmp_path('test-meta', self.tdir) + metadata_content = dedent("""\ + instance-id: cloud-vm + local-hostname: my-host.domain.com + network: + version: 2 + ethernets: + nics: + match: + name: ens* + dhcp4: yes + """) + util.write_file(metadata_file, metadata_content) + + with mock.patch(MPATH + 'set_customization_status', + return_value=('msg', b'')): + result = wrap_and_call( + 'cloudinit.sources.DataSourceOVF', + {'dmi.read_dmi_data': 'vmware', + 'util.del_dir': True, + 'search_file': self.tdir, + 'wait_for_imc_cfg_file': conf_file, + 'collect_imc_file_paths': [self.tdir + '/test-meta', '', ''], + 'get_nics_to_enable': ''}, + ds._get_data) + + self.assertTrue(result) + self.assertEqual("cloud-vm", ds.metadata['instance-id']) + self.assertEqual("my-host.domain.com", ds.metadata['local-hostname']) + self.assertEqual(2, ds.network_config['version']) + self.assertTrue(ds.network_config['ethernets']['nics']['dhcp4']) + + def test_get_data_cloudinit_metadata_not_valid(self): + """Test metadata is not JSON or YAML format. + """ + paths = Paths({'cloud_dir': self.tdir}) + ds = self.datasource( + sys_cfg={'disable_vmware_customization': True}, distro={}, + paths=paths) + + # Prepare the conf file + conf_file = self.tmp_path('test-cust', self.tdir) + conf_content = dedent("""\ + [CLOUDINIT] + METADATA = test-meta + """) + util.write_file(conf_file, conf_content) + + # Prepare the meta data file + metadata_file = self.tmp_path('test-meta', self.tdir) + metadata_content = "[This is not json or yaml format]a=b" + util.write_file(metadata_file, metadata_content) + + with mock.patch(MPATH + 'set_customization_status', + return_value=('msg', b'')): + with self.assertRaises(YAMLError) as context: + wrap_and_call( + 'cloudinit.sources.DataSourceOVF', + {'dmi.read_dmi_data': 'vmware', + 'util.del_dir': True, + 'search_file': self.tdir, + 'wait_for_imc_cfg_file': conf_file, + 'collect_imc_file_paths': [ + self.tdir + '/test-meta', '', '' + ], + 'get_nics_to_enable': ''}, + ds.get_data) + + self.assertIn("expected '', but found ''", + str(context.exception)) + + def test_get_data_cloudinit_metadata_not_found(self): + """Test metadata file can't be found. + """ + paths = Paths({'cloud_dir': self.tdir}) + ds = self.datasource( + sys_cfg={'disable_vmware_customization': True}, distro={}, + paths=paths) + # Prepare the conf file + conf_file = self.tmp_path('test-cust', self.tdir) + conf_content = dedent("""\ + [CLOUDINIT] + METADATA = test-meta + """) + util.write_file(conf_file, conf_content) + # Don't prepare the meta data file + + with mock.patch(MPATH + 'set_customization_status', + return_value=('msg', b'')): + with self.assertRaises(FileNotFoundError) as context: + wrap_and_call( + 'cloudinit.sources.DataSourceOVF', + {'dmi.read_dmi_data': 'vmware', + 'util.del_dir': True, + 'search_file': self.tdir, + 'wait_for_imc_cfg_file': conf_file, + 'get_nics_to_enable': ''}, + ds.get_data) + + self.assertIn('is not found', str(context.exception)) + + def test_get_data_cloudinit_userdata(self): + """Test user data can be loaded to cloud-init user data. + """ + paths = Paths({'cloud_dir': self.tdir}) + ds = self.datasource( + sys_cfg={'disable_vmware_customization': False}, distro={}, + paths=paths) + + # Prepare the conf file + conf_file = self.tmp_path('test-cust', self.tdir) + conf_content = dedent("""\ + [CLOUDINIT] + METADATA = test-meta + USERDATA = test-user + """) + util.write_file(conf_file, conf_content) + + # Prepare the meta data file + metadata_file = self.tmp_path('test-meta', self.tdir) + metadata_content = dedent("""\ + instance-id: cloud-vm + local-hostname: my-host.domain.com + network: + version: 2 + ethernets: + nics: + match: + name: ens* + dhcp4: yes + """) + util.write_file(metadata_file, metadata_content) + + # Prepare the user data file + userdata_file = self.tmp_path('test-user', self.tdir) + userdata_content = "This is the user data" + util.write_file(userdata_file, userdata_content) + + with mock.patch(MPATH + 'set_customization_status', + return_value=('msg', b'')): + result = wrap_and_call( + 'cloudinit.sources.DataSourceOVF', + {'dmi.read_dmi_data': 'vmware', + 'util.del_dir': True, + 'search_file': self.tdir, + 'wait_for_imc_cfg_file': conf_file, + 'collect_imc_file_paths': [self.tdir + '/test-meta', + self.tdir + '/test-user', ''], + 'get_nics_to_enable': ''}, + ds._get_data) + + self.assertTrue(result) + self.assertEqual("cloud-vm", ds.metadata['instance-id']) + self.assertEqual(userdata_content, ds.userdata_raw) + + def test_get_data_cloudinit_userdata_not_found(self): + """Test userdata file can't be found. + """ + paths = Paths({'cloud_dir': self.tdir}) + ds = self.datasource( + sys_cfg={'disable_vmware_customization': True}, distro={}, + paths=paths) + + # Prepare the conf file + conf_file = self.tmp_path('test-cust', self.tdir) + conf_content = dedent("""\ + [CLOUDINIT] + METADATA = test-meta + USERDATA = test-user + """) + util.write_file(conf_file, conf_content) + + # Prepare the meta data file + metadata_file = self.tmp_path('test-meta', self.tdir) + metadata_content = dedent("""\ + instance-id: cloud-vm + local-hostname: my-host.domain.com + network: + version: 2 + ethernets: + nics: + match: + name: ens* + dhcp4: yes + """) + util.write_file(metadata_file, metadata_content) + + # Don't prepare the user data file + + with mock.patch(MPATH + 'set_customization_status', + return_value=('msg', b'')): + with self.assertRaises(FileNotFoundError) as context: + wrap_and_call( + 'cloudinit.sources.DataSourceOVF', + {'dmi.read_dmi_data': 'vmware', + 'util.del_dir': True, + 'search_file': self.tdir, + 'wait_for_imc_cfg_file': conf_file, + 'get_nics_to_enable': ''}, + ds.get_data) + + self.assertIn('is not found', str(context.exception)) + class TestTransportIso9660(CiTestCase): diff --git a/tests/unittests/test_net.py b/tests/unittests/test_net.py index bf0cdabb..38d934d4 100644 --- a/tests/unittests/test_net.py +++ b/tests/unittests/test_net.py @@ -4820,6 +4820,9 @@ class TestEniRoundTrip(CiTestCase): {'type': 'route', 'id': 6, 'metric': 1, 'destination': '10.0.200.0/16', 'gateway': '172.23.31.1'}, + {'type': 'route', 'id': 7, + 'metric': 1, 'destination': '10.0.0.100/32', + 'gateway': '172.23.31.1'}, ] files = self._render_and_read( @@ -4843,6 +4846,10 @@ class TestEniRoundTrip(CiTestCase): '172.23.31.1 metric 1 || true'), ('pre-down route del -net 10.0.200.0/16 gw ' '172.23.31.1 metric 1 || true'), + ('post-up route add -host 10.0.0.100/32 gw ' + '172.23.31.1 metric 1 || true'), + ('pre-down route del -host 10.0.0.100/32 gw ' + '172.23.31.1 metric 1 || true'), ] found = files['/etc/network/interfaces'].splitlines() diff --git a/tests/unittests/test_sshutil.py b/tests/unittests/test_sshutil.py index 88a111e3..fd1d1bac 100644 --- a/tests/unittests/test_sshutil.py +++ b/tests/unittests/test_sshutil.py @@ -593,7 +593,7 @@ class TestMultipleSshAuthorizedKeysFile(test_helpers.CiTestCase): fpw.pw_name, sshd_config) content = ssh_util.update_authorized_keys(auth_key_entries, []) - self.assertEqual(authorized_keys, auth_key_fn) + self.assertEqual("%s/.ssh/authorized_keys" % fpw.pw_dir, auth_key_fn) self.assertTrue(VALID_CONTENT['rsa'] in content) self.assertTrue(VALID_CONTENT['dsa'] in content) @@ -610,7 +610,7 @@ class TestMultipleSshAuthorizedKeysFile(test_helpers.CiTestCase): sshd_config = self.tmp_path('sshd_config') util.write_file( sshd_config, - "AuthorizedKeysFile %s %s" % (user_keys, authorized_keys) + "AuthorizedKeysFile %s %s" % (authorized_keys, user_keys) ) (auth_key_fn, auth_key_entries) = ssh_util.extract_authorized_keys( @@ -618,7 +618,7 @@ class TestMultipleSshAuthorizedKeysFile(test_helpers.CiTestCase): ) content = ssh_util.update_authorized_keys(auth_key_entries, []) - self.assertEqual(user_keys, auth_key_fn) + self.assertEqual("%s/.ssh/authorized_keys" % fpw.pw_dir, auth_key_fn) self.assertTrue(VALID_CONTENT['rsa'] in content) self.assertTrue(VALID_CONTENT['dsa'] in content) diff --git a/tests/unittests/test_vmware_config_file.py b/tests/unittests/test_vmware_config_file.py index 9c7d25fa..430cc69f 100644 --- a/tests/unittests/test_vmware_config_file.py +++ b/tests/unittests/test_vmware_config_file.py @@ -525,5 +525,21 @@ class TestVmwareNetConfig(CiTestCase): 'gateway': '10.20.87.253'}]}], nc.generate()) + def test_meta_data(self): + cf = ConfigFile("tests/data/vmware/cust-dhcp-2nic.cfg") + conf = Config(cf) + self.assertIsNone(conf.meta_data_name) + cf._insertKey("CLOUDINIT|METADATA", "test-metadata") + conf = Config(cf) + self.assertEqual("test-metadata", conf.meta_data_name) + + def test_user_data(self): + cf = ConfigFile("tests/data/vmware/cust-dhcp-2nic.cfg") + conf = Config(cf) + self.assertIsNone(conf.user_data_name) + cf._insertKey("CLOUDINIT|USERDATA", "test-userdata") + conf = Config(cf) + self.assertEqual("test-userdata", conf.user_data_name) + # vi: ts=4 expandtab diff --git a/tools/.github-cla-signers b/tools/.github-cla-signers index d6ca6d1b..4cee82ac 100644 --- a/tools/.github-cla-signers +++ b/tools/.github-cla-signers @@ -2,6 +2,7 @@ ader1990 ajmyyra AlexBaranowski Aman306 +antonyc aswinrajamannar beezly bipinbachhao @@ -26,11 +27,13 @@ nishigori olivierlemasle omBratteng onitake +qubidt riedel slyon smoser sshedi TheRealFalcon +tnt-dev tomponline tsanghan WebSpider diff --git a/tox.ini b/tox.ini index 7c71f0f6..9444e66c 100644 --- a/tox.ini +++ b/tox.ini @@ -184,5 +184,6 @@ markers = user_data: the user data to be passed to the test instance instance_name: the name to be used for the test instance sru_2020_11: test is part of the 2020/11 SRU verification + sru_2021_01: test is part of the 2021/01 SRU verification ubuntu: this test should run on Ubuntu unstable: skip this test because it is flakey -- cgit v1.2.1