21.1-19-gbad84ad4-0ubuntu1 (patches unapplied)

Imported using git-ubuntu import.

36 files changed, 956 insertions, 116 deletions

diff --git a/.travis.yml b/.travis.yml
index f15752bc..690ab644 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -120,6 +120,7 @@ matrix:
                 fi
             # Use sudo to get a new shell where we're in the sbuild group
             - sudo -E su $USER -c 'sbuild --nolog --no-run-lintian --verbose --dist=xenial cloud-init_*.dsc'
+            - ssh-keygen -P "" -q -f ~/.ssh/id_rsa
             - sg lxd -c 'CLOUD_INIT_CLOUD_INIT_SOURCE="$(ls *.deb)" tox -e integration-tests-ci' &
             - |
               SECONDS=0
diff --git a/cloudinit/config/cc_keys_to_console.py b/cloudinit/config/cc_keys_to_console.py
index 646d1f67..d72b5244 100644
--- a/cloudinit/config/cc_keys_to_console.py
+++ b/cloudinit/config/cc_keys_to_console.py
@@ -9,14 +9,17 @@
 """
 Keys to Console
 ---------------
-**Summary:** control which SSH keys may be written to console
-
-For security reasons it may be desirable not to write SSH fingerprints and keys
-to the console. To avoid the fingerprint of types of SSH keys being written to
-console the ``ssh_fp_console_blacklist`` config key can be used. By default all
-types of keys will have their fingerprints written to console. To avoid keys
-of a key type being written to console the ``ssh_key_console_blacklist`` config
-key can be used. By default ``ssh-dss`` keys are not written to console.
+**Summary:** control which SSH host keys may be written to console
+
+For security reasons it may be desirable not to write SSH host keys and their
+fingerprints to the console. To avoid either being written to the console the
+``emit_keys_to_console`` config key under the main ``ssh`` config key can be
+used. To avoid the fingerprint of types of SSH host keys being written to
+console the ``ssh_fp_console_blacklist`` config key can be used. By default
+all types of keys will have their fingerprints written to console. To avoid
+host keys of a key type being written to console the
+``ssh_key_console_blacklist`` config key can be used. By default ``ssh-dss``
+host keys are not written to console.
 
 **Internal name:** ``cc_keys_to_console``
 
@@ -26,6 +29,9 @@ key can be used. By default ``ssh-dss`` keys are not written to console.
 
 **Config keys**::
 
+    ssh:
+      emit_keys_to_console: false
+
     ssh_fp_console_blacklist: <list of key types>
     ssh_key_console_blacklist: <list of key types>
 """
diff --git a/cloudinit/config/cc_set_hostname.py b/cloudinit/config/cc_set_hostname.py
index 1d23d80d..d4017478 100644
--- a/cloudinit/config/cc_set_hostname.py
+++ b/cloudinit/config/cc_set_hostname.py
@@ -18,8 +18,8 @@ A hostname and fqdn can be provided by specifying a full domain name under the
 ``fqdn`` key. Alternatively, a hostname can be specified using the ``hostname``
 key, and the fqdn of the cloud wil be used. If a fqdn specified with the
 ``hostname`` key, it will be handled properly, although it is better to use
-the ``fqdn`` config key. If both ``fqdn`` and ``hostname`` are set, ``fqdn``
-will be used.
+the ``fqdn`` config key. If both ``fqdn`` and ``hostname`` are set,
+it is distro dependent whether ``hostname`` or ``fqdn`` is used.
 
 This module will run in the init-local stage before networking is configured
 if the hostname is set by metadata or user data on the local system.
diff --git a/cloudinit/config/cc_set_passwords.py b/cloudinit/config/cc_set_passwords.py
index d6b5682d..433de751 100755
--- a/cloudinit/config/cc_set_passwords.py
+++ b/cloudinit/config/cc_set_passwords.py
@@ -78,7 +78,6 @@ password.
 """
 
 import re
-import sys
 
 from cloudinit.distros import ug_util
 from cloudinit import log as logging
@@ -214,7 +213,9 @@ def handle(_name, cfg, cloud, log, args):
         if len(randlist):
             blurb = ("Set the following 'random' passwords\n",
                      '\n'.join(randlist))
-            sys.stderr.write("%s\n%s\n" % blurb)
+            util.multi_log(
+                "%s\n%s\n" % blurb, stderr=False, fallback_to_stdout=False
+            )
 
         if expire:
             expired_users = []
diff --git a/cloudinit/config/tests/test_set_passwords.py b/cloudinit/config/tests/test_set_passwords.py
index daa1ef51..bbe2ee8f 100644
--- a/cloudinit/config/tests/test_set_passwords.py
+++ b/cloudinit/config/tests/test_set_passwords.py
@@ -74,10 +74,6 @@ class TestSetPasswordsHandle(CiTestCase):
 
     with_logs = True
 
-    def setUp(self):
-        super(TestSetPasswordsHandle, self).setUp()
-        self.add_patch('cloudinit.config.cc_set_passwords.sys.stderr', 'm_err')
-
     def test_handle_on_empty_config(self, *args):
         """handle logs that no password has changed when config is empty."""
         cloud = self.tmp_cloud(distro='ubuntu')
@@ -129,10 +125,12 @@ class TestSetPasswordsHandle(CiTestCase):
             mock.call(['pw', 'usermod', 'ubuntu', '-p', '01-Jan-1970'])],
             m_subp.call_args_list)
 
+    @mock.patch(MODPATH + "util.multi_log")
     @mock.patch(MODPATH + "util.is_BSD")
     @mock.patch(MODPATH + "subp.subp")
-    def test_handle_on_chpasswd_list_creates_random_passwords(self, m_subp,
-                                                              m_is_bsd):
+    def test_handle_on_chpasswd_list_creates_random_passwords(
+        self, m_subp, m_is_bsd, m_multi_log
+    ):
         """handle parses command set random passwords."""
         m_is_bsd.return_value = False
         cloud = self.tmp_cloud(distro='ubuntu')
@@ -146,10 +144,32 @@ class TestSetPasswordsHandle(CiTestCase):
         self.assertIn(
             'DEBUG: Handling input for chpasswd as list.',
             self.logs.getvalue())
-        self.assertNotEqual(
-            [mock.call(['chpasswd'],
-             '\n'.join(valid_random_pwds) + '\n')],
-            m_subp.call_args_list)
+
+        self.assertEqual(1, m_subp.call_count)
+        args, _kwargs = m_subp.call_args
+        self.assertEqual(["chpasswd"], args[0])
+
+        stdin = args[1]
+        user_pass = {
+            user: password
+            for user, password
+            in (line.split(":") for line in stdin.splitlines())
+        }
+
+        self.assertEqual(1, m_multi_log.call_count)
+        self.assertEqual(
+            mock.call(mock.ANY, stderr=False, fallback_to_stdout=False),
+            m_multi_log.call_args
+        )
+
+        self.assertEqual(set(["root", "ubuntu"]), set(user_pass.keys()))
+        written_lines = m_multi_log.call_args[0][0].splitlines()
+        for password in user_pass.values():
+            for line in written_lines:
+                if password in line:
+                    break
+            else:
+                self.fail("Password not emitted to console")
 
 
 # vi: ts=4 expandtab
diff --git a/cloudinit/distros/arch.py b/cloudinit/distros/arch.py
index 378a6daa..f8385f7f 100644
--- a/cloudinit/distros/arch.py
+++ b/cloudinit/distros/arch.py
@@ -23,7 +23,7 @@ LOG = logging.getLogger(__name__)
 
 
 class Distro(distros.Distro):
-    locale_conf_fn = "/etc/locale.gen"
+    locale_gen_fn = "/etc/locale.gen"
     network_conf_dir = "/etc/netctl"
     resolve_conf_fn = "/etc/resolv.conf"
     init_cmd = ['systemctl']  # init scripts
@@ -43,16 +43,20 @@ class Distro(distros.Distro):
         cfg['ssh_svcname'] = 'sshd'
 
     def apply_locale(self, locale, out_fn=None):
-        if not out_fn:
-            out_fn = self.locale_conf_fn
-        subp.subp(['locale-gen', '-G', locale], capture=False)
-        # "" provides trailing newline during join
+        if out_fn is not None and out_fn != "/etc/locale.conf":
+            LOG.warning("Invalid locale_configfile %s, only supported "
+                        "value is /etc/locale.conf", out_fn)
         lines = [
             util.make_header(),
-            'LANG="%s"' % (locale),
+            # Hard-coding the charset isn't ideal, but there is no other way.
+            '%s UTF-8' % (locale),
             "",
         ]
-        util.write_file(out_fn, "\n".join(lines))
+        util.write_file(self.locale_gen_fn, "\n".join(lines))
+        subp.subp(['locale-gen'], capture=False)
+        # In the future systemd can handle locale-gen stuff:
+        # https://github.com/systemd/systemd/pull/9864
+        subp.subp(['localectl', 'set-locale', locale], capture=False)
 
     def install_packages(self, pkglist):
         self.update_package_sources()
@@ -137,6 +141,17 @@ class Distro(distros.Distro):
             return default
         return hostname
 
+    # hostname (inetutils) isn't installed per default on arch, so we use
+    # hostnamectl which is installed per default (systemd).
+    def _apply_hostname(self, hostname):
+        LOG.debug("Non-persistently setting the system hostname to %s",
+                  hostname)
+        try:
+            subp.subp(['hostnamectl', '--transient', 'set-hostname', hostname])
+        except subp.ProcessExecutionError:
+            util.logexc(LOG, "Failed to non-persistently adjust the system "
+                        "hostname to %s", hostname)
+
     def set_timezone(self, tz):
         distros.set_etc_timezone(tz=tz, tz_file=self._find_tz_file(tz))
 
diff --git a/cloudinit/net/__init__.py b/cloudinit/net/__init__.py
index de65e7af..385b7bcc 100644
--- a/cloudinit/net/__init__.py
+++ b/cloudinit/net/__init__.py
@@ -6,6 +6,7 @@
 # This file is part of cloud-init. See LICENSE file for license information.
 
 import errno
+import functools
 import ipaddress
 import logging
 import os
@@ -19,6 +20,19 @@ from cloudinit.url_helper import UrlError, readurl
 LOG = logging.getLogger(__name__)
 SYS_CLASS_NET = "/sys/class/net/"
 DEFAULT_PRIMARY_INTERFACE = 'eth0'
+OVS_INTERNAL_INTERFACE_LOOKUP_CMD = [
+    "ovs-vsctl",
+    "--format",
+    "csv",
+    "--no-headings",
+    "--timeout",
+    "10",
+    "--columns",
+    "name",
+    "find",
+    "interface",
+    "type=internal",
+]
 
 
 def natural_sort_key(s, _nsre=re.compile('([0-9]+)')):
@@ -133,6 +147,52 @@ def master_is_openvswitch(devname):
     return os.path.exists(ovs_path)
 
 
+@functools.lru_cache(maxsize=None)
+def openvswitch_is_installed() -> bool:
+    """Return a bool indicating if Open vSwitch is installed in the system."""
+    ret = bool(subp.which("ovs-vsctl"))
+    if not ret:
+        LOG.debug(
+            "ovs-vsctl not in PATH; not detecting Open vSwitch interfaces"
+        )
+    return ret
+
+
+@functools.lru_cache(maxsize=None)
+def get_ovs_internal_interfaces() -> list:
+    """Return a list of the names of OVS internal interfaces on the system.
+
+    These will all be strings, and are used to exclude OVS-specific interface
+    from cloud-init's network configuration handling.
+    """
+    try:
+        out, _err = subp.subp(OVS_INTERNAL_INTERFACE_LOOKUP_CMD)
+    except subp.ProcessExecutionError as exc:
+        if "database connection failed" in exc.stderr:
+            LOG.info(
+                "Open vSwitch is not yet up; no interfaces will be detected as"
+                " OVS-internal"
+            )
+            return []
+        raise
+    else:
+        return out.splitlines()
+
+
+def is_openvswitch_internal_interface(devname: str) -> bool:
+    """Returns True if this is an OVS internal interface.
+
+    If OVS is not installed or not yet running, this will return False.
+    """
+    if not openvswitch_is_installed():
+        return False
+    ovs_bridges = get_ovs_internal_interfaces()
+    if devname in ovs_bridges:
+        LOG.debug("Detected %s as an OVS interface", devname)
+        return True
+    return False
+
+
 def is_netfailover(devname, driver=None):
     """ netfailover driver uses 3 nics, master, primary and standby.
         this returns True if the device is either the primary or standby
@@ -884,6 +944,8 @@ def get_interfaces(blacklist_drivers=None) -> list:
         # skip nics that have no mac (00:00....)
         if name != 'lo' and mac == zero_mac[:len(mac)]:
             continue
+        if is_openvswitch_internal_interface(name):
+            continue
         # skip nics that have drivers blacklisted
         driver = device_driver(name)
         if driver in blacklist_drivers:
diff --git a/cloudinit/net/tests/test_init.py b/cloudinit/net/tests/test_init.py
index 0535387a..946f8ee2 100644
--- a/cloudinit/net/tests/test_init.py
+++ b/cloudinit/net/tests/test_init.py
@@ -391,6 +391,10 @@ class TestGetDeviceList(CiTestCase):
         self.assertCountEqual(['eth0', 'eth1'], net.get_devicelist())
 
 
+@mock.patch(
+    "cloudinit.net.is_openvswitch_internal_interface",
+    mock.Mock(return_value=False),
+)
 class TestGetInterfaceMAC(CiTestCase):
 
     def setUp(self):
@@ -1224,6 +1228,121 @@ class TestNetFailOver(CiTestCase):
         self.assertFalse(net.is_netfailover(devname, driver))
 
 
+class TestOpenvswitchIsInstalled:
+    """Test cloudinit.net.openvswitch_is_installed.
+
+    Uses the ``clear_lru_cache`` local autouse fixture to allow us to test
+    despite the ``lru_cache`` decorator on the unit under test.
+    """
+
+    @pytest.fixture(autouse=True)
+    def clear_lru_cache(self):
+        net.openvswitch_is_installed.cache_clear()
+
+    @pytest.mark.parametrize(
+        "expected,which_return", [(True, "/some/path"), (False, None)]
+    )
+    @mock.patch("cloudinit.net.subp.which")
+    def test_mirrors_which_result(self, m_which, expected, which_return):
+        m_which.return_value = which_return
+        assert expected == net.openvswitch_is_installed()
+
+    @mock.patch("cloudinit.net.subp.which")
+    def test_only_calls_which_once(self, m_which):
+        net.openvswitch_is_installed()
+        net.openvswitch_is_installed()
+        assert 1 == m_which.call_count
+
+
+@mock.patch("cloudinit.net.subp.subp", return_value=("", ""))
+class TestGetOVSInternalInterfaces:
+    """Test cloudinit.net.get_ovs_internal_interfaces.
+
+    Uses the ``clear_lru_cache`` local autouse fixture to allow us to test
+    despite the ``lru_cache`` decorator on the unit under test.
+    """
+    @pytest.fixture(autouse=True)
+    def clear_lru_cache(self):
+        net.get_ovs_internal_interfaces.cache_clear()
+
+    def test_command_used(self, m_subp):
+        """Test we use the correct command when we call subp"""
+        net.get_ovs_internal_interfaces()
+
+        assert [
+            mock.call(net.OVS_INTERNAL_INTERFACE_LOOKUP_CMD)
+        ] == m_subp.call_args_list
+
+    def test_subp_contents_split_and_returned(self, m_subp):
+        """Test that the command output is appropriately mangled."""
+        stdout = "iface1\niface2\niface3\n"
+        m_subp.return_value = (stdout, "")
+
+        assert [
+            "iface1",
+            "iface2",
+            "iface3",
+        ] == net.get_ovs_internal_interfaces()
+
+    def test_database_connection_error_handled_gracefully(self, m_subp):
+        """Test that the error indicating OVS is down is handled gracefully."""
+        m_subp.side_effect = ProcessExecutionError(
+            stderr="database connection failed"
+        )
+
+        assert [] == net.get_ovs_internal_interfaces()
+
+    def test_other_errors_raised(self, m_subp):
+        """Test that only database connection errors are handled."""
+        m_subp.side_effect = ProcessExecutionError()
+
+        with pytest.raises(ProcessExecutionError):
+            net.get_ovs_internal_interfaces()
+
+    def test_only_runs_once(self, m_subp):
+        """Test that we cache the value."""
+        net.get_ovs_internal_interfaces()
+        net.get_ovs_internal_interfaces()
+
+        assert 1 == m_subp.call_count
+
+
+@mock.patch("cloudinit.net.get_ovs_internal_interfaces")
+@mock.patch("cloudinit.net.openvswitch_is_installed")
+class TestIsOpenVSwitchInternalInterface:
+    def test_false_if_ovs_not_installed(
+        self, m_openvswitch_is_installed, _m_get_ovs_internal_interfaces
+    ):
+        """Test that OVS' absence returns False."""
+        m_openvswitch_is_installed.return_value = False
+
+        assert not net.is_openvswitch_internal_interface("devname")
+
+    @pytest.mark.parametrize(
+        "detected_interfaces,devname,expected_return",
+        [
+            ([], "devname", False),
+            (["notdevname"], "devname", False),
+            (["devname"], "devname", True),
+            (["some", "other", "devices", "and", "ours"], "ours", True),
+        ],
+    )
+    def test_return_value_based_on_detected_interfaces(
+        self,
+        m_openvswitch_is_installed,
+        m_get_ovs_internal_interfaces,
+        detected_interfaces,
+        devname,
+        expected_return,
+    ):
+        """Test that the detected interfaces are used correctly."""
+        m_openvswitch_is_installed.return_value = True
+        m_get_ovs_internal_interfaces.return_value = detected_interfaces
+        assert expected_return == net.is_openvswitch_internal_interface(
+            devname
+        )
+
+
 class TestIsIpAddress:
     """Tests for net.is_ip_address.
 
diff --git a/cloudinit/sources/DataSourceAzure.py b/cloudinit/sources/DataSourceAzure.py
index cee630f7..6cae9e82 100755
--- a/cloudinit/sources/DataSourceAzure.py
+++ b/cloudinit/sources/DataSourceAzure.py
@@ -78,17 +78,15 @@ AGENT_SEED_DIR = '/var/lib/waagent'
 # In the event where the IMDS primary server is not
 # available, it takes 1s to fallback to the secondary one
 IMDS_TIMEOUT_IN_SECONDS = 2
-IMDS_URL = "http://169.254.169.254/metadata/"
-IMDS_VER = "2019-06-01"
-IMDS_VER_PARAM = "api-version={}".format(IMDS_VER)
+IMDS_URL = "http://169.254.169.254/metadata"
+IMDS_VER_MIN = "2019-06-01"
+IMDS_VER_WANT = "2020-09-01"
 
 
 class metadata_type(Enum):
-    compute = "{}instance?{}".format(IMDS_URL, IMDS_VER_PARAM)
-    network = "{}instance/network?{}".format(IMDS_URL,
-                                             IMDS_VER_PARAM)
-    reprovisiondata = "{}reprovisiondata?{}".format(IMDS_URL,
-                                                    IMDS_VER_PARAM)
+    compute = "{}/instance".format(IMDS_URL)
+    network = "{}/instance/network".format(IMDS_URL)
+    reprovisiondata = "{}/reprovisiondata".format(IMDS_URL)
 
 
 PLATFORM_ENTROPY_SOURCE = "/sys/firmware/acpi/tables/OEM0"
@@ -349,6 +347,8 @@ class DataSourceAzure(sources.DataSource):
         self.update_events['network'].add(EventType.BOOT)
         self._ephemeral_dhcp_ctx = None
 
+        self.failed_desired_api_version = False
+
     def __str__(self):
         root = sources.DataSource.__str__(self)
         return "%s [seed=%s]" % (root, self.seed)
@@ -520,8 +520,10 @@ class DataSourceAzure(sources.DataSource):
                     self._wait_for_all_nics_ready()
                 ret = self._reprovision()
 
-            imds_md = get_metadata_from_imds(
-                self.fallback_interface, retries=10)
+            imds_md = self.get_imds_data_with_api_fallback(
+                self.fallback_interface,
+                retries=10
+            )
             (md, userdata_raw, cfg, files) = ret
             self.seed = cdev
             crawled_data.update({
@@ -652,6 +654,57 @@ class DataSourceAzure(sources.DataSource):
             self.ds_cfg['data_dir'], crawled_data['files'], dirmode=0o700)
         return True
 
+    @azure_ds_telemetry_reporter
+    def get_imds_data_with_api_fallback(
+            self,
+            fallback_nic,
+            retries,
+            md_type=metadata_type.compute):
+        """
+        Wrapper for get_metadata_from_imds so that we can have flexibility
+        in which IMDS api-version we use. If a particular instance of IMDS
+        does not have the api version that is desired, we want to make
+        this fault tolerant and fall back to a good known minimum api
+        version.
+        """
+
+        if not self.failed_desired_api_version:
+            for _ in range(retries):
+                try:
+                    LOG.info(
+                        "Attempting IMDS api-version: %s",
+                        IMDS_VER_WANT
+                    )
+                    return get_metadata_from_imds(
+                        fallback_nic=fallback_nic,
+                        retries=0,
+                        md_type=md_type,
+                        api_version=IMDS_VER_WANT
+                    )
+                except UrlError as err:
+                    LOG.info(
+                        "UrlError with IMDS api-version: %s",
+                        IMDS_VER_WANT
+                    )
+                    if err.code == 400:
+                        log_msg = "Fall back to IMDS api-version: {}".format(
+                            IMDS_VER_MIN
+                        )
+                        report_diagnostic_event(
+                            log_msg,
+                            logger_func=LOG.info
+                        )
+                        self.failed_desired_api_version = True
+                        break
+
+        LOG.info("Using IMDS api-version: %s", IMDS_VER_MIN)
+        return get_metadata_from_imds(
+            fallback_nic=fallback_nic,
+            retries=retries,
+            md_type=md_type,
+            api_version=IMDS_VER_MIN
+        )
+
     def device_name_to_device(self, name):
         return self.ds_cfg['disk_aliases'].get(name)
 
@@ -880,10 +933,11 @@ class DataSourceAzure(sources.DataSource):
         # primary nic is being attached first helps here. Otherwise each nic
         # could add several seconds of delay.
         try:
-            imds_md = get_metadata_from_imds(
+            imds_md = self.get_imds_data_with_api_fallback(
                 ifname,
                 5,
-                metadata_type.network)
+                metadata_type.network
+            )
         except Exception as e:
             LOG.warning(
                 "Failed to get network metadata using nic %s. Attempt to "
@@ -1017,7 +1071,10 @@ class DataSourceAzure(sources.DataSource):
     def _poll_imds(self):
         """Poll IMDS for the new provisioning data until we get a valid
         response. Then return the returned JSON object."""
-        url = metadata_type.reprovisiondata.value
+        url = "{}?api-version={}".format(
+            metadata_type.reprovisiondata.value,
+            IMDS_VER_MIN
+        )
         headers = {"Metadata": "true"}
         nl_sock = None
         report_ready = bool(not os.path.isfile(REPORTED_READY_MARKER_FILE))
@@ -2059,7 +2116,8 @@ def _generate_network_config_from_fallback_config() -> dict:
 @azure_ds_telemetry_reporter
 def get_metadata_from_imds(fallback_nic,
                            retries,
-                           md_type=metadata_type.compute):
+                           md_type=metadata_type.compute,
+                           api_version=IMDS_VER_MIN):
     """Query Azure's instance metadata service, returning a dictionary.
 
     If network is not up, setup ephemeral dhcp on fallback_nic to talk to the
@@ -2069,13 +2127,16 @@ def get_metadata_from_imds(fallback_nic,
     @param fallback_nic: String. The name of the nic which requires active
         network in order to query IMDS.
     @param retries: The number of retries of the IMDS_URL.
+    @param md_type: Metadata type for IMDS request.
+    @param api_version: IMDS api-version to use in the request.
 
     @return: A dict of instance metadata containing compute and network
         info.
     """
     kwargs = {'logfunc': LOG.debug,
               'msg': 'Crawl of Azure Instance Metadata Service (IMDS)',
-              'func': _get_metadata_from_imds, 'args': (retries, md_type,)}
+              'func': _get_metadata_from_imds,
+              'args': (retries, md_type, api_version,)}
     if net.is_up(fallback_nic):
         return util.log_time(**kwargs)
     else:
@@ -2091,20 +2152,26 @@ def get_metadata_from_imds(fallback_nic,
 
 
 @azure_ds_telemetry_reporter
-def _get_metadata_from_imds(retries, md_type=metadata_type.compute):
-
-    url = md_type.value
+def _get_metadata_from_imds(
+        retries,
+        md_type=metadata_type.compute,
+        api_version=IMDS_VER_MIN):
+    url = "{}?api-version={}".format(md_type.value, api_version)
     headers = {"Metadata": "true"}
     try:
         response = readurl(
             url, timeout=IMDS_TIMEOUT_IN_SECONDS, headers=headers,
             retries=retries, exception_cb=retry_on_url_exc)
     except Exception as e:
-        report_diagnostic_event(
-            'Ignoring IMDS instance metadata. '
-            'Get metadata from IMDS failed: %s' % e,
-            logger_func=LOG.warning)
-        return {}
+        # pylint:disable=no-member
+        if isinstance(e, UrlError) and e.code == 400:
+            raise
+        else:
+            report_diagnostic_event(
+                'Ignoring IMDS instance metadata. '
+                'Get metadata from IMDS failed: %s' % e,
+                logger_func=LOG.warning)
+            return {}
     try:
         from json.decoder import JSONDecodeError
         json_decode_error = JSONDecodeError
diff --git a/cloudinit/sources/DataSourceEc2.py b/cloudinit/sources/DataSourceEc2.py
index 1930a509..a2105dc7 100644
--- a/cloudinit/sources/DataSourceEc2.py
+++ b/cloudinit/sources/DataSourceEc2.py
@@ -765,13 +765,14 @@ def convert_ec2_metadata_network_config(
         netcfg['ethernets'][nic_name] = dev_config
         return netcfg
     # Apply network config for all nics and any secondary IPv4/v6 addresses
+    nic_idx = 0
     for mac, nic_name in sorted(macs_to_nics.items()):
         nic_metadata = macs_metadata.get(mac)
         if not nic_metadata:
             continue  # Not a physical nic represented in metadata
         # device-number is zero-indexed, we want it 1-indexed for the
         # multiplication on the following line
-        nic_idx = int(nic_metadata['device-number']) + 1
+        nic_idx = int(nic_metadata.get('device-number', nic_idx)) + 1
         dhcp_override = {'route-metric': nic_idx * 100}
         dev_config = {'dhcp4': True, 'dhcp4-overrides': dhcp_override,
                       'dhcp6': False,
diff --git a/cloudinit/sources/helpers/tests/test_openstack.py b/cloudinit/sources/helpers/tests/test_openstack.py
index 2bde1e3f..95fb9743 100644
--- a/cloudinit/sources/helpers/tests/test_openstack.py
+++ b/cloudinit/sources/helpers/tests/test_openstack.py
@@ -1,10 +1,15 @@
 # This file is part of cloud-init. See LICENSE file for license information.
 # ./cloudinit/sources/helpers/tests/test_openstack.py
+from unittest import mock
 
 from cloudinit.sources.helpers import openstack
 from cloudinit.tests import helpers as test_helpers
 
 
+@mock.patch(
+    "cloudinit.net.is_openvswitch_internal_interface",
+    mock.Mock(return_value=False)
+)
 class TestConvertNetJson(test_helpers.CiTestCase):
 
     def test_phy_types(self):
diff --git a/cloudinit/sources/tests/test_oracle.py b/cloudinit/sources/tests/test_oracle.py
index a7bbdfd9..dcf33b9b 100644
--- a/cloudinit/sources/tests/test_oracle.py
+++ b/cloudinit/sources/tests/test_oracle.py
@@ -173,6 +173,10 @@ class TestIsPlatformViable(test_helpers.CiTestCase):
         m_read_dmi_data.assert_has_calls([mock.call('chassis-asset-tag')])
 
 
+@mock.patch(
+    "cloudinit.net.is_openvswitch_internal_interface",
+    mock.Mock(return_value=False)
+)
 class TestNetworkConfigFromOpcImds:
     def test_no_secondary_nics_does_not_mutate_input(self, oracle_ds):
         oracle_ds._vnics_data = [{}]
diff --git a/cloudinit/stages.py b/cloudinit/stages.py
index 3ef4491c..5bacc85d 100644
--- a/cloudinit/stages.py
+++ b/cloudinit/stages.py
@@ -364,12 +364,12 @@ class Init(object):
                             'userdata')
         self._store_processeddata(self.datasource.get_userdata(),
                                   'userdata')
-        self._store_rawdata(self.datasource.get_vendordata_raw(),
-                            'vendordata')
+        self._store_raw_vendordata(self.datasource.get_vendordata_raw(),
+                                   'vendordata')
         self._store_processeddata(self.datasource.get_vendordata(),
                                   'vendordata')
-        self._store_rawdata(self.datasource.get_vendordata2_raw(),
-                            'vendordata2')
+        self._store_raw_vendordata(self.datasource.get_vendordata2_raw(),
+                                   'vendordata2')
         self._store_processeddata(self.datasource.get_vendordata2(),
                                   'vendordata2')
 
@@ -397,6 +397,16 @@ class Init(object):
             data = b''
         util.write_file(self._get_ipath('%s_raw' % datasource), data, 0o600)
 
+    def _store_raw_vendordata(self, data, datasource):
+        # Only these data types
+        if data is not None and type(data) not in [bytes, str, list]:
+            raise TypeError("vendordata_raw is unsupported type '%s'" %
+                            str(type(data)))
+        # This data may be a list, convert it to a string if so
+        if isinstance(data, list):
+            data = util.json_dumps(data)
+        self._store_rawdata(data, datasource)
+
     def _store_processeddata(self, processed_data, datasource):
         # processed is a Mime message, so write as string.
         if processed_data is None:
diff --git a/cloudinit/tests/test_util.py b/cloudinit/tests/test_util.py
index b7a302f1..e811917e 100644
--- a/cloudinit/tests/test_util.py
+++ b/cloudinit/tests/test_util.py
@@ -851,4 +851,60 @@ class TestEnsureFile:
         assert "ab" == kwargs["omode"]
 
 
+@mock.patch("cloudinit.util.grp.getgrnam")
+@mock.patch("cloudinit.util.os.setgid")
+@mock.patch("cloudinit.util.os.umask")
+class TestRedirectOutputPreexecFn:
+    """This tests specifically the preexec_fn used in redirect_output."""
+
+    @pytest.fixture(params=["outfmt", "errfmt"])
+    def preexec_fn(self, request):
+        """A fixture to gather the preexec_fn used by redirect_output.
+
+        This enables simpler direct testing of it, and parameterises any tests
+        using it to cover both the stdout and stderr code paths.
+        """
+        test_string = "| piped output to invoke subprocess"
+        if request.param == "outfmt":
+            args = (test_string, None)
+        elif request.param == "errfmt":
+            args = (None, test_string)
+        with mock.patch("cloudinit.util.subprocess.Popen") as m_popen:
+            util.redirect_output(*args)
+
+        assert 1 == m_popen.call_count
+        _args, kwargs = m_popen.call_args
+        assert "preexec_fn" in kwargs, "preexec_fn not passed to Popen"
+        return kwargs["preexec_fn"]
+
+    def test_preexec_fn_sets_umask(
+        self, m_os_umask, _m_setgid, _m_getgrnam, preexec_fn
+    ):
+        """preexec_fn should set a mask that avoids world-readable files."""
+        preexec_fn()
+
+        assert [mock.call(0o037)] == m_os_umask.call_args_list
+
+    def test_preexec_fn_sets_group_id_if_adm_group_present(
+        self, _m_os_umask, m_setgid, m_getgrnam, preexec_fn
+    ):
+        """We should setgrp to adm if present, so files are owned by them."""
+        fake_group = mock.Mock(gr_gid=mock.sentinel.gr_gid)
+        m_getgrnam.return_value = fake_group
+
+        preexec_fn()
+
+        assert [mock.call("adm")] == m_getgrnam.call_args_list
+        assert [mock.call(mock.sentinel.gr_gid)] == m_setgid.call_args_list
+
+    def test_preexec_fn_handles_absent_adm_group_gracefully(
+        self, _m_os_umask, m_setgid, m_getgrnam, preexec_fn
+    ):
+        """We should handle an absent adm group gracefully."""
+        m_getgrnam.side_effect = KeyError("getgrnam(): name not found: 'adm'")
+
+        preexec_fn()
+
+        assert 0 == m_setgid.call_count
+
 # vi: ts=4 expandtab
diff --git a/cloudinit/util.py b/cloudinit/util.py
index 769f3425..4e0a72db 100644
--- a/cloudinit/util.py
+++ b/cloudinit/util.py
@@ -359,7 +359,7 @@ def find_modules(root_dir):
 
 
 def multi_log(text, console=True, stderr=True,
-              log=None, log_level=logging.DEBUG):
+              log=None, log_level=logging.DEBUG, fallback_to_stdout=True):
     if stderr:
         sys.stderr.write(text)
     if console:
@@ -368,7 +368,7 @@ def multi_log(text, console=True, stderr=True,
             with open(conpath, 'w') as wfh:
                 wfh.write(text)
                 wfh.flush()
-        else:
+        elif fallback_to_stdout:
             # A container may lack /dev/console (arguably a container bug).  If
             # it does not exist, then write output to stdout.  this will result
             # in duplicate stderr and stdout messages if stderr was True.
@@ -623,6 +623,26 @@ def redirect_output(outfmt, errfmt, o_out=None, o_err=None):
     if not o_err:
         o_err = sys.stderr
 
+    # pylint: disable=subprocess-popen-preexec-fn
+    def set_subprocess_umask_and_gid():
+        """Reconfigure umask and group ID to create output files securely.
+
+        This is passed to subprocess.Popen as preexec_fn, so it is executed in
+        the context of the newly-created process.  It:
+
+        * sets the umask of the process so created files aren't world-readable
+        * if an adm group exists in the system, sets that as the process' GID
+          (so that the created file(s) are owned by root:adm)
+        """
+        os.umask(0o037)
+        try:
+            group_id = grp.getgrnam("adm").gr_gid
+        except KeyError:
+            # No adm group, don't set a group
+            pass
+        else:
+            os.setgid(group_id)
+
     if outfmt:
         LOG.debug("Redirecting %s to %s", o_out, outfmt)
         (mode, arg) = outfmt.split(" ", 1)
@@ -632,7 +652,12 @@ def redirect_output(outfmt, errfmt, o_out=None, o_err=None):
                 owith = "wb"
             new_fp = open(arg, owith)
         elif mode == "|":
-            proc = subprocess.Popen(arg, shell=True, stdin=subprocess.PIPE)
+            proc = subprocess.Popen(
+                arg,
+                shell=True,
+                stdin=subprocess.PIPE,
+                preexec_fn=set_subprocess_umask_and_gid,
+            )
             new_fp = proc.stdin
         else:
             raise TypeError("Invalid type for output format: %s" % outfmt)
@@ -654,7 +679,12 @@ def redirect_output(outfmt, errfmt, o_out=None, o_err=None):
                 owith = "wb"
             new_fp = open(arg, owith)
         elif mode == "|":
-            proc = subprocess.Popen(arg, shell=True, stdin=subprocess.PIPE)
+            proc = subprocess.Popen(
+                arg,
+                shell=True,
+                stdin=subprocess.PIPE,
+                preexec_fn=set_subprocess_umask_and_gid,
+            )
             new_fp = proc.stdin
         else:
             raise TypeError("Invalid type for error format: %s" % errfmt)
diff --git a/debian/changelog b/debian/changelog
index 38d0314e..96a2688c 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,3 +1,35 @@
+cloud-init (21.1-19-gbad84ad4-0ubuntu1) hirsute; urgency=medium
+  * d/cloud-init.postinst: Change output log permissions on upgrade
+    (LP: #1918303)
+  * New upstream snapshot.
+    - .travis.yml: generate an SSH key before running tests (#848)
+    - write passwords only to serial console, lock down cloud-init-output.log
+      (#847) (LP: #1918303)
+    - Fix apt default integration test (#845)
+    - integration_tests: bump pycloudlib dependency (#846)
+    - commit f35181fa970453ba6c7c14575b12185533391b97 [eb3095]
+    - archlinux: Fix broken locale logic (#841)
+      [Kristian Klausen] (LP: #1402406)
+    - Integration test for #783 (#832)
+    - integration_tests: mount more paths IN_PLACE (#838)
+    - Fix requiring device-number on EC2 derivatives (#836) (LP: #1917875)
+    - Remove the vi comment from the part-handler example (#835)
+    - net: exclude OVS internal interfaces in get_interfaces (#829)
+      (LP: #1912844)
+    - tox.ini: pass OS_* environment variables to integration tests (#830)
+    - integration_tests: add OpenStack as a platform (#804)
+    - Add flexibility to IMDS api-version (#793) [Thomas Stringer]
+    - Fix the TestApt tests using apt-key on Xenial and Hirsute (#823)
+      [Paride Legovini] (LP: #1916629)
+    - doc: remove duplicate "it" from nocloud.rst (#825) [V.I. Wood]
+    - archlinux: Use hostnamectl to set the transient hostname (#797)
+      [Kristian Klausen]
+    - cc_keys_to_console.py: Add documentation for recently added config key
+      (#824) [dermotbradley]
+    - Update cc_set_hostname documentation (#818) [Toshi Aoyama]
+
+ -- James Falcon <james.falcon@canonical.com>  Fri, 19 Mar 2021 14:32:13 -0500
+
 cloud-init (21.1-0ubuntu1) hirsute; urgency=medium
 
   * New upstream release.
diff --git a/debian/cloud-init.postinst b/debian/cloud-init.postinst
index bb1535e8..683ba86d 100644
--- a/debian/cloud-init.postinst
+++ b/debian/cloud-init.postinst
@@ -327,6 +327,22 @@ fix_lp1889555() {
     db_set grub-pc/install_devices_empty "false"
 }
 
+change_cloud_init_output_log_permissions() {
+    # As a consequence of LP: #1918303
+    local oldver="$1" last_bad_ver="21.1-0ubuntu1"
+    dpkg --compare-versions "$oldver" le-nl "$last_bad_ver" || return 0
+
+    output_file="/var/log/cloud-init-output.log"
+    if [ -f "$output_file" ]; then
+        if getent group adm > /dev/null; then
+            chown root:adm $output_file
+        else
+            chown root $output_file
+        fi
+        chmod 640 $output_file
+    fi
+}
+
 
 if [ "$1" = "configure" ]; then
    if db_get cloud-init/datasources; then
@@ -358,6 +374,7 @@ EOF
 
    cleanup_ureadahead "$2"
    fix_lp1889555 "$2"
+   change_cloud_init_output_log_permissions "$2"
 fi
 
 #DEBHELPER#
diff --git a/doc/examples/part-handler.txt b/doc/examples/part-handler.txt
index a6e66415..1484e1a0 100644
--- a/doc/examples/part-handler.txt
+++ b/doc/examples/part-handler.txt
@@ -1,5 +1,4 @@
 #part-handler
-# vi: syntax=python ts=4
 
 def list_types():
     # return a list of mime-types that are handled by this module
diff --git a/doc/rtd/topics/datasources/nocloud.rst b/doc/rtd/topics/datasources/nocloud.rst
index 0ca79102..edb41e2a 100644
--- a/doc/rtd/topics/datasources/nocloud.rst
+++ b/doc/rtd/topics/datasources/nocloud.rst
@@ -117,7 +117,7 @@ yaml formatted data in a file named ``network-config``.  If found,
 this file will override a ``network-interfaces`` file.
 
 See an example below.  Note specifically that this file does not
-have a top level ``network`` key as it it is already assumed to
+have a top level ``network`` key as it is already assumed to
 be network configuration based on the filename.
 
 .. code:: yaml
diff --git a/integration-requirements.txt b/integration-requirements.txt
index 6b596426..95891356 100644
--- a/integration-requirements.txt
+++ b/integration-requirements.txt
@@ -1,5 +1,5 @@
 # PyPI requirements for cloud-init integration testing
 # https://cloudinit.readthedocs.io/en/latest/topics/integration_tests.html
 #
-pycloudlib @ git+https://github.com/canonical/pycloudlib.git@da8445325875674394ffd85aaefaa3d2d0e0020d
+pycloudlib @ git+https://github.com/canonical/pycloudlib.git@96b146ee1beb99b8e44e36525e18a9a20e00c3f2
 pytest
diff --git a/tests/integration_tests/bugs/test_lp1912844.py b/tests/integration_tests/bugs/test_lp1912844.py
new file mode 100644
index 00000000..efafae50
--- /dev/null
+++ b/tests/integration_tests/bugs/test_lp1912844.py
@@ -0,0 +1,103 @@
+"""Integration test for LP: #1912844
+
+cloud-init should ignore OVS-internal interfaces when performing its own
+interface determination: these interfaces are handled fully by OVS, so
+cloud-init should never need to touch them.
+
+This test is a semi-synthetic reproducer for the bug.  It uses a similar
+network configuration, tweaked slightly to DHCP in a way that will succeed even
+on "failed" boots.  The exact bug doesn't reproduce with the NoCloud
+datasource, because it runs at init-local time (whereas the MAAS datasource,
+from the report, runs only at init (network) time): this means that the
+networking code runs before OVS creates its interfaces (which happens after
+init-local but, of course, before networking is up), and so doesn't generate
+the traceback that they cause.  We work around this by calling
+``get_interfaces_by_mac` directly in the test code.
+"""
+import pytest
+
+from tests.integration_tests import random_mac_address
+
+MAC_ADDRESS = random_mac_address()
+
+NETWORK_CONFIG = """\
+bonds:
+    bond0:
+        interfaces:
+            - enp5s0
+        macaddress: {0}
+        mtu: 1500
+bridges:
+        ovs-br:
+            interfaces:
+            - bond0
+            macaddress: {0}
+            mtu: 1500
+            openvswitch: {{}}
+            dhcp4: true
+ethernets:
+    enp5s0:
+      mtu: 1500
+      set-name: enp5s0
+      match:
+          macaddress: {0}
+version: 2
+vlans:
+  ovs-br.100:
+    id: 100
+    link: ovs-br
+    mtu: 1500
+  ovs-br.200:
+    id: 200
+    link: ovs-br
+    mtu: 1500
+""".format(MAC_ADDRESS)
+
+
+SETUP_USER_DATA = """\
+#cloud-config
+packages:
+- openvswitch-switch
+"""
+
+
+@pytest.fixture
+def ovs_enabled_session_cloud(session_cloud):
+    """A session_cloud wrapper, to use an OVS-enabled image for tests.
+
+    This implementation is complicated by wanting to use ``session_cloud``s
+    snapshot cleanup/retention logic, to avoid having to reimplement that here.
+    """
+    old_snapshot_id = session_cloud.snapshot_id
+    with session_cloud.launch(
+        user_data=SETUP_USER_DATA,
+    ) as instance:
+        instance.instance.clean()
+        session_cloud.snapshot_id = instance.snapshot()
+
+    yield session_cloud
+
+    try:
+        session_cloud.delete_snapshot()
+    finally:
+        session_cloud.snapshot_id = old_snapshot_id
+
+
+@pytest.mark.lxd_vm
+def test_get_interfaces_by_mac_doesnt_traceback(ovs_enabled_session_cloud):
+    """Launch our OVS-enabled image and confirm the bug doesn't reproduce."""
+    launch_kwargs = {
+        "config_dict": {
+            "user.network-config": NETWORK_CONFIG,
+            "volatile.eth0.hwaddr": MAC_ADDRESS,
+        },
+    }
+    with ovs_enabled_session_cloud.launch(
+        launch_kwargs=launch_kwargs,
+    ) as client:
+        result = client.execute(
+            "python3 -c"
+            "'from cloudinit.net import get_interfaces_by_mac;"
+            "get_interfaces_by_mac()'"
+        )
+        assert result.ok
diff --git a/tests/integration_tests/clouds.py b/tests/integration_tests/clouds.py
index 9527a413..a6026309 100644
--- a/tests/integration_tests/clouds.py
+++ b/tests/integration_tests/clouds.py
@@ -1,8 +1,18 @@
 # This file is part of cloud-init. See LICENSE file for license information.
 from abc import ABC, abstractmethod
 import logging
-
-from pycloudlib import EC2, GCE, Azure, OCI, LXDContainer, LXDVirtualMachine
+import os.path
+from uuid import UUID
+
+from pycloudlib import (
+    EC2,
+    GCE,
+    Azure,
+    OCI,
+    LXDContainer,
+    LXDVirtualMachine,
+    Openstack,
+)
 from pycloudlib.lxd.instance import LXDInstance
 
 import cloudinit
@@ -253,23 +263,32 @@ class _LxdIntegrationCloud(IntegrationCloud):
 
     @staticmethod
     def _mount_source(instance: LXDInstance):
-        target_path = '/usr/lib/python3/dist-packages/cloudinit'
-        format_variables = {
-            'name': instance.name,
-            'source_path': cloudinit.__path__[0],
-            'container_path': target_path,
-        }
-        log.info(
-            'Mounting source %(source_path)s directly onto LXD container/vm '
-            'named %(name)s at %(container_path)s',
-            format_variables
-        )
-        command = (
-            'lxc config device add {name} host-cloud-init disk '
-            'source={source_path} '
-            'path={container_path}'
-        ).format(**format_variables)
-        subp(command.split())
+        cloudinit_path = cloudinit.__path__[0]
+        mounts = [
+            (cloudinit_path, '/usr/lib/python3/dist-packages/cloudinit'),
+            (os.path.join(cloudinit_path, '..', 'config', 'cloud.cfg.d'),
+             '/etc/cloud/cloud.cfg.d'),
+            (os.path.join(cloudinit_path, '..', 'templates'),
+             '/etc/cloud/templates'),
+        ]
+        for (n, (source_path, target_path)) in enumerate(mounts):
+            format_variables = {
+                'name': instance.name,
+                'source_path': os.path.realpath(source_path),
+                'container_path': target_path,
+                'idx': n,
+            }
+            log.info(
+                'Mounting source %(source_path)s directly onto LXD'
+                ' container/VM named %(name)s at %(container_path)s',
+                format_variables
+            )
+            command = (
+                'lxc config device add {name} host-cloud-init-{idx} disk '
+                'source={source_path} '
+                'path={container_path}'
+            ).format(**format_variables)
+            subp(command.split())
 
     def _perform_launch(self, launch_kwargs):
         launch_kwargs['inst_type'] = launch_kwargs.pop('instance_type', None)
@@ -311,3 +330,32 @@ class LxdVmCloud(_LxdIntegrationCloud):
         self._profile_list = self.cloud_instance.build_necessary_profiles(
             release)
         return self._profile_list
+
+
+class OpenstackCloud(IntegrationCloud):
+    datasource = 'openstack'
+    integration_instance_cls = IntegrationInstance
+
+    def _get_cloud_instance(self):
+        if not integration_settings.OPENSTACK_NETWORK:
+            raise Exception(
+                'OPENSTACK_NETWORK must be set to a valid Openstack network. '
+                'If using the openstack CLI, try `openstack network list`'
+            )
+        return Openstack(
+            tag='openstack-integration-test',
+            network=integration_settings.OPENSTACK_NETWORK,
+        )
+
+    def _get_initial_image(self):
+        image = ImageSpecification.from_os_image()
+        try:
+            UUID(image.image_id)
+        except ValueError as e:
+            raise Exception(
+                'When using Openstack, `OS_IMAGE` MUST be specified with '
+                'a 36-character UUID image ID. Passing in a release name is '
+                'not valid here.\n'
+                'OS image id: {}'.format(image.image_id)
+            ) from e
+        return image.image_id
diff --git a/tests/integration_tests/conftest.py b/tests/integration_tests/conftest.py
index 61ad8a71..6f4ce8d3 100644
--- a/tests/integration_tests/conftest.py
+++ b/tests/integration_tests/conftest.py
@@ -20,6 +20,7 @@ from tests.integration_tests.clouds import (
     LxdVmCloud,
     OciCloud,
     _LxdIntegrationCloud,
+    OpenstackCloud,
 )
 from tests.integration_tests.instances import (
     CloudInitSource,
@@ -38,11 +39,23 @@ platforms = {
     'oci': OciCloud,
     'lxd_container': LxdContainerCloud,
     'lxd_vm': LxdVmCloud,
+    'openstack': OpenstackCloud,
 }
 os_list = ["ubuntu"]
 
 session_start_time = datetime.datetime.now().strftime('%y%m%d%H%M%S')
 
+XENIAL_LXD_VM_EXEC_MSG = """\
+The default xenial images do not support `exec` for LXD VMs.
+
+Specify an image known to work using:
+
+    OS_IMAGE=<image id>::ubuntu::xenial
+
+You can re-run specifically tests that require this by passing `-m
+lxd_use_exec` to pytest.
+"""
+
 
 def pytest_runtest_setup(item):
     """Skip tests on unsupported clouds.
@@ -214,6 +227,16 @@ def _client(request, fixture_utils, session_cloud: IntegrationCloud):
     if lxd_use_exec is not None:
         if not isinstance(session_cloud, _LxdIntegrationCloud):
             pytest.skip("lxd_use_exec requires LXD")
+        if isinstance(session_cloud, LxdVmCloud):
+            image_spec = ImageSpecification.from_os_image()
+            if image_spec.release == image_spec.image_id == "xenial":
+                # Why fail instead of skip?  We expect that skipped tests will
+                # be run in a different one of our usual battery of test runs
+                # (e.g. LXD-only tests are skipped on EC2 but will run in our
+                # normal LXD test runs).  This is not true of this test: it
+                # can't run in our usual xenial LXD VM test run, and it may not
+                # run anywhere else.  A failure flags up this discrepancy.
+                pytest.fail(XENIAL_LXD_VM_EXEC_MSG)
         launch_kwargs["execute_via_ssh"] = False
 
     with session_cloud.launch(
diff --git a/tests/integration_tests/integration_settings.py b/tests/integration_tests/integration_settings.py
index 157d34ad..0703be58 100644
--- a/tests/integration_tests/integration_settings.py
+++ b/tests/integration_tests/integration_settings.py
@@ -21,6 +21,7 @@ RUN_UNSTABLE = False
 #  ec2
 #  gce
 #  oci
+#  openstack
 PLATFORM = 'lxd_container'
 
 # The cloud-specific instance type to run. E.g., a1.medium on AWS
@@ -90,6 +91,13 @@ PUBLIC_SSH_KEY = None
 KEYPAIR_NAME = None
 
 ##################################################################
+# OPENSTACK SETTINGS
+##################################################################
+# Network to use for Openstack. Should be one of the names/ids found
+# in `openstack network list`
+OPENSTACK_NETWORK = None
+
+##################################################################
 # USER SETTINGS OVERRIDES
 ##################################################################
 # Bring in any user-file defined settings
diff --git a/tests/integration_tests/modules/test_apt.py b/tests/integration_tests/modules/test_apt.py
index 5e3d474c..54711fc0 100644
--- a/tests/integration_tests/modules/test_apt.py
+++ b/tests/integration_tests/modules/test_apt.py
@@ -86,25 +86,11 @@ EXPECTED_REGEXES = [
     r"deb-src http://badsecurity.ubuntu.com/ubuntu [a-z]+-security multiverse",
 ]
 
-TEST_KEYSERVER_KEY = """\
-pub   rsa1024 2013-12-09 [SC]
-      7260 0DB1 5B8E 4C8B 1964  B868 038A CC97 C660 A937
-uid           [ unknown] Launchpad PPA for Ryan Harper
-"""
+TEST_KEYSERVER_KEY = "7260 0DB1 5B8E 4C8B 1964  B868 038A CC97 C660 A937"
 
-TEST_PPA_KEY = """\
-/etc/apt/trusted.gpg.d/simplestreams-dev_ubuntu_trunk.gpg
----------------------------------------------------------
-pub   rsa4096 2016-05-04 [SC]
-      3552 C902 B4DD F7BD 3842  1821 015D 28D7 4416 14D8
-uid           [ unknown] Launchpad PPA for simplestreams-dev
-"""
+TEST_PPA_KEY = "3552 C902 B4DD F7BD 3842  1821 015D 28D7 4416 14D8"
 
-TEST_KEY = """\
-pub   rsa4096 2016-03-04 [SC]
-      1FF0 D853 5EF7 E719 E5C8  1B9C 083D 06FB E4D3 04DF
-uid           [ unknown] Launchpad PPA for cloud init development team
-"""
+TEST_KEY = "1FF0 D853 5EF7 E719 E5C8  1B9C 083D 06FB E4D3 04DF"
 
 
 @pytest.mark.ci
@@ -214,29 +200,32 @@ class TestApt:
         assert conf_exists is False
 
 
-DEFAULT_DATA = """\
+_DEFAULT_DATA = """\
 #cloud-config
 apt:
   primary:
     - arches:
       - default
+      {uri}
   security:
     - arches:
       - default
 """
+DEFAULT_DATA = _DEFAULT_DATA.format(uri='')
 
 
 @pytest.mark.ubuntu
 @pytest.mark.user_data(DEFAULT_DATA)
 class TestDefaults:
-    def test_primary(self, class_client: IntegrationInstance):
-        """Test apt default primary sources.
+    @pytest.mark.openstack
+    def test_primary_on_openstack(self, class_client: IntegrationInstance):
+        """Test apt default primary source on openstack.
 
-        Ported from
-        tests/cloud_tests/testcases/modules/apt_configure_primary.py
+        When no uri is provided.
         """
+        zone = class_client.execute('cloud-init query v1.availability_zone')
         sources_list = class_client.read_from_file('/etc/apt/sources.list')
-        assert 'deb http://archive.ubuntu.com/ubuntu' in sources_list
+        assert '{}.clouds.archive.ubuntu.com'.format(zone) in sources_list
 
     def test_security(self, class_client: IntegrationInstance):
         """Test apt default security sources.
@@ -253,6 +242,24 @@ class TestDefaults:
         )
 
 
+DEFAULT_DATA_WITH_URI = _DEFAULT_DATA.format(
+    uri='uri: "http://something.random.invalid/ubuntu"'
+)
+
+
+@pytest.mark.user_data(DEFAULT_DATA_WITH_URI)
+def test_default_primary_with_uri(client: IntegrationInstance):
+    """Test apt default primary sources.
+
+    Ported from
+    tests/cloud_tests/testcases/modules/apt_configure_primary.py
+    """
+    sources_list = client.read_from_file('/etc/apt/sources.list')
+    assert 'archive.ubuntu.com' not in sources_list
+
+    assert 'something.random.invalid' in sources_list
+
+
 DISABLED_DATA = """\
 #cloud-config
 apt:
diff --git a/tests/integration_tests/modules/test_set_password.py b/tests/integration_tests/modules/test_set_password.py
index b13f76fb..d7cf91a5 100644
--- a/tests/integration_tests/modules/test_set_password.py
+++ b/tests/integration_tests/modules/test_set_password.py
@@ -116,6 +116,30 @@ class Mixin:
         # Which are not the same
         assert shadow_users["harry"] != shadow_users["dick"]
 
+    def test_random_passwords_not_stored_in_cloud_init_output_log(
+        self, class_client
+    ):
+        """We should not emit passwords to the in-instance log file.
+
+        LP: #1918303
+        """
+        cloud_init_output = class_client.read_from_file(
+            "/var/log/cloud-init-output.log"
+        )
+        assert "dick:" not in cloud_init_output
+        assert "harry:" not in cloud_init_output
+
+    def test_random_passwords_emitted_to_serial_console(self, class_client):
+        """We should emit passwords to the serial console. (LP: #1918303)"""
+        try:
+            console_log = class_client.instance.console_log()
+        except NotImplementedError:
+            # Assume that an exception here means that we can't use the console
+            # log
+            pytest.skip("NotImplementedError when requesting console log")
+        assert "dick:" in console_log
+        assert "harry:" in console_log
+
     def test_explicit_password_set_correctly(self, class_client):
         """Test that an explicitly-specified password is set correctly."""
         shadow_users, _ = self._fetch_and_parse_etc_shadow(class_client)
diff --git a/tests/integration_tests/modules/test_users_groups.py b/tests/integration_tests/modules/test_users_groups.py
index ee08d87b..bcb17b7f 100644
--- a/tests/integration_tests/modules/test_users_groups.py
+++ b/tests/integration_tests/modules/test_users_groups.py
@@ -1,16 +1,16 @@
-"""Integration test for the user_groups module.
-
-This test specifies a number of users and groups via user-data, and confirms
-that they have been configured correctly in the system under test.
+"""Integration tests for the user_groups module.
 
 TODO:
-* This test assumes that the "ubuntu" user will be created when "default" is
+* This module assumes that the "ubuntu" user will be created when "default" is
   specified; this will need modification to run on other OSes.
 """
 import re
 
 import pytest
 
+from tests.integration_tests.clouds import ImageSpecification
+from tests.integration_tests.instances import IntegrationInstance
+
 
 USER_DATA = """\
 #cloud-config
@@ -45,6 +45,12 @@ AHWYPYb2FT.lbioDm2RrkJPb9BZMN1O/
 @pytest.mark.ci
 @pytest.mark.user_data(USER_DATA)
 class TestUsersGroups:
+    """Test users and groups.
+
+    This test specifies a number of users and groups via user-data, and
+    confirms that they have been configured correctly in the system under test.
+    """
+
     @pytest.mark.ubuntu
     @pytest.mark.parametrize(
         "getent_args,regex",
@@ -86,3 +92,32 @@ class TestUsersGroups:
         _, groups_str = output.split(":", maxsplit=1)
         groups = groups_str.split()
         assert "secret" in groups
+
+
+@pytest.mark.user_data(USER_DATA)
+def test_sudoers_includedir(client: IntegrationInstance):
+    """Ensure we don't add additional #includedir to sudoers.
+
+    Newer versions of /etc/sudoers will use @includedir rather than
+    #includedir. Ensure we handle that properly and don't include an
+    additional #includedir when one isn't warranted.
+
+    https://github.com/canonical/cloud-init/pull/783
+    """
+    if ImageSpecification.from_os_image().release in [
+        'xenial', 'bionic', 'focal'
+    ]:
+        raise pytest.skip(
+            'Test requires version of sudo installed on groovy and later'
+        )
+    client.execute("sed -i 's/#include/@include/g' /etc/sudoers")
+
+    sudoers = client.read_from_file('/etc/sudoers')
+    if '@includedir /etc/sudoers.d' not in sudoers:
+        client.execute("echo '@includedir /etc/sudoers.d' >> /etc/sudoers")
+    client.instance.clean()
+    client.restart()
+    sudoers = client.read_from_file('/etc/sudoers')
+
+    assert '#includedir' not in sudoers
+    assert sudoers.count('includedir /etc/sudoers.d') == 1
diff --git a/tests/integration_tests/test_logging.py b/tests/integration_tests/test_logging.py
new file mode 100644
index 00000000..b31a0434
--- /dev/null
+++ b/tests/integration_tests/test_logging.py
@@ -0,0 +1,22 @@
+"""Integration tests relating to cloud-init's logging."""
+
+
+class TestVarLogCloudInitOutput:
+    """Integration tests relating to /var/log/cloud-init-output.log."""
+
+    def test_var_log_cloud_init_output_not_world_readable(self, client):
+        """
+        The log can contain sensitive data, it shouldn't be world-readable.
+
+        LP: #1918303
+        """
+        # Check the file exists
+        assert client.execute("test -f /var/log/cloud-init-output.log").ok
+
+        # Check its permissions are as we expect
+        perms, user, group = client.execute(
+            "stat -c %a:%U:%G /var/log/cloud-init-output.log"
+        ).split(":")
+        assert "640" == perms
+        assert "root" == user
+        assert "adm" == group
diff --git a/tests/unittests/test_datasource/test_aliyun.py b/tests/unittests/test_datasource/test_aliyun.py
index eb2828d5..cab1ac2b 100644
--- a/tests/unittests/test_datasource/test_aliyun.py
+++ b/tests/unittests/test_datasource/test_aliyun.py
@@ -7,6 +7,7 @@ from unittest import mock
 
 from cloudinit import helpers
 from cloudinit.sources import DataSourceAliYun as ay
+from cloudinit.sources.DataSourceEc2 import convert_ec2_metadata_network_config
 from cloudinit.tests import helpers as test_helpers
 
 DEFAULT_METADATA = {
@@ -183,6 +184,35 @@ class TestAliYunDatasource(test_helpers.HttprettyTestCase):
         self.assertEqual(ay.parse_public_keys(public_keys),
                          public_keys['key-pair-0']['openssh-key'])
 
+    def test_route_metric_calculated_without_device_number(self):
+        """Test that route-metric code works without `device-number`
+
+        `device-number` is part of EC2 metadata, but not supported on aliyun.
+        Attempting to access it will raise a KeyError.
+
+        LP: #1917875
+        """
+        netcfg = convert_ec2_metadata_network_config(
+            {"interfaces": {"macs": {
+                "06:17:04:d7:26:09": {
+                    "interface-id": "eni-e44ef49e",
+                },
+                "06:17:04:d7:26:08": {
+                    "interface-id": "eni-e44ef49f",
+                }
+            }}},
+            macs_to_nics={
+                '06:17:04:d7:26:09': 'eth0',
+                '06:17:04:d7:26:08': 'eth1',
+            }
+        )
+
+        met0 = netcfg['ethernets']['eth0']['dhcp4-overrides']['route-metric']
+        met1 = netcfg['ethernets']['eth1']['dhcp4-overrides']['route-metric']
+
+        # route-metric numbers should be 100 apart
+        assert 100 == abs(met0 - met1)
+
 
 class TestIsAliYun(test_helpers.CiTestCase):
     ALIYUN_PRODUCT = 'Alibaba Cloud ECS'
diff --git a/tests/unittests/test_datasource/test_azure.py b/tests/unittests/test_datasource/test_azure.py
index f597c723..dedebeb1 100644
--- a/tests/unittests/test_datasource/test_azure.py
+++ b/tests/unittests/test_datasource/test_azure.py
@@ -408,7 +408,9 @@ class TestGetMetadataFromIMDS(HttprettyTestCase):
 
     def setUp(self):
         super(TestGetMetadataFromIMDS, self).setUp()
-        self.network_md_url = dsaz.IMDS_URL + "instance?api-version=2019-06-01"
+        self.network_md_url = "{}/instance?api-version=2019-06-01".format(
+            dsaz.IMDS_URL
+        )
 
     @mock.patch(MOCKPATH + 'readurl')
     @mock.patch(MOCKPATH + 'EphemeralDHCPv4', autospec=True)
@@ -518,7 +520,7 @@ class TestGetMetadataFromIMDS(HttprettyTestCase):
         """Return empty dict when IMDS network metadata is absent."""
         httpretty.register_uri(
             httpretty.GET,
-            dsaz.IMDS_URL + 'instance?api-version=2017-12-01',
+            dsaz.IMDS_URL + '/instance?api-version=2017-12-01',
             body={}, status=404)
 
         m_net_is_up.return_value = True  # skips dhcp
@@ -1877,6 +1879,40 @@ scbus-1 on xpt0 bus 0
         ssh_keys = dsrc.get_public_ssh_keys()
         self.assertEqual(ssh_keys, ['key2'])
 
+    @mock.patch(MOCKPATH + 'get_metadata_from_imds')
+    def test_imds_api_version_wanted_nonexistent(
+            self,
+            m_get_metadata_from_imds):
+        def get_metadata_from_imds_side_eff(*args, **kwargs):
+            if kwargs['api_version'] == dsaz.IMDS_VER_WANT:
+                raise url_helper.UrlError("No IMDS version", code=400)
+            return NETWORK_METADATA
+        m_get_metadata_from_imds.side_effect = get_metadata_from_imds_side_eff
+        sys_cfg = {'datasource': {'Azure': {'apply_network_config': True}}}
+        odata = {'HostName': "myhost", 'UserName': "myuser"}
+        data = {
+            'ovfcontent': construct_valid_ovf_env(data=odata),
+            'sys_cfg': sys_cfg
+        }
+        dsrc = self._get_ds(data)
+        dsrc.get_data()
+        self.assertIsNotNone(dsrc.metadata)
+        self.assertTrue(dsrc.failed_desired_api_version)
+
+    @mock.patch(
+        MOCKPATH + 'get_metadata_from_imds', return_value=NETWORK_METADATA)
+    def test_imds_api_version_wanted_exists(self, m_get_metadata_from_imds):
+        sys_cfg = {'datasource': {'Azure': {'apply_network_config': True}}}
+        odata = {'HostName': "myhost", 'UserName': "myuser"}
+        data = {
+            'ovfcontent': construct_valid_ovf_env(data=odata),
+            'sys_cfg': sys_cfg
+        }
+        dsrc = self._get_ds(data)
+        dsrc.get_data()
+        self.assertIsNotNone(dsrc.metadata)
+        self.assertFalse(dsrc.failed_desired_api_version)
+
 
 class TestAzureBounce(CiTestCase):
 
@@ -2657,7 +2693,7 @@ class TestPreprovisioningHotAttachNics(CiTestCase):
     @mock.patch(MOCKPATH + 'DataSourceAzure.wait_for_link_up')
     @mock.patch('cloudinit.sources.helpers.netlink.wait_for_nic_attach_event')
     @mock.patch('cloudinit.sources.net.find_fallback_nic')
-    @mock.patch(MOCKPATH + 'get_metadata_from_imds')
+    @mock.patch(MOCKPATH + 'DataSourceAzure.get_imds_data_with_api_fallback')
     @mock.patch(MOCKPATH + 'EphemeralDHCPv4')
     @mock.patch(MOCKPATH + 'DataSourceAzure._wait_for_nic_detach')
     @mock.patch('os.path.isfile')
diff --git a/tests/unittests/test_datasource/test_configdrive.py b/tests/unittests/test_datasource/test_configdrive.py
index 6f830cc6..2e2b7847 100644
--- a/tests/unittests/test_datasource/test_configdrive.py
+++ b/tests/unittests/test_datasource/test_configdrive.py
@@ -494,6 +494,10 @@ class TestConfigDriveDataSource(CiTestCase):
         self.assertEqual('config-disk (/dev/anything)', cfg_ds.subplatform)
 
 
+@mock.patch(
+    "cloudinit.net.is_openvswitch_internal_interface",
+    mock.Mock(return_value=False)
+)
 class TestNetJson(CiTestCase):
     def setUp(self):
         super(TestNetJson, self).setUp()
@@ -654,6 +658,10 @@ class TestNetJson(CiTestCase):
             self.assertEqual(out_data, conv_data)
 
 
+@mock.patch(
+    "cloudinit.net.is_openvswitch_internal_interface",
+    mock.Mock(return_value=False)
+)
 class TestConvertNetworkData(CiTestCase):
 
     with_logs = True
diff --git a/tests/unittests/test_handler/test_handler_locale.py b/tests/unittests/test_handler/test_handler_locale.py
index 47e7d804..15fe7b23 100644
--- a/tests/unittests/test_handler/test_handler_locale.py
+++ b/tests/unittests/test_handler/test_handler_locale.py
@@ -44,6 +44,29 @@ class TestLocale(t_help.FilesystemMockingTestCase):
         cc = cloud.Cloud(ds, paths, {}, d, None)
         return cc
 
+    def test_set_locale_arch(self):
+        locale = 'en_GB.UTF-8'
+        locale_configfile = '/etc/invalid-locale-path'
+        cfg = {
+            'locale': locale,
+            'locale_configfile': locale_configfile,
+        }
+        cc = self._get_cloud('arch')
+
+        with mock.patch('cloudinit.distros.arch.subp.subp') as m_subp:
+            with mock.patch('cloudinit.distros.arch.LOG.warning') as m_LOG:
+                cc_locale.handle('cc_locale', cfg, cc, LOG, [])
+                m_LOG.assert_called_with('Invalid locale_configfile %s, '
+                                         'only supported value is '
+                                         '/etc/locale.conf',
+                                         locale_configfile)
+
+                contents = util.load_file(cc.distro.locale_gen_fn)
+                self.assertIn('%s UTF-8' % locale, contents)
+                m_subp.assert_called_with(['localectl',
+                                           'set-locale',
+                                           locale], capture=False)
+
     def test_set_locale_sles(self):
 
         cfg = {
diff --git a/tests/unittests/test_net.py b/tests/unittests/test_net.py
index 38d934d4..cb636f41 100644
--- a/tests/unittests/test_net.py
+++ b/tests/unittests/test_net.py
@@ -2933,6 +2933,10 @@ iface eth1 inet dhcp
         self.assertEqual(0, mock_settle.call_count)
 
 
+@mock.patch(
+    "cloudinit.net.is_openvswitch_internal_interface",
+    mock.Mock(return_value=False)
+)
 class TestRhelSysConfigRendering(CiTestCase):
 
     with_logs = True
@@ -3620,6 +3624,10 @@ USERCTL=no
                 expected, self._render_and_read(network_config=v2data))
 
 
+@mock.patch(
+    "cloudinit.net.is_openvswitch_internal_interface",
+    mock.Mock(return_value=False)
+)
 class TestOpenSuseSysConfigRendering(CiTestCase):
 
     with_logs = True
@@ -5037,6 +5045,10 @@ class TestNetRenderers(CiTestCase):
             self.assertTrue(result)
 
 
+@mock.patch(
+    "cloudinit.net.is_openvswitch_internal_interface",
+    mock.Mock(return_value=False)
+)
 class TestGetInterfaces(CiTestCase):
     _data = {'bonds': ['bond1'],
              'bridges': ['bridge1'],
@@ -5186,6 +5198,10 @@ class TestInterfaceHasOwnMac(CiTestCase):
         self.assertFalse(interface_has_own_mac("eth0"))
 
 
+@mock.patch(
+    "cloudinit.net.is_openvswitch_internal_interface",
+    mock.Mock(return_value=False)
+)
 class TestGetInterfacesByMac(CiTestCase):
     _data = {'bonds': ['bond1'],
              'bridges': ['bridge1'],
@@ -5342,6 +5358,10 @@ class TestInterfacesSorting(CiTestCase):
             ['enp0s3', 'enp0s8', 'enp0s13', 'enp1s2', 'enp2s0', 'enp2s3'])
 
 
+@mock.patch(
+    "cloudinit.net.is_openvswitch_internal_interface",
+    mock.Mock(return_value=False)
+)
 class TestGetIBHwaddrsByInterface(CiTestCase):
 
     _ib_addr = '80:00:00:28:fe:80:00:00:00:00:00:00:00:11:22:03:00:33:44:56'
diff --git a/tests/unittests/test_util.py b/tests/unittests/test_util.py
index 857629f1..e5292001 100644
--- a/tests/unittests/test_util.py
+++ b/tests/unittests/test_util.py
@@ -572,6 +572,10 @@ class TestMultiLog(helpers.FilesystemMockingTestCase):
         util.multi_log(logged_string)
         self.assertEqual(logged_string, self.stdout.getvalue())
 
+    def test_logs_dont_go_to_stdout_if_fallback_to_stdout_is_false(self):
+        util.multi_log('something', fallback_to_stdout=False)
+        self.assertEqual('', self.stdout.getvalue())
+
     def test_logs_go_to_log_if_given(self):
         log = mock.MagicMock()
         logged_string = 'something very important'
diff --git a/tools/.github-cla-signers b/tools/.github-cla-signers
index 689d7902..5c57acac 100644
--- a/tools/.github-cla-signers
+++ b/tools/.github-cla-signers
@@ -14,12 +14,14 @@ dankenigsberg
 dermotbradley
 dhensby
 eandersson
+eb3095
 emmanuelthome
 izzyleung
 johnsonshi
 jordimassaguerpla
 jqueuniet
 jsf9k
+klausenbusk
 landon912
 lucasmoura
 lungj
@@ -37,6 +39,7 @@ slyon
 smoser
 sshedi
 TheRealFalcon
+taoyama
 tnt-dev
 tomponline
 tsanghan
diff --git a/tox.ini b/tox.ini
index 0e2eae46..3158ebd5 100644
--- a/tox.ini
+++ b/tox.ini
@@ -147,13 +147,13 @@ deps =
 [testenv:integration-tests]
 basepython = python3
 commands = {envpython} -m pytest --log-cli-level=INFO {posargs:tests/integration_tests}
-passenv = CLOUD_INIT_* SSH_AUTH_SOCK
+passenv = CLOUD_INIT_* SSH_AUTH_SOCK OS_*
 deps =
     -r{toxinidir}/integration-requirements.txt
 
 [testenv:integration-tests-ci]
 commands = {envpython} -m pytest --log-cli-level=INFO {posargs:tests/integration_tests}
-passenv = CLOUD_INIT_* SSH_AUTH_SOCK
+passenv = CLOUD_INIT_* SSH_AUTH_SOCK OS_*
 deps =
     -r{toxinidir}/integration-requirements.txt
 setenv =
@@ -174,6 +174,7 @@ markers =
     gce: test will only run on GCE platform
     azure: test will only run on Azure platform
     oci: test will only run on OCI platform
+    openstack: test will only run on openstack
     lxd_config_dict: set the config_dict passed on LXD instance creation
     lxd_container: test will only run in LXD container
     lxd_use_exec: `execute` will use `lxc exec` instead of SSH