19.2-21-ge6383719-0ubuntu1 (patches unapplied)

Imported using git-ubuntu import.

13 files changed, 210 insertions, 35 deletions

diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md
new file mode 100644
index 00000000..170a71e5
--- /dev/null
+++ b/.github/pull_request_template.md
@@ -0,0 +1,9 @@
+***This GitHub repo is only a mirror.  Do not submit pull requests
+here!***
+
+Thank you for taking the time to write and submit a change to
+cloud-init!   Please follow [our hacking
+guide](https://cloudinit.readthedocs.io/en/latest/topics/hacking.html)
+to submit your change to cloud-init's [Launchpad git
+repository](https://code.launchpad.net/cloud-init/), where cloud-init
+development happens.
diff --git a/cloudinit/config/cc_apt_configure.py b/cloudinit/config/cc_apt_configure.py
index 919d1995..f01e2aaf 100644
--- a/cloudinit/config/cc_apt_configure.py
+++ b/cloudinit/config/cc_apt_configure.py
@@ -332,6 +332,8 @@ def apply_apt(cfg, cloud, target):
 
 
 def debconf_set_selections(selections, target=None):
+    if not selections.endswith(b'\n'):
+        selections += b'\n'
     util.subp(['debconf-set-selections'], data=selections, target=target,
               capture=True)
 
@@ -374,7 +376,7 @@ def apply_debconf_selections(cfg, target=None):
 
     selections = '\n'.join(
         [selsets[key] for key in sorted(selsets.keys())])
-    debconf_set_selections(selections.encode() + b"\n", target=target)
+    debconf_set_selections(selections.encode(), target=target)
 
     # get a complete list of packages listed in input
     pkgs_cfgd = set()
diff --git a/cloudinit/config/cc_ssh.py b/cloudinit/config/cc_ssh.py
index 53f69399..fdd8f4d3 100755
--- a/cloudinit/config/cc_ssh.py
+++ b/cloudinit/config/cc_ssh.py
@@ -197,7 +197,7 @@ def handle(_name, cfg, cloud, log, _args):
         hostkeys = get_public_host_keys(blacklist=host_key_blacklist)
         try:
             cloud.datasource.publish_host_keys(hostkeys)
-        except Exception as e:
+        except Exception:
             util.logexc(log, "Publishing host keys failed!")
 
     try:
diff --git a/cloudinit/config/cc_ubuntu_drivers.py b/cloudinit/config/cc_ubuntu_drivers.py
index 91feb603..297451d6 100644
--- a/cloudinit/config/cc_ubuntu_drivers.py
+++ b/cloudinit/config/cc_ubuntu_drivers.py
@@ -2,12 +2,14 @@
 
 """Ubuntu Drivers: Interact with third party drivers in Ubuntu."""
 
+import os
 from textwrap import dedent
 
 from cloudinit.config.schema import (
     get_schema_doc, validate_cloudconfig_schema)
 from cloudinit import log as logging
 from cloudinit.settings import PER_INSTANCE
+from cloudinit import temp_utils
 from cloudinit import type_utils
 from cloudinit import util
 
@@ -64,6 +66,33 @@ OLD_UBUNTU_DRIVERS_STDERR_NEEDLE = (
 __doc__ = get_schema_doc(schema)  # Supplement python help()
 
 
+# Use a debconf template to configure a global debconf variable
+# (linux/nvidia/latelink) setting this to "true" allows the
+# 'linux-restricted-modules' deb to accept the NVIDIA EULA and the package
+# will automatically link the drivers to the running kernel.
+
+# EOL_XENIAL: can then drop this script and use python3-debconf which is only
+# available in Bionic and later. Can't use python3-debconf currently as it
+# isn't in Xenial and doesn't yet support X_LOADTEMPLATEFILE debconf command.
+
+NVIDIA_DEBCONF_CONTENT = """\
+Template: linux/nvidia/latelink
+Type: boolean
+Default: true
+Description: Late-link NVIDIA kernel modules?
+ Enable this to link the NVIDIA kernel modules in cloud-init and
+ make them available for use.
+"""
+
+NVIDIA_DRIVER_LATELINK_DEBCONF_SCRIPT = """\
+#!/bin/sh
+# Allow cloud-init to trigger EULA acceptance via registering a debconf
+# template to set linux/nvidia/latelink true
+. /usr/share/debconf/confmodule
+db_x_loadtemplatefile "$1" cloud-init
+"""
+
+
 def install_drivers(cfg, pkg_install_func):
     if not isinstance(cfg, dict):
         raise TypeError(
@@ -89,9 +118,28 @@ def install_drivers(cfg, pkg_install_func):
     if version_cfg:
         driver_arg += ':{}'.format(version_cfg)
 
-    LOG.debug("Installing NVIDIA drivers (%s=%s, version=%s)",
+    LOG.debug("Installing and activating NVIDIA drivers (%s=%s, version=%s)",
               cfgpath, nv_acc, version_cfg if version_cfg else 'latest')
 
+    # Register and set debconf selection linux/nvidia/latelink = true
+    tdir = temp_utils.mkdtemp(needs_exe=True)
+    debconf_file = os.path.join(tdir, 'nvidia.template')
+    debconf_script = os.path.join(tdir, 'nvidia-debconf.sh')
+    try:
+        util.write_file(debconf_file, NVIDIA_DEBCONF_CONTENT)
+        util.write_file(
+            debconf_script,
+            util.encode_text(NVIDIA_DRIVER_LATELINK_DEBCONF_SCRIPT),
+            mode=0o755)
+        util.subp([debconf_script, debconf_file])
+    except Exception as e:
+        util.logexc(
+            LOG, "Failed to register NVIDIA debconf template: %s", str(e))
+        raise
+    finally:
+        if os.path.isdir(tdir):
+            util.del_dir(tdir)
+
     try:
         util.subp(['ubuntu-drivers', 'install', '--gpgpu', driver_arg])
     except util.ProcessExecutionError as exc:
diff --git a/cloudinit/config/tests/test_ubuntu_drivers.py b/cloudinit/config/tests/test_ubuntu_drivers.py
index efba4ce7..46952692 100644
--- a/cloudinit/config/tests/test_ubuntu_drivers.py
+++ b/cloudinit/config/tests/test_ubuntu_drivers.py
@@ -1,6 +1,7 @@
 # This file is part of cloud-init. See LICENSE file for license information.
 
 import copy
+import os
 
 from cloudinit.tests.helpers import CiTestCase, skipUnlessJsonSchema, mock
 from cloudinit.config.schema import (
@@ -9,11 +10,27 @@ from cloudinit.config import cc_ubuntu_drivers as drivers
 from cloudinit.util import ProcessExecutionError
 
 MPATH = "cloudinit.config.cc_ubuntu_drivers."
+M_TMP_PATH = MPATH + "temp_utils.mkdtemp"
 OLD_UBUNTU_DRIVERS_ERROR_STDERR = (
     "ubuntu-drivers: error: argument <command>: invalid choice: 'install' "
     "(choose from 'list', 'autoinstall', 'devices', 'debug')\n")
 
 
+class AnyTempScriptAndDebconfFile(object):
+
+    def __init__(self, tmp_dir, debconf_file):
+        self.tmp_dir = tmp_dir
+        self.debconf_file = debconf_file
+
+    def __eq__(self, cmd):
+        if not len(cmd) == 2:
+            return False
+        script, debconf_file = cmd
+        if bool(script.startswith(self.tmp_dir) and script.endswith('.sh')):
+            return debconf_file == self.debconf_file
+        return False
+
+
 class TestUbuntuDrivers(CiTestCase):
     cfg_accepted = {'drivers': {'nvidia': {'license-accepted': True}}}
     install_gpgpu = ['ubuntu-drivers', 'install', '--gpgpu', 'nvidia']
@@ -28,16 +45,23 @@ class TestUbuntuDrivers(CiTestCase):
                 {'drivers': {'nvidia': {'license-accepted': "TRUE"}}},
                 schema=drivers.schema, strict=True)
 
+    @mock.patch(M_TMP_PATH)
     @mock.patch(MPATH + "util.subp", return_value=('', ''))
     @mock.patch(MPATH + "util.which", return_value=False)
-    def _assert_happy_path_taken(self, config, m_which, m_subp):
+    def _assert_happy_path_taken(
+            self, config, m_which, m_subp, m_tmp):
         """Positive path test through handle. Package should be installed."""
+        tdir = self.tmp_dir()
+        debconf_file = os.path.join(tdir, 'nvidia.template')
+        m_tmp.return_value = tdir
         myCloud = mock.MagicMock()
         drivers.handle('ubuntu_drivers', config, myCloud, None, None)
         self.assertEqual([mock.call(['ubuntu-drivers-common'])],
                          myCloud.distro.install_packages.call_args_list)
-        self.assertEqual([mock.call(self.install_gpgpu)],
-                         m_subp.call_args_list)
+        self.assertEqual(
+            [mock.call(AnyTempScriptAndDebconfFile(tdir, debconf_file)),
+             mock.call(self.install_gpgpu)],
+            m_subp.call_args_list)
 
     def test_handle_does_package_install(self):
         self._assert_happy_path_taken(self.cfg_accepted)
@@ -48,19 +72,33 @@ class TestUbuntuDrivers(CiTestCase):
             new_config['drivers']['nvidia']['license-accepted'] = true_value
             self._assert_happy_path_taken(new_config)
 
-    @mock.patch(MPATH + "util.subp", side_effect=ProcessExecutionError(
-        stdout='No drivers found for installation.\n', exit_code=1))
+    @mock.patch(M_TMP_PATH)
+    @mock.patch(MPATH + "util.subp")
     @mock.patch(MPATH + "util.which", return_value=False)
-    def test_handle_raises_error_if_no_drivers_found(self, m_which, m_subp):
+    def test_handle_raises_error_if_no_drivers_found(
+            self, m_which, m_subp, m_tmp):
         """If ubuntu-drivers doesn't install any drivers, raise an error."""
+        tdir = self.tmp_dir()
+        debconf_file = os.path.join(tdir, 'nvidia.template')
+        m_tmp.return_value = tdir
         myCloud = mock.MagicMock()
+
+        def fake_subp(cmd):
+            if cmd[0].startswith(tdir):
+                return
+            raise ProcessExecutionError(
+                stdout='No drivers found for installation.\n', exit_code=1)
+        m_subp.side_effect = fake_subp
+
         with self.assertRaises(Exception):
             drivers.handle(
                 'ubuntu_drivers', self.cfg_accepted, myCloud, None, None)
         self.assertEqual([mock.call(['ubuntu-drivers-common'])],
                          myCloud.distro.install_packages.call_args_list)
-        self.assertEqual([mock.call(self.install_gpgpu)],
-                         m_subp.call_args_list)
+        self.assertEqual(
+            [mock.call(AnyTempScriptAndDebconfFile(tdir, debconf_file)),
+             mock.call(self.install_gpgpu)],
+            m_subp.call_args_list)
         self.assertIn('ubuntu-drivers found no drivers for installation',
                       self.logs.getvalue())
 
@@ -108,18 +146,25 @@ class TestUbuntuDrivers(CiTestCase):
                       myLog.debug.call_args_list[0][0][0])
         self.assertEqual(0, m_install_drivers.call_count)
 
+    @mock.patch(M_TMP_PATH)
     @mock.patch(MPATH + "util.subp", return_value=('', ''))
     @mock.patch(MPATH + "util.which", return_value=True)
-    def test_install_drivers_no_install_if_present(self, m_which, m_subp):
+    def test_install_drivers_no_install_if_present(
+            self, m_which, m_subp, m_tmp):
         """If 'ubuntu-drivers' is present, no package install should occur."""
+        tdir = self.tmp_dir()
+        debconf_file = os.path.join(tdir, 'nvidia.template')
+        m_tmp.return_value = tdir
         pkg_install = mock.MagicMock()
         drivers.install_drivers(self.cfg_accepted['drivers'],
                                 pkg_install_func=pkg_install)
         self.assertEqual(0, pkg_install.call_count)
         self.assertEqual([mock.call('ubuntu-drivers')],
                          m_which.call_args_list)
-        self.assertEqual([mock.call(self.install_gpgpu)],
-                         m_subp.call_args_list)
+        self.assertEqual(
+            [mock.call(AnyTempScriptAndDebconfFile(tdir, debconf_file)),
+             mock.call(self.install_gpgpu)],
+            m_subp.call_args_list)
 
     def test_install_drivers_rejects_invalid_config(self):
         """install_drivers should raise TypeError if not given a config dict"""
@@ -128,20 +173,33 @@ class TestUbuntuDrivers(CiTestCase):
             drivers.install_drivers("mystring", pkg_install_func=pkg_install)
         self.assertEqual(0, pkg_install.call_count)
 
-    @mock.patch(MPATH + "util.subp", side_effect=ProcessExecutionError(
-        stderr=OLD_UBUNTU_DRIVERS_ERROR_STDERR, exit_code=2))
+    @mock.patch(M_TMP_PATH)
+    @mock.patch(MPATH + "util.subp")
     @mock.patch(MPATH + "util.which", return_value=False)
     def test_install_drivers_handles_old_ubuntu_drivers_gracefully(
-            self, m_which, m_subp):
+            self, m_which, m_subp, m_tmp):
         """Older ubuntu-drivers versions should emit message and raise error"""
+        tdir = self.tmp_dir()
+        debconf_file = os.path.join(tdir, 'nvidia.template')
+        m_tmp.return_value = tdir
         myCloud = mock.MagicMock()
+
+        def fake_subp(cmd):
+            if cmd[0].startswith(tdir):
+                return
+            raise ProcessExecutionError(
+                stderr=OLD_UBUNTU_DRIVERS_ERROR_STDERR, exit_code=2)
+        m_subp.side_effect = fake_subp
+
         with self.assertRaises(Exception):
             drivers.handle(
                 'ubuntu_drivers', self.cfg_accepted, myCloud, None, None)
         self.assertEqual([mock.call(['ubuntu-drivers-common'])],
                          myCloud.distro.install_packages.call_args_list)
-        self.assertEqual([mock.call(self.install_gpgpu)],
-                         m_subp.call_args_list)
+        self.assertEqual(
+            [mock.call(AnyTempScriptAndDebconfFile(tdir, debconf_file)),
+             mock.call(self.install_gpgpu)],
+            m_subp.call_args_list)
         self.assertIn('WARNING: the available version of ubuntu-drivers is'
                       ' too old to perform requested driver installation',
                       self.logs.getvalue())
@@ -153,16 +211,21 @@ class TestUbuntuDriversWithVersion(TestUbuntuDrivers):
         'drivers': {'nvidia': {'license-accepted': True, 'version': '123'}}}
     install_gpgpu = ['ubuntu-drivers', 'install', '--gpgpu', 'nvidia:123']
 
+    @mock.patch(M_TMP_PATH)
     @mock.patch(MPATH + "util.subp", return_value=('', ''))
     @mock.patch(MPATH + "util.which", return_value=False)
-    def test_version_none_uses_latest(self, m_which, m_subp):
+    def test_version_none_uses_latest(self, m_which, m_subp, m_tmp):
+        tdir = self.tmp_dir()
+        debconf_file = os.path.join(tdir, 'nvidia.template')
+        m_tmp.return_value = tdir
         myCloud = mock.MagicMock()
         version_none_cfg = {
             'drivers': {'nvidia': {'license-accepted': True, 'version': None}}}
         drivers.handle(
             'ubuntu_drivers', version_none_cfg, myCloud, None, None)
         self.assertEqual(
-            [mock.call(['ubuntu-drivers', 'install', '--gpgpu', 'nvidia'])],
+            [mock.call(AnyTempScriptAndDebconfFile(tdir, debconf_file)),
+             mock.call(['ubuntu-drivers', 'install', '--gpgpu', 'nvidia'])],
             m_subp.call_args_list)
 
     def test_specifying_a_version_doesnt_override_license_acceptance(self):
diff --git a/cloudinit/distros/parsers/sys_conf.py b/cloudinit/distros/parsers/sys_conf.py
index c27b5d5d..44df17de 100644
--- a/cloudinit/distros/parsers/sys_conf.py
+++ b/cloudinit/distros/parsers/sys_conf.py
@@ -43,6 +43,13 @@ def _contains_shell_variable(text):
 
 
 class SysConf(configobj.ConfigObj):
+    """A configobj.ConfigObj subclass specialised for sysconfig files.
+
+    :param contents:
+        The sysconfig file to parse, in a format accepted by
+        ``configobj.ConfigObj.__init__`` (i.e. "a filename, file like object,
+        or list of lines").
+    """
     def __init__(self, contents):
         configobj.ConfigObj.__init__(self, contents,
                                      interpolation=False,
diff --git a/cloudinit/sources/DataSourceOracle.py b/cloudinit/sources/DataSourceOracle.py
index 086af799..6e73f568 100644
--- a/cloudinit/sources/DataSourceOracle.py
+++ b/cloudinit/sources/DataSourceOracle.py
@@ -109,6 +109,13 @@ class DataSourceOracle(sources.DataSource):
     dsname = 'Oracle'
     system_uuid = None
     vendordata_pure = None
+    network_config_sources = (
+        sources.NetworkConfigSource.cmdline,
+        sources.NetworkConfigSource.ds,
+        sources.NetworkConfigSource.initramfs,
+        sources.NetworkConfigSource.system_cfg,
+    )
+
     _network_config = sources.UNSET
 
     def __init__(self, sys_cfg, *args, **kwargs):
diff --git a/cloudinit/sources/tests/test_oracle.py b/cloudinit/sources/tests/test_oracle.py
index 3e146776..3ddf7dfd 100644
--- a/cloudinit/sources/tests/test_oracle.py
+++ b/cloudinit/sources/tests/test_oracle.py
@@ -1,7 +1,7 @@
 # This file is part of cloud-init. See LICENSE file for license information.
 
 from cloudinit.sources import DataSourceOracle as oracle
-from cloudinit.sources import BrokenMetadata
+from cloudinit.sources import BrokenMetadata, NetworkConfigSource
 from cloudinit import helpers
 
 from cloudinit.tests import helpers as test_helpers
@@ -303,6 +303,14 @@ class TestDataSourceOracle(test_helpers.CiTestCase):
         self.assertIn('Failed to fetch secondary network configuration',
                       self.logs.getvalue())
 
+    def test_ds_network_cfg_preferred_over_initramfs(self):
+        """Ensure that DS net config is preferred over initramfs config"""
+        network_config_sources = oracle.DataSourceOracle.network_config_sources
+        self.assertLess(
+            network_config_sources.index(NetworkConfigSource.ds),
+            network_config_sources.index(NetworkConfigSource.initramfs)
+        )
+
 
 @mock.patch(DS_PATH + "._read_system_uuid", return_value=str(uuid.uuid4()))
 class TestReadMetaData(test_helpers.HttprettyTestCase):
diff --git a/cloudinit/tests/helpers.py b/cloudinit/tests/helpers.py
index f41180fd..23fddd07 100644
--- a/cloudinit/tests/helpers.py
+++ b/cloudinit/tests/helpers.py
@@ -198,7 +198,8 @@ class CiTestCase(TestCase):
                 prefix="ci-%s." % self.__class__.__name__)
         else:
             tmpd = tempfile.mkdtemp(dir=dir)
-        self.addCleanup(functools.partial(shutil.rmtree, tmpd))
+        self.addCleanup(
+            functools.partial(shutil.rmtree, tmpd, ignore_errors=True))
         return tmpd
 
     def tmp_path(self, path, dir=None):
diff --git a/debian/changelog b/debian/changelog
index 0a31157a..84b106d3 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,3 +1,20 @@
+cloud-init (19.2-21-ge6383719-0ubuntu1) eoan; urgency=medium
+
+  * New upstream snapshot.
+    - ubuntu-drivers: call db_x_loadtemplatefile to accept NVIDIA EULA
+      (LP: #1840080)
+    - Add missing #cloud-config comment on first example in documentation.
+      [Florian Müller]
+    - ubuntu-drivers: emit latelink=true debconf to accept nvidia eula
+      (LP: #1840080)
+    - DataSourceOracle: prefer DS network config over initramfs
+    - format.rst: add text/jinja2 to list of content types (+ cleanups)
+    - Add GitHub pull request template to point people at hacking doc
+    - cloudinit/distros/parsers/sys_conf: add docstring to SysConf
+    - pyflakes: remove unused variable [Joshua Powers]
+
+ -- Chad Smith <chad.smith@canonical.com>  Thu, 22 Aug 2019 11:15:08 -0600
+
 cloud-init (19.2-13-g2f3bb764-0ubuntu1) eoan; urgency=medium
 
   * New upstream snapshot.
diff --git a/doc/examples/cloud-config-user-groups.txt b/doc/examples/cloud-config-user-groups.txt
index 6a363b77..f588bfbc 100644
--- a/doc/examples/cloud-config-user-groups.txt
+++ b/doc/examples/cloud-config-user-groups.txt
@@ -1,3 +1,4 @@
+#cloud-config
 # Add groups to the system
 # The following example adds the ubuntu group with members 'root' and 'sys'
 # and the empty group cloud-users.
diff --git a/doc/rtd/topics/format.rst b/doc/rtd/topics/format.rst
index 15234d21..74d1fee9 100644
--- a/doc/rtd/topics/format.rst
+++ b/doc/rtd/topics/format.rst
@@ -23,14 +23,15 @@ For example, both a user data script and a cloud-config type could be specified.
 
 Supported content-types:
 
-- text/x-include-once-url
-- text/x-include-url
-- text/cloud-config-archive
-- text/upstart-job
+- text/cloud-boothook
 - text/cloud-config
+- text/cloud-config-archive
+- text/jinja2
 - text/part-handler
+- text/upstart-job
+- text/x-include-once-url
+- text/x-include-url
 - text/x-shellscript
-- text/cloud-boothook
 
 Helper script to generate mime messages
 ---------------------------------------
@@ -38,16 +39,16 @@ Helper script to generate mime messages
 .. code-block:: python
 
    #!/usr/bin/python
-   
+
    import sys
-   
+
    from email.mime.multipart import MIMEMultipart
    from email.mime.text import MIMEText
-   
+
    if len(sys.argv) == 1:
        print("%s input-file:type ..." % (sys.argv[0]))
        sys.exit(1)
-   
+
    combined_message = MIMEMultipart()
    for i in sys.argv[1:]:
        (filename, format_type) = i.split(":", 1)
@@ -56,7 +57,7 @@ Helper script to generate mime messages
        sub_message = MIMEText(contents, format_type, sys.getdefaultencoding())
        sub_message.add_header('Content-Disposition', 'attachment; filename="%s"' % (filename))
        combined_message.attach(sub_message)
-   
+
    print(combined_message)
 
 
@@ -78,10 +79,10 @@ Example
 ::
 
   $ cat myscript.sh
-  
+
   #!/bin/sh
   echo "Hello World.  The time is now $(date -R)!" | tee /root/output.txt
-  
+
   $ euca-run-instances --key mykey --user-data-file myscript.sh ami-a07d95c9 
 
 Include File
diff --git a/tests/unittests/test_handler/test_handler_apt_source_v3.py b/tests/unittests/test_handler/test_handler_apt_source_v3.py
index 90fe6eed..2f21b6dc 100644
--- a/tests/unittests/test_handler/test_handler_apt_source_v3.py
+++ b/tests/unittests/test_handler/test_handler_apt_source_v3.py
@@ -998,6 +998,17 @@ deb http://ubuntu.com/ubuntu/ xenial-proposed main""")
 
 class TestDebconfSelections(TestCase):
 
+    @mock.patch("cloudinit.config.cc_apt_configure.util.subp")
+    def test_set_sel_appends_newline_if_absent(self, m_subp):
+        """Automatically append a newline to debconf-set-selections config."""
+        selections = b'some/setting boolean true'
+        cc_apt_configure.debconf_set_selections(selections=selections)
+        cc_apt_configure.debconf_set_selections(selections=selections + b'\n')
+        m_call = mock.call(
+            ['debconf-set-selections'], data=selections + b'\n', capture=True,
+            target=None)
+        self.assertEqual([m_call, m_call], m_subp.call_args_list)
+
     @mock.patch("cloudinit.config.cc_apt_configure.debconf_set_selections")
     def test_no_set_sel_if_none_to_set(self, m_set_sel):
         cc_apt_configure.apply_debconf_selections({'foo': 'bar'})