update_metadata: a datasource can support network re-config every boot

Very basic type definitions are now defined to distinguish 'boot'
events from 'new instance (first boot)'. Event types will now be handed
to a datasource.update_metadata method which can determine whether
to refresh its metadata and re-render configuration based on that
source event.

A datasource can 'subscribe' to an event by setting up the update_events
attribute on the datasource class which describe what config scope is
updated by a list of matching events. By default datasources will have
the following update_events: {'network': [EventType.BOOT_NEW_INSTANCE]}
This setting says the datasource will re-write network configuration only
on first boot of a new instance or when the instance id changes.

New methods are now present on the datasource:
  - clear_cached_attrs: Resets cached datasource attributes to values
    listed in datasource.cached_attr_defaults. This is performed prior to
    processing a fresh metadata process to avoid keeping old/invalid
    cached data around.
  - update_metadata: accepts source_event_types to determine if the
    metadata should be crawled again and processed

5 files changed, 417 insertions, 6 deletions

diff --git a/cloudinit/event.py b/cloudinit/event.py
new file mode 100644
index 00000000..f7b311fb
--- /dev/null
+++ b/cloudinit/event.py
@@ -0,0 +1,17 @@
+# This file is part of cloud-init. See LICENSE file for license information.
+
+"""Classes and functions related to event handling."""
+
+
+# Event types which can generate maintenance requests for cloud-init.
+class EventType(object):
+    BOOT = "System boot"
+    BOOT_NEW_INSTANCE = "New instance first boot"
+
+    # TODO: Cloud-init will grow support for the follow event types:
+    # UDEV
+    # METADATA_CHANGE
+    # USER_REQUEST
+
+
+# vi: ts=4 expandtab
diff --git a/cloudinit/sources/__init__.py b/cloudinit/sources/__init__.py
index 90d74575..f424316a 100644
--- a/cloudinit/sources/__init__.py
+++ b/cloudinit/sources/__init__.py
@@ -19,6 +19,7 @@ from cloudinit.atomic_helper import write_json
 from cloudinit import importer
 from cloudinit import log as logging
 from cloudinit import net
+from cloudinit.event import EventType
 from cloudinit import type_utils
 from cloudinit import user_data as ud
 from cloudinit import util
@@ -102,6 +103,25 @@ class DataSource(object):
     url_timeout = 10    # timeout for each metadata url read attempt
     url_retries = 5     # number of times to retry url upon 404
 
+    # The datasource defines a list of supported EventTypes during which
+    # the datasource can react to changes in metadata and regenerate
+    # network configuration on metadata changes.
+    # A datasource which supports writing network config on each system boot
+    # would set update_events = {'network': [EventType.BOOT]}
+
+    # Default: generate network config on new instance id (first boot).
+    update_events = {'network': [EventType.BOOT_NEW_INSTANCE]}
+
+    # N-tuple listing default values for any metadata-related class
+    # attributes cached on an instance by a process_data runs. These attribute
+    # values are reset via clear_cached_attrs during any update_metadata call.
+    cached_attr_defaults = (
+        ('ec2_metadata', UNSET), ('network_json', UNSET),
+        ('metadata', {}), ('userdata', None), ('userdata_raw', None),
+        ('vendordata', None), ('vendordata_raw', None))
+
+    _dirty_cache = False
+
     def __init__(self, sys_cfg, distro, paths, ud_proc=None):
         self.sys_cfg = sys_cfg
         self.distro = distro
@@ -134,11 +154,31 @@ class DataSource(object):
             'region': self.region,
             'availability-zone': self.availability_zone}}
 
+    def clear_cached_attrs(self, attr_defaults=()):
+        """Reset any cached metadata attributes to datasource defaults.
+
+        @param attr_defaults: Optional tuple of (attr, value) pairs to
+           set instead of cached_attr_defaults.
+        """
+        if not self._dirty_cache:
+            return
+        if attr_defaults:
+            attr_values = attr_defaults
+        else:
+            attr_values = self.cached_attr_defaults
+
+        for attribute, value in attr_values:
+            if hasattr(self, attribute):
+                setattr(self, attribute, value)
+        if not attr_defaults:
+            self._dirty_cache = False
+
     def get_data(self):
         """Datasources implement _get_data to setup metadata and userdata_raw.
 
         Minimally, the datasource should return a boolean True on success.
         """
+        self._dirty_cache = True
         return_value = self._get_data()
         json_file = os.path.join(self.paths.run_dir, INSTANCE_JSON_FILE)
         if not return_value:
@@ -174,6 +214,7 @@ class DataSource(object):
         return return_value
 
     def _get_data(self):
+        """Walk metadata sources, process crawled data and save attributes."""
         raise NotImplementedError(
             'Subclasses of DataSource must implement _get_data which'
             ' sets self.metadata, vendordata_raw and userdata_raw.')
@@ -416,6 +457,41 @@ class DataSource(object):
     def get_package_mirror_info(self):
         return self.distro.get_package_mirror_info(data_source=self)
 
+    def update_metadata(self, source_event_types):
+        """Refresh cached metadata if the datasource supports this event.
+
+        The datasource has a list of update_events which
+        trigger refreshing all cached metadata as well as refreshing the
+        network configuration.
+
+        @param source_event_types: List of EventTypes which may trigger a
+            metadata update.
+
+        @return True if the datasource did successfully update cached metadata
+            due to source_event_type.
+        """
+        supported_events = {}
+        for event in source_event_types:
+            for update_scope, update_events in self.update_events.items():
+                if event in update_events:
+                    if not supported_events.get(update_scope):
+                        supported_events[update_scope] = []
+                    supported_events[update_scope].append(event)
+        for scope, matched_events in supported_events.items():
+            LOG.debug(
+                "Update datasource metadata and %s config due to events: %s",
+                scope, ', '.join(matched_events))
+            # Each datasource has a cached config property which needs clearing
+            # Once cleared that config property will be regenerated from
+            # current metadata.
+            self.clear_cached_attrs((('_%s_config' % scope, UNSET),))
+        if supported_events:
+            self.clear_cached_attrs()
+            result = self.get_data()
+            if result:
+                return True
+        return False
+
     def check_instance_id(self, sys_cfg):
         # quickly (local check only) if self.instance_id is still
         return False
@@ -520,7 +596,7 @@ def find_source(sys_cfg, distro, paths, ds_deps, cfg_list, pkg_list, reporter):
             with myrep:
                 LOG.debug("Seeing if we can get any data from %s", cls)
                 s = cls(sys_cfg, distro, paths)
-                if s.get_data():
+                if s.update_metadata([EventType.BOOT_NEW_INSTANCE]):
                     myrep.message = "found %s data from %s" % (mode, name)
                     return (s, type_utils.obj_name(cls))
         except Exception:
diff --git a/cloudinit/sources/tests/test_init.py b/cloudinit/sources/tests/test_init.py
index d5bc98a4..dcd221be 100644
--- a/cloudinit/sources/tests/test_init.py
+++ b/cloudinit/sources/tests/test_init.py
@@ -5,10 +5,11 @@ import os
 import six
 import stat
 
+from cloudinit.event import EventType
 from cloudinit.helpers import Paths
 from cloudinit import importer
 from cloudinit.sources import (
-    INSTANCE_JSON_FILE, DataSource)
+    INSTANCE_JSON_FILE, DataSource, UNSET)
 from cloudinit.tests.helpers import CiTestCase, skipIf, mock
 from cloudinit.user_data import UserDataProcessor
 from cloudinit import util
@@ -381,3 +382,83 @@ class TestDataSource(CiTestCase):
                     get_args(grandchild.get_hostname),  # pylint: disable=W1505
                     '%s does not implement DataSource.get_hostname params'
                     % grandchild)
+
+    def test_clear_cached_attrs_resets_cached_attr_class_attributes(self):
+        """Class attributes listed in cached_attr_defaults are reset."""
+        count = 0
+        # Setup values for all cached class attributes
+        for attr, value in self.datasource.cached_attr_defaults:
+            setattr(self.datasource, attr, count)
+            count += 1
+        self.datasource._dirty_cache = True
+        self.datasource.clear_cached_attrs()
+        for attr, value in self.datasource.cached_attr_defaults:
+            self.assertEqual(value, getattr(self.datasource, attr))
+
+    def test_clear_cached_attrs_noops_on_clean_cache(self):
+        """Class attributes listed in cached_attr_defaults are reset."""
+        count = 0
+        # Setup values for all cached class attributes
+        for attr, _ in self.datasource.cached_attr_defaults:
+            setattr(self.datasource, attr, count)
+            count += 1
+        self.datasource._dirty_cache = False   # Fake clean cache
+        self.datasource.clear_cached_attrs()
+        count = 0
+        for attr, _ in self.datasource.cached_attr_defaults:
+            self.assertEqual(count, getattr(self.datasource, attr))
+            count += 1
+
+    def test_clear_cached_attrs_skips_non_attr_class_attributes(self):
+        """Skip any cached_attr_defaults which aren't class attributes."""
+        self.datasource._dirty_cache = True
+        self.datasource.clear_cached_attrs()
+        for attr in ('ec2_metadata', 'network_json'):
+            self.assertFalse(hasattr(self.datasource, attr))
+
+    def test_clear_cached_attrs_of_custom_attrs(self):
+        """Custom attr_values can be passed to clear_cached_attrs."""
+        self.datasource._dirty_cache = True
+        cached_attr_name = self.datasource.cached_attr_defaults[0][0]
+        setattr(self.datasource, cached_attr_name, 'himom')
+        self.datasource.myattr = 'orig'
+        self.datasource.clear_cached_attrs(
+            attr_defaults=(('myattr', 'updated'),))
+        self.assertEqual('himom', getattr(self.datasource, cached_attr_name))
+        self.assertEqual('updated', self.datasource.myattr)
+
+    def test_update_metadata_only_acts_on_supported_update_events(self):
+        """update_metadata won't get_data on unsupported update events."""
+        self.assertEqual(
+            {'network': [EventType.BOOT_NEW_INSTANCE]},
+            self.datasource.update_events)
+
+        def fake_get_data():
+            raise Exception('get_data should not be called')
+
+        self.datasource.get_data = fake_get_data
+        self.assertFalse(
+            self.datasource.update_metadata(
+                source_event_types=[EventType.BOOT]))
+
+    def test_update_metadata_returns_true_on_supported_update_event(self):
+        """update_metadata returns get_data response on supported events."""
+
+        def fake_get_data():
+            return True
+
+        self.datasource.get_data = fake_get_data
+        self.datasource._network_config = 'something'
+        self.datasource._dirty_cache = True
+        self.assertTrue(
+            self.datasource.update_metadata(
+                source_event_types=[
+                    EventType.BOOT, EventType.BOOT_NEW_INSTANCE]))
+        self.assertEqual(UNSET, self.datasource._network_config)
+        self.assertIn(
+            "DEBUG: Update datasource metadata and network config due to"
+            " events: New instance first boot",
+            self.logs.getvalue())
+
+
+# vi: ts=4 expandtab
diff --git a/cloudinit/stages.py b/cloudinit/stages.py
index 286607bf..c132b57d 100644
--- a/cloudinit/stages.py
+++ b/cloudinit/stages.py
@@ -22,6 +22,8 @@ from cloudinit.handlers import cloud_config as cc_part
 from cloudinit.handlers import shell_script as ss_part
 from cloudinit.handlers import upstart_job as up_part
 
+from cloudinit.event import EventType
+
 from cloudinit import cloud
 from cloudinit import config
 from cloudinit import distros
@@ -648,10 +650,14 @@ class Init(object):
         except Exception as e:
             LOG.warning("Failed to rename devices: %s", e)
 
-        if (self.datasource is not NULL_DATA_SOURCE and
-                not self.is_new_instance()):
-            LOG.debug("not a new instance. network config is not applied.")
-            return
+        if self.datasource is not NULL_DATA_SOURCE:
+            if not self.is_new_instance():
+                if not self.datasource.update_metadata([EventType.BOOT]):
+                    LOG.debug(
+                        "No network config applied. Neither a new instance"
+                        " nor datasource network update on '%s' event",
+                        EventType.BOOT)
+                    return
 
         LOG.info("Applying network configuration from %s bringup=%s: %s",
                  src, bring_up, netcfg)
diff --git a/cloudinit/tests/test_stages.py b/cloudinit/tests/test_stages.py
new file mode 100644
index 00000000..94b6b255
--- /dev/null
+++ b/cloudinit/tests/test_stages.py
@@ -0,0 +1,231 @@
+# This file is part of cloud-init. See LICENSE file for license information.
+
+"""Tests related to cloudinit.stages module."""
+
+import os
+
+from cloudinit import stages
+from cloudinit import sources
+
+from cloudinit.event import EventType
+from cloudinit.util import write_file
+
+from cloudinit.tests.helpers import CiTestCase, mock
+
+TEST_INSTANCE_ID = 'i-testing'
+
+
+class FakeDataSource(sources.DataSource):
+
+    def __init__(self, paths=None, userdata=None, vendordata=None,
+                 network_config=''):
+        super(FakeDataSource, self).__init__({}, None, paths=paths)
+        self.metadata = {'instance-id': TEST_INSTANCE_ID}
+        self.userdata_raw = userdata
+        self.vendordata_raw = vendordata
+        self._network_config = None
+        if network_config:   # Permit for None value to setup attribute
+            self._network_config = network_config
+
+    @property
+    def network_config(self):
+        return self._network_config
+
+    def _get_data(self):
+        return True
+
+
+class TestInit(CiTestCase):
+    with_logs = True
+
+    def setUp(self):
+        super(TestInit, self).setUp()
+        self.tmpdir = self.tmp_dir()
+        self.init = stages.Init()
+        # Setup fake Paths for Init to reference
+        self.init._cfg = {'system_info': {
+            'distro': 'ubuntu', 'paths': {'cloud_dir': self.tmpdir,
+                                          'run_dir': self.tmpdir}}}
+        self.init.datasource = FakeDataSource(paths=self.init.paths)
+
+    def test_wb__find_networking_config_disabled(self):
+        """find_networking_config returns no config when disabled."""
+        disable_file = os.path.join(
+            self.init.paths.get_cpath('data'), 'upgraded-network')
+        write_file(disable_file, '')
+        self.assertEqual(
+            (None, disable_file),
+            self.init._find_networking_config())
+
+    @mock.patch('cloudinit.stages.cmdline.read_kernel_cmdline_config')
+    def test_wb__find_networking_config_disabled_by_kernel(self, m_cmdline):
+        """find_networking_config returns when disabled by kernel cmdline."""
+        m_cmdline.return_value = {'config': 'disabled'}
+        self.assertEqual(
+            (None, 'cmdline'),
+            self.init._find_networking_config())
+        self.assertEqual('DEBUG: network config disabled by cmdline\n',
+                         self.logs.getvalue())
+
+    @mock.patch('cloudinit.stages.cmdline.read_kernel_cmdline_config')
+    def test_wb__find_networking_config_disabled_by_datasrc(self, m_cmdline):
+        """find_networking_config returns when disabled by datasource cfg."""
+        m_cmdline.return_value = {}  # Kernel doesn't disable networking
+        self.init._cfg = {'system_info': {'paths': {'cloud_dir': self.tmpdir}},
+                          'network': {}}  # system config doesn't disable
+
+        self.init.datasource = FakeDataSource(
+            network_config={'config': 'disabled'})
+        self.assertEqual(
+            (None, 'ds'),
+            self.init._find_networking_config())
+        self.assertEqual('DEBUG: network config disabled by ds\n',
+                         self.logs.getvalue())
+
+    @mock.patch('cloudinit.stages.cmdline.read_kernel_cmdline_config')
+    def test_wb__find_networking_config_disabled_by_sysconfig(self, m_cmdline):
+        """find_networking_config returns when disabled by system config."""
+        m_cmdline.return_value = {}  # Kernel doesn't disable networking
+        self.init._cfg = {'system_info': {'paths': {'cloud_dir': self.tmpdir}},
+                          'network': {'config': 'disabled'}}
+        self.assertEqual(
+            (None, 'system_cfg'),
+            self.init._find_networking_config())
+        self.assertEqual('DEBUG: network config disabled by system_cfg\n',
+                         self.logs.getvalue())
+
+    @mock.patch('cloudinit.stages.cmdline.read_kernel_cmdline_config')
+    def test_wb__find_networking_config_returns_kernel(self, m_cmdline):
+        """find_networking_config returns kernel cmdline config if present."""
+        expected_cfg = {'config': ['fakekernel']}
+        m_cmdline.return_value = expected_cfg
+        self.init._cfg = {'system_info': {'paths': {'cloud_dir': self.tmpdir}},
+                          'network': {'config': ['fakesys_config']}}
+        self.init.datasource = FakeDataSource(
+            network_config={'config': ['fakedatasource']})
+        self.assertEqual(
+            (expected_cfg, 'cmdline'),
+            self.init._find_networking_config())
+
+    @mock.patch('cloudinit.stages.cmdline.read_kernel_cmdline_config')
+    def test_wb__find_networking_config_returns_system_cfg(self, m_cmdline):
+        """find_networking_config returns system config when present."""
+        m_cmdline.return_value = {}  # No kernel network config
+        expected_cfg = {'config': ['fakesys_config']}
+        self.init._cfg = {'system_info': {'paths': {'cloud_dir': self.tmpdir}},
+                          'network': expected_cfg}
+        self.init.datasource = FakeDataSource(
+            network_config={'config': ['fakedatasource']})
+        self.assertEqual(
+            (expected_cfg, 'system_cfg'),
+            self.init._find_networking_config())
+
+    @mock.patch('cloudinit.stages.cmdline.read_kernel_cmdline_config')
+    def test_wb__find_networking_config_returns_datasrc_cfg(self, m_cmdline):
+        """find_networking_config returns datasource net config if present."""
+        m_cmdline.return_value = {}  # No kernel network config
+        # No system config for network in setUp
+        expected_cfg = {'config': ['fakedatasource']}
+        self.init.datasource = FakeDataSource(network_config=expected_cfg)
+        self.assertEqual(
+            (expected_cfg, 'ds'),
+            self.init._find_networking_config())
+
+    @mock.patch('cloudinit.stages.cmdline.read_kernel_cmdline_config')
+    def test_wb__find_networking_config_returns_fallback(self, m_cmdline):
+        """find_networking_config returns fallback config if not defined."""
+        m_cmdline.return_value = {}  # Kernel doesn't disable networking
+        # Neither datasource nor system_info disable or provide network
+
+        fake_cfg = {'config': [{'type': 'physical', 'name': 'eth9'}],
+                    'version': 1}
+
+        def fake_generate_fallback():
+            return fake_cfg
+
+        # Monkey patch distro which gets cached on self.init
+        distro = self.init.distro
+        distro.generate_fallback_config = fake_generate_fallback
+        self.assertEqual(
+            (fake_cfg, 'fallback'),
+            self.init._find_networking_config())
+        self.assertNotIn('network config disabled', self.logs.getvalue())
+
+    def test_apply_network_config_disabled(self):
+        """Log when network is disabled by upgraded-network."""
+        disable_file = os.path.join(
+            self.init.paths.get_cpath('data'), 'upgraded-network')
+
+        def fake_network_config():
+            return (None, disable_file)
+
+        self.init._find_networking_config = fake_network_config
+
+        self.init.apply_network_config(True)
+        self.assertIn(
+            'INFO: network config is disabled by %s' % disable_file,
+            self.logs.getvalue())
+
+    @mock.patch('cloudinit.distros.ubuntu.Distro')
+    def test_apply_network_on_new_instance(self, m_ubuntu):
+        """Call distro apply_network_config methods on is_new_instance."""
+        net_cfg = {
+            'version': 1, 'config': [
+                {'subnets': [{'type': 'dhcp'}], 'type': 'physical',
+                 'name': 'eth9', 'mac_address': '42:42:42:42:42:42'}]}
+
+        def fake_network_config():
+            return net_cfg, 'fallback'
+
+        self.init._find_networking_config = fake_network_config
+        self.init.apply_network_config(True)
+        self.init.distro.apply_network_config_names.assert_called_with(net_cfg)
+        self.init.distro.apply_network_config.assert_called_with(
+            net_cfg, bring_up=True)
+
+    @mock.patch('cloudinit.distros.ubuntu.Distro')
+    def test_apply_network_on_same_instance_id(self, m_ubuntu):
+        """Only call distro.apply_network_config_names on same instance id."""
+        old_instance_id = os.path.join(
+            self.init.paths.get_cpath('data'), 'instance-id')
+        write_file(old_instance_id, TEST_INSTANCE_ID)
+        net_cfg = {
+            'version': 1, 'config': [
+                {'subnets': [{'type': 'dhcp'}], 'type': 'physical',
+                 'name': 'eth9', 'mac_address': '42:42:42:42:42:42'}]}
+
+        def fake_network_config():
+            return net_cfg, 'fallback'
+
+        self.init._find_networking_config = fake_network_config
+        self.init.apply_network_config(True)
+        self.init.distro.apply_network_config_names.assert_called_with(net_cfg)
+        self.init.distro.apply_network_config.assert_not_called()
+        self.assertIn(
+            'No network config applied. Neither a new instance'
+            " nor datasource network update on '%s' event" % EventType.BOOT,
+            self.logs.getvalue())
+
+    @mock.patch('cloudinit.distros.ubuntu.Distro')
+    def test_apply_network_on_datasource_allowed_event(self, m_ubuntu):
+        """Apply network if datasource.update_metadata permits BOOT event."""
+        old_instance_id = os.path.join(
+            self.init.paths.get_cpath('data'), 'instance-id')
+        write_file(old_instance_id, TEST_INSTANCE_ID)
+        net_cfg = {
+            'version': 1, 'config': [
+                {'subnets': [{'type': 'dhcp'}], 'type': 'physical',
+                 'name': 'eth9', 'mac_address': '42:42:42:42:42:42'}]}
+
+        def fake_network_config():
+            return net_cfg, 'fallback'
+
+        self.init._find_networking_config = fake_network_config
+        self.init.datasource = FakeDataSource(paths=self.init.paths)
+        self.init.datasource.update_events = {'network': [EventType.BOOT]}
+        self.init.apply_network_config(True)
+        self.init.distro.apply_network_config_names.assert_called_with(net_cfg)
+        self.init.distro.apply_network_config.assert_called_with(
+            net_cfg, bring_up=True)
+
+# vi: ts=4 expandtab