instance-data: Add standard keys platform and subplatform. Refactor ec2.

Add the following instance-data.json standardized keys:
* v1._beta_keys: List any v1 keys in beta development,
  e.g. ['subplatform'].
* v1.public_ssh_keys: List of any cloud-provided ssh keys for the
  instance.
* v1.platform: String representing the cloud platform api supporting the
  datasource. For example: 'ec2' for aws, aliyun and brightbox cloud
  names.
* v1.subplatform: String with more details about the source of the
  metadata consumed. For example, metadata uri, config drive device path
  or seed directory.

To support the new platform and subplatform standardized instance-data,
DataSource and its subclasses grew platform and subplatform attributes.
The platform attribute defaults to the lowercase string datasource name at
self.dsname. This method is overridden in NoCloud, Ec2 and ConfigDrive
datasources.

The subplatform attribute calls a _get_subplatform method which will
return a string containing a simple slug for subplatform type such as
metadata, seed-dir or config-drive followed by a detailed uri, device or
directory path where the datasource consumed its configuration.

As part of this work, DatasourceEC2 methods _get_data and _crawl_metadata
have been refactored for a few reasons:
- crawl_metadata is now a read-only operation, persisting no attributes on
  the datasource instance and returns a dictionary of consumed metadata.
- crawl_metadata now closely represents the raw stucture of the ec2
  metadata consumed, so that end-users can leverage public ec2 metadata
  documentation where possible.
- crawl_metadata adds a '_metadata_api_version' key to the crawled
  ds.metadata to advertise what version of EC2's api was consumed by
  cloud-init.
- _get_data now does all the processing of crawl_metadata and saves
  datasource instance attributes userdata_raw, metadata etc.

Additional drive-bys:
* unit test rework for test_altcloud and test_azure to simplify mocks
  and make use of existing util and test_helpers functions.

18 files changed, 264 insertions, 106 deletions

diff --git a/cloudinit/sources/DataSourceAliYun.py b/cloudinit/sources/DataSourceAliYun.py
index 858e0827..45cc9f00 100644
--- a/cloudinit/sources/DataSourceAliYun.py
+++ b/cloudinit/sources/DataSourceAliYun.py
@@ -1,7 +1,5 @@
 # This file is part of cloud-init. See LICENSE file for license information.
 
-import os
-
 from cloudinit import sources
 from cloudinit.sources import DataSourceEc2 as EC2
 from cloudinit import util
@@ -18,25 +16,17 @@ class DataSourceAliYun(EC2.DataSourceEc2):
     min_metadata_version = '2016-01-01'
     extended_metadata_versions = []
 
-    def __init__(self, sys_cfg, distro, paths):
-        super(DataSourceAliYun, self).__init__(sys_cfg, distro, paths)
-        self.seed_dir = os.path.join(paths.seed_dir, "AliYun")
-
     def get_hostname(self, fqdn=False, resolve_ip=False, metadata_only=False):
         return self.metadata.get('hostname', 'localhost.localdomain')
 
     def get_public_ssh_keys(self):
         return parse_public_keys(self.metadata.get('public-keys', {}))
 
-    @property
-    def cloud_platform(self):
-        if self._cloud_platform is None:
-            if _is_aliyun():
-                self._cloud_platform = EC2.Platforms.ALIYUN
-            else:
-                self._cloud_platform = EC2.Platforms.NO_EC2_METADATA
-
-        return self._cloud_platform
+    def _get_cloud_name(self):
+        if _is_aliyun():
+            return EC2.CloudNames.ALIYUN
+        else:
+            return EC2.CloudNames.NO_EC2_METADATA
 
 
 def _is_aliyun():
diff --git a/cloudinit/sources/DataSourceAltCloud.py b/cloudinit/sources/DataSourceAltCloud.py
index 8cd312d0..5270fda8 100644
--- a/cloudinit/sources/DataSourceAltCloud.py
+++ b/cloudinit/sources/DataSourceAltCloud.py
@@ -89,7 +89,9 @@ class DataSourceAltCloud(sources.DataSource):
         '''
         Description:
             Get the type for the cloud back end this instance is running on
-            by examining the string returned by reading the dmi data.
+            by examining the string returned by reading either:
+                CLOUD_INFO_FILE or
+                the dmi data.
 
         Input:
             None
@@ -99,7 +101,14 @@ class DataSourceAltCloud(sources.DataSource):
             'RHEV', 'VSPHERE' or 'UNKNOWN'
 
         '''
-
+        if os.path.exists(CLOUD_INFO_FILE):
+            try:
+                cloud_type = util.load_file(CLOUD_INFO_FILE).strip().upper()
+            except IOError:
+                util.logexc(LOG, 'Unable to access cloud info file at %s.',
+                            CLOUD_INFO_FILE)
+                return 'UNKNOWN'
+            return cloud_type
         system_name = util.read_dmi_data("system-product-name")
         if not system_name:
             return 'UNKNOWN'
@@ -134,15 +143,7 @@ class DataSourceAltCloud(sources.DataSource):
 
         LOG.debug('Invoked get_data()')
 
-        if os.path.exists(CLOUD_INFO_FILE):
-            try:
-                cloud_type = util.load_file(CLOUD_INFO_FILE).strip().upper()
-            except IOError:
-                util.logexc(LOG, 'Unable to access cloud info file at %s.',
-                            CLOUD_INFO_FILE)
-                return False
-        else:
-            cloud_type = self.get_cloud_type()
+        cloud_type = self.get_cloud_type()
 
         LOG.debug('cloud_type: %s', str(cloud_type))
 
@@ -161,6 +162,15 @@ class DataSourceAltCloud(sources.DataSource):
         util.logexc(LOG, 'Failed accessing user data.')
         return False
 
+    def _get_subplatform(self):
+        """Return the subplatform metadata details."""
+        cloud_type = self.get_cloud_type()
+        if not hasattr(self, 'source'):
+            self.source = sources.METADATA_UNKNOWN
+        if cloud_type == 'RHEV':
+            self.source = '/dev/fd0'
+        return '%s (%s)' % (cloud_type.lower(), self.source)
+
     def user_data_rhevm(self):
         '''
         RHEVM specific userdata read
@@ -232,6 +242,7 @@ class DataSourceAltCloud(sources.DataSource):
             try:
                 return_str = util.mount_cb(cdrom_dev, read_user_data_callback)
                 if return_str:
+                    self.source = cdrom_dev
                     break
             except OSError as err:
                 if err.errno != errno.ENOENT:
diff --git a/cloudinit/sources/DataSourceAzure.py b/cloudinit/sources/DataSourceAzure.py
index 783445e1..39391d01 100644
--- a/cloudinit/sources/DataSourceAzure.py
+++ b/cloudinit/sources/DataSourceAzure.py
@@ -351,6 +351,14 @@ class DataSourceAzure(sources.DataSource):
         metadata['public-keys'] = key_value or pubkeys_from_crt_files(fp_files)
         return metadata
 
+    def _get_subplatform(self):
+        """Return the subplatform metadata source details."""
+        if self.seed.startswith('/dev'):
+            subplatform_type = 'config-disk'
+        else:
+            subplatform_type = 'seed-dir'
+        return '%s (%s)' % (subplatform_type, self.seed)
+
     def crawl_metadata(self):
         """Walk all instance metadata sources returning a dict on success.
 
diff --git a/cloudinit/sources/DataSourceBigstep.py b/cloudinit/sources/DataSourceBigstep.py
index 699a85b5..52fff20a 100644
--- a/cloudinit/sources/DataSourceBigstep.py
+++ b/cloudinit/sources/DataSourceBigstep.py
@@ -36,6 +36,10 @@ class DataSourceBigstep(sources.DataSource):
         self.userdata_raw = decoded["userdata_raw"]
         return True
 
+    def _get_subplatform(self):
+        """Return the subplatform metadata source details."""
+        return 'metadata (%s)' % get_url_from_file()
+
 
 def get_url_from_file():
     try:
diff --git a/cloudinit/sources/DataSourceCloudSigma.py b/cloudinit/sources/DataSourceCloudSigma.py
index c816f349..2955d3f0 100644
--- a/cloudinit/sources/DataSourceCloudSigma.py
+++ b/cloudinit/sources/DataSourceCloudSigma.py
@@ -7,7 +7,7 @@
 from base64 import b64decode
 import re
 
-from cloudinit.cs_utils import Cepko
+from cloudinit.cs_utils import Cepko, SERIAL_PORT
 
 from cloudinit import log as logging
 from cloudinit import sources
@@ -84,6 +84,10 @@ class DataSourceCloudSigma(sources.DataSource):
 
         return True
 
+    def _get_subplatform(self):
+        """Return the subplatform metadata source details."""
+        return 'cepko (%s)' % SERIAL_PORT
+
     def get_hostname(self, fqdn=False, resolve_ip=False, metadata_only=False):
         """
         Cleans up and uses the server's name if the latter is set. Otherwise
diff --git a/cloudinit/sources/DataSourceConfigDrive.py b/cloudinit/sources/DataSourceConfigDrive.py
index 664dc4b7..564e3eb3 100644
--- a/cloudinit/sources/DataSourceConfigDrive.py
+++ b/cloudinit/sources/DataSourceConfigDrive.py
@@ -160,6 +160,18 @@ class DataSourceConfigDrive(openstack.SourceMixin, sources.DataSource):
                 LOG.debug("no network configuration available")
         return self._network_config
 
+    @property
+    def platform(self):
+        return 'openstack'
+
+    def _get_subplatform(self):
+        """Return the subplatform metadata source details."""
+        if self.seed_dir in self.source:
+            subplatform_type = 'seed-dir'
+        elif self.source.startswith('/dev'):
+            subplatform_type = 'config-disk'
+        return '%s (%s)' % (subplatform_type, self.source)
+
 
 def read_config_drive(source_dir):
     reader = openstack.ConfigDriveReader(source_dir)
diff --git a/cloudinit/sources/DataSourceEc2.py b/cloudinit/sources/DataSourceEc2.py
index 968ab3f7..9ccf2cdc 100644
--- a/cloudinit/sources/DataSourceEc2.py
+++ b/cloudinit/sources/DataSourceEc2.py
@@ -28,18 +28,16 @@ STRICT_ID_PATH = ("datasource", "Ec2", "strict_id")
 STRICT_ID_DEFAULT = "warn"
 
 
-class Platforms(object):
-    # TODO Rename and move to cloudinit.cloud.CloudNames
-    ALIYUN = "AliYun"
-    AWS = "AWS"
-    BRIGHTBOX = "Brightbox"
-    SEEDED = "Seeded"
+class CloudNames(object):
+    ALIYUN = "aliyun"
+    AWS = "aws"
+    BRIGHTBOX = "brightbox"
     # UNKNOWN indicates no positive id.  If strict_id is 'warn' or 'false',
     # then an attempt at the Ec2 Metadata service will be made.
-    UNKNOWN = "Unknown"
+    UNKNOWN = "unknown"
     # NO_EC2_METADATA indicates this platform does not have a Ec2 metadata
     # service available. No attempt at the Ec2 Metadata service will be made.
-    NO_EC2_METADATA = "No-EC2-Metadata"
+    NO_EC2_METADATA = "no-ec2-metadata"
 
 
 class DataSourceEc2(sources.DataSource):
@@ -61,8 +59,6 @@ class DataSourceEc2(sources.DataSource):
     url_max_wait = 120
     url_timeout = 50
 
-    _cloud_platform = None
-
     _network_config = sources.UNSET  # Used to cache calculated network cfg v1
 
     # Whether we want to get network configuration from the metadata service.
@@ -71,30 +67,21 @@ class DataSourceEc2(sources.DataSource):
     def __init__(self, sys_cfg, distro, paths):
         super(DataSourceEc2, self).__init__(sys_cfg, distro, paths)
         self.metadata_address = None
-        self.seed_dir = os.path.join(paths.seed_dir, "ec2")
 
     def _get_cloud_name(self):
         """Return the cloud name as identified during _get_data."""
-        return self.cloud_platform
+        return identify_platform()
 
     def _get_data(self):
-        seed_ret = {}
-        if util.read_optional_seed(seed_ret, base=(self.seed_dir + "/")):
-            self.userdata_raw = seed_ret['user-data']
-            self.metadata = seed_ret['meta-data']
-            LOG.debug("Using seeded ec2 data from %s", self.seed_dir)
-            self._cloud_platform = Platforms.SEEDED
-            return True
-
         strict_mode, _sleep = read_strict_mode(
             util.get_cfg_by_path(self.sys_cfg, STRICT_ID_PATH,
                                  STRICT_ID_DEFAULT), ("warn", None))
 
-        LOG.debug("strict_mode: %s, cloud_platform=%s",
-                  strict_mode, self.cloud_platform)
-        if strict_mode == "true" and self.cloud_platform == Platforms.UNKNOWN:
+        LOG.debug("strict_mode: %s, cloud_name=%s cloud_platform=%s",
+                  strict_mode, self.cloud_name, self.platform)
+        if strict_mode == "true" and self.cloud_name == CloudNames.UNKNOWN:
             return False
-        elif self.cloud_platform == Platforms.NO_EC2_METADATA:
+        elif self.cloud_name == CloudNames.NO_EC2_METADATA:
             return False
 
         if self.perform_dhcp_setup:  # Setup networking in init-local stage.
@@ -103,13 +90,22 @@ class DataSourceEc2(sources.DataSource):
                 return False
             try:
                 with EphemeralDHCPv4(self.fallback_interface):
-                    return util.log_time(
+                    self._crawled_metadata = util.log_time(
                         logfunc=LOG.debug, msg='Crawl of metadata service',
-                        func=self._crawl_metadata)
+                        func=self.crawl_metadata)
             except NoDHCPLeaseError:
                 return False
         else:
-            return self._crawl_metadata()
+            self._crawled_metadata = util.log_time(
+                logfunc=LOG.debug, msg='Crawl of metadata service',
+                func=self.crawl_metadata)
+        if not self._crawled_metadata:
+            return False
+        self.metadata = self._crawled_metadata.get('meta-data', None)
+        self.userdata_raw = self._crawled_metadata.get('user-data', None)
+        self.identity = self._crawled_metadata.get(
+            'dynamic', {}).get('instance-identity', {}).get('document', {})
+        return True
 
     @property
     def launch_index(self):
@@ -117,6 +113,15 @@ class DataSourceEc2(sources.DataSource):
             return None
         return self.metadata.get('ami-launch-index')
 
+    @property
+    def platform(self):
+        # Handle upgrade path of pickled ds
+        if not hasattr(self, '_platform_type'):
+            self._platform_type = DataSourceEc2.dsname.lower()
+        if not self._platform_type:
+            self._platform_type = DataSourceEc2.dsname.lower()
+        return self._platform_type
+
     def get_metadata_api_version(self):
         """Get the best supported api version from the metadata service.
 
@@ -144,7 +149,7 @@ class DataSourceEc2(sources.DataSource):
         return self.min_metadata_version
 
     def get_instance_id(self):
-        if self.cloud_platform == Platforms.AWS:
+        if self.cloud_name == CloudNames.AWS:
             # Prefer the ID from the instance identity document, but fall back
             if not getattr(self, 'identity', None):
                 # If re-using cached datasource, it's get_data run didn't
@@ -254,7 +259,7 @@ class DataSourceEc2(sources.DataSource):
     @property
     def availability_zone(self):
         try:
-            if self.cloud_platform == Platforms.AWS:
+            if self.cloud_name == CloudNames.AWS:
                 return self.identity.get(
                     'availabilityZone',
                     self.metadata['placement']['availability-zone'])
@@ -265,7 +270,7 @@ class DataSourceEc2(sources.DataSource):
 
     @property
     def region(self):
-        if self.cloud_platform == Platforms.AWS:
+        if self.cloud_name == CloudNames.AWS:
             region = self.identity.get('region')
             # Fallback to trimming the availability zone if region is missing
             if self.availability_zone and not region:
@@ -277,16 +282,10 @@ class DataSourceEc2(sources.DataSource):
                 return az[:-1]
         return None
 
-    @property
-    def cloud_platform(self):  # TODO rename cloud_name
-        if self._cloud_platform is None:
-            self._cloud_platform = identify_platform()
-        return self._cloud_platform
-
     def activate(self, cfg, is_new_instance):
         if not is_new_instance:
             return
-        if self.cloud_platform == Platforms.UNKNOWN:
+        if self.cloud_name == CloudNames.UNKNOWN:
             warn_if_necessary(
                 util.get_cfg_by_path(cfg, STRICT_ID_PATH, STRICT_ID_DEFAULT),
                 cfg)
@@ -306,13 +305,13 @@ class DataSourceEc2(sources.DataSource):
         result = None
         no_network_metadata_on_aws = bool(
             'network' not in self.metadata and
-            self.cloud_platform == Platforms.AWS)
+            self.cloud_name == CloudNames.AWS)
         if no_network_metadata_on_aws:
             LOG.debug("Metadata 'network' not present:"
                       " Refreshing stale metadata from prior to upgrade.")
             util.log_time(
                 logfunc=LOG.debug, msg='Re-crawl of metadata service',
-                func=self._crawl_metadata)
+                func=self.get_data)
 
         # Limit network configuration to only the primary/fallback nic
         iface = self.fallback_interface
@@ -340,28 +339,32 @@ class DataSourceEc2(sources.DataSource):
                 return super(DataSourceEc2, self).fallback_interface
         return self._fallback_interface
 
-    def _crawl_metadata(self):
+    def crawl_metadata(self):
         """Crawl metadata service when available.
 
-        @returns: True on success, False otherwise.
+        @returns: Dictionary of crawled metadata content containing the keys:
+          meta-data, user-data and dynamic.
         """
         if not self.wait_for_metadata_service():
-            return False
+            return {}
         api_version = self.get_metadata_api_version()
+        crawled_metadata = {}
         try:
-            self.userdata_raw = ec2.get_instance_userdata(
+            crawled_metadata['user-data'] = ec2.get_instance_userdata(
                 api_version, self.metadata_address)
-            self.metadata = ec2.get_instance_metadata(
+            crawled_metadata['meta-data'] = ec2.get_instance_metadata(
                 api_version, self.metadata_address)
-            if self.cloud_platform == Platforms.AWS:
-                self.identity = ec2.get_instance_identity(
-                    api_version, self.metadata_address).get('document', {})
+            if self.cloud_name == CloudNames.AWS:
+                identity = ec2.get_instance_identity(
+                    api_version, self.metadata_address)
+                crawled_metadata['dynamic'] = {'instance-identity': identity}
         except Exception:
             util.logexc(
                 LOG, "Failed reading from metadata address %s",
                 self.metadata_address)
-            return False
-        return True
+            return {}
+        crawled_metadata['_metadata_api_version'] = api_version
+        return crawled_metadata
 
 
 class DataSourceEc2Local(DataSourceEc2):
@@ -375,10 +378,10 @@ class DataSourceEc2Local(DataSourceEc2):
     perform_dhcp_setup = True  # Use dhcp before querying metadata
 
     def get_data(self):
-        supported_platforms = (Platforms.AWS,)
-        if self.cloud_platform not in supported_platforms:
+        supported_platforms = (CloudNames.AWS,)
+        if self.cloud_name not in supported_platforms:
             LOG.debug("Local Ec2 mode only supported on %s, not %s",
-                      supported_platforms, self.cloud_platform)
+                      supported_platforms, self.cloud_name)
             return False
         return super(DataSourceEc2Local, self).get_data()
 
@@ -439,20 +442,20 @@ def identify_aws(data):
     if (data['uuid'].startswith('ec2') and
             (data['uuid_source'] == 'hypervisor' or
              data['uuid'] == data['serial'])):
-            return Platforms.AWS
+            return CloudNames.AWS
 
     return None
 
 
 def identify_brightbox(data):
     if data['serial'].endswith('brightbox.com'):
-        return Platforms.BRIGHTBOX
+        return CloudNames.BRIGHTBOX
 
 
 def identify_platform():
-    # identify the platform and return an entry in Platforms.
+    # identify the platform and return an entry in CloudNames.
     data = _collect_platform_data()
-    checks = (identify_aws, identify_brightbox, lambda x: Platforms.UNKNOWN)
+    checks = (identify_aws, identify_brightbox, lambda x: CloudNames.UNKNOWN)
     for checker in checks:
         try:
             result = checker(data)
diff --git a/cloudinit/sources/DataSourceIBMCloud.py b/cloudinit/sources/DataSourceIBMCloud.py
index a5358148..21e6ae6b 100644
--- a/cloudinit/sources/DataSourceIBMCloud.py
+++ b/cloudinit/sources/DataSourceIBMCloud.py
@@ -157,6 +157,10 @@ class DataSourceIBMCloud(sources.DataSource):
 
         return True
 
+    def _get_subplatform(self):
+        """Return the subplatform metadata source details."""
+        return '%s (%s)' % (self.platform, self.source)
+
     def check_instance_id(self, sys_cfg):
         """quickly (local check only) if self.instance_id is still valid
 
diff --git a/cloudinit/sources/DataSourceMAAS.py b/cloudinit/sources/DataSourceMAAS.py
index bcb38544..61aa6d7e 100644
--- a/cloudinit/sources/DataSourceMAAS.py
+++ b/cloudinit/sources/DataSourceMAAS.py
@@ -109,6 +109,10 @@ class DataSourceMAAS(sources.DataSource):
                 LOG.warning("Invalid content in vendor-data: %s", e)
                 self.vendordata_raw = None
 
+    def _get_subplatform(self):
+        """Return the subplatform metadata source details."""
+        return 'seed-dir (%s)' % self.base_url
+
     def wait_for_metadata_service(self, url):
         mcfg = self.ds_cfg
         max_wait = 120
diff --git a/cloudinit/sources/DataSourceNoCloud.py b/cloudinit/sources/DataSourceNoCloud.py
index 2daea59d..9010f06c 100644
--- a/cloudinit/sources/DataSourceNoCloud.py
+++ b/cloudinit/sources/DataSourceNoCloud.py
@@ -186,6 +186,27 @@ class DataSourceNoCloud(sources.DataSource):
         self._network_eni = mydata['meta-data'].get('network-interfaces')
         return True
 
+    @property
+    def platform_type(self):
+        # Handle upgrade path of pickled ds
+        if not hasattr(self, '_platform_type'):
+            self._platform_type = None
+        if not self._platform_type:
+            self._platform_type = 'lxd' if util.is_lxd() else 'nocloud'
+        return self._platform_type
+
+    def _get_cloud_name(self):
+        """Return unknown when 'cloud-name' key is absent from metadata."""
+        return sources.METADATA_UNKNOWN
+
+    def _get_subplatform(self):
+        """Return the subplatform metadata source details."""
+        if self.seed.startswith('/dev'):
+            subplatform_type = 'config-disk'
+        else:
+            subplatform_type = 'seed-dir'
+        return '%s (%s)' % (subplatform_type, self.seed)
+
     def check_instance_id(self, sys_cfg):
         # quickly (local check only) if self.instance_id is still valid
         # we check kernel command line or files.
diff --git a/cloudinit/sources/DataSourceNone.py b/cloudinit/sources/DataSourceNone.py
index e63a7e39..e6250801 100644
--- a/cloudinit/sources/DataSourceNone.py
+++ b/cloudinit/sources/DataSourceNone.py
@@ -28,6 +28,10 @@ class DataSourceNone(sources.DataSource):
             self.metadata = self.ds_cfg['metadata']
         return True
 
+    def _get_subplatform(self):
+        """Return the subplatform metadata source details."""
+        return 'config'
+
     def get_instance_id(self):
         return 'iid-datasource-none'
 
diff --git a/cloudinit/sources/DataSourceOVF.py b/cloudinit/sources/DataSourceOVF.py
index 178ccb0f..045291e7 100644
--- a/cloudinit/sources/DataSourceOVF.py
+++ b/cloudinit/sources/DataSourceOVF.py
@@ -275,6 +275,12 @@ class DataSourceOVF(sources.DataSource):
         self.cfg = cfg
         return True
 
+    def _get_subplatform(self):
+        system_type = util.read_dmi_data("system-product-name").lower()
+        if system_type == 'vmware':
+            return 'vmware (%s)' % self.seed
+        return 'ovf (%s)' % self.seed
+
     def get_public_ssh_keys(self):
         if 'public-keys' not in self.metadata:
             return []
diff --git a/cloudinit/sources/DataSourceOpenNebula.py b/cloudinit/sources/DataSourceOpenNebula.py
index 77ccd128..e62e9729 100644
--- a/cloudinit/sources/DataSourceOpenNebula.py
+++ b/cloudinit/sources/DataSourceOpenNebula.py
@@ -95,6 +95,14 @@ class DataSourceOpenNebula(sources.DataSource):
         self.userdata_raw = results.get('userdata')
         return True
 
+    def _get_subplatform(self):
+        """Return the subplatform metadata source details."""
+        if self.seed_dir in self.seed:
+            subplatform_type = 'seed-dir'
+        else:
+            subplatform_type = 'config-disk'
+        return '%s (%s)' % (subplatform_type, self.seed)
+
     @property
     def network_config(self):
         if self.network is not None:
diff --git a/cloudinit/sources/DataSourceOracle.py b/cloudinit/sources/DataSourceOracle.py
index fab39af3..70b9c58a 100644
--- a/cloudinit/sources/DataSourceOracle.py
+++ b/cloudinit/sources/DataSourceOracle.py
@@ -91,6 +91,10 @@ class DataSourceOracle(sources.DataSource):
     def crawl_metadata(self):
         return read_metadata()
 
+    def _get_subplatform(self):
+        """Return the subplatform metadata source details."""
+        return 'metadata (%s)' % METADATA_ENDPOINT
+
     def check_instance_id(self, sys_cfg):
         """quickly check (local only) if self.instance_id is still valid
 
diff --git a/cloudinit/sources/DataSourceSmartOS.py b/cloudinit/sources/DataSourceSmartOS.py
index 593ac91a..32b57cdd 100644
--- a/cloudinit/sources/DataSourceSmartOS.py
+++ b/cloudinit/sources/DataSourceSmartOS.py
@@ -303,6 +303,9 @@ class DataSourceSmartOS(sources.DataSource):
         self._set_provisioned()
         return True
 
+    def _get_subplatform(self):
+        return 'serial (%s)' % SERIAL_DEVICE
+
     def device_name_to_device(self, name):
         return self.ds_cfg['disk_aliases'].get(name)
 
diff --git a/cloudinit/sources/__init__.py b/cloudinit/sources/__init__.py
index 5ac98826..9b90680f 100644
--- a/cloudinit/sources/__init__.py
+++ b/cloudinit/sources/__init__.py
@@ -54,6 +54,7 @@ REDACT_SENSITIVE_VALUE = 'redacted for non-root user'
 METADATA_CLOUD_NAME_KEY = 'cloud-name'
 
 UNSET = "_unset"
+METADATA_UNKNOWN = 'unknown'
 
 LOG = logging.getLogger(__name__)
 
@@ -133,6 +134,14 @@ class DataSource(object):
     # Cached cloud_name as determined by _get_cloud_name
     _cloud_name = None
 
+    # Cached cloud platform api type: e.g. ec2, openstack, kvm, lxd, azure etc.
+    _platform_type = None
+
+    # More details about the cloud platform:
+    #  - metadata (http://169.254.169.254/)
+    #  - seed-dir (<dirname>)
+    _subplatform = None
+
     # Track the discovered fallback nic for use in configuration generation.
     _fallback_interface = None
 
@@ -192,21 +201,24 @@ class DataSource(object):
         local_hostname = self.get_hostname()
         instance_id = self.get_instance_id()
         availability_zone = self.availability_zone
-        cloud_name = self.cloud_name
-        # When adding new standard keys prefer underscore-delimited instead
-        # of hyphen-delimted to support simple variable references in jinja
-        # templates.
+        # In the event of upgrade from existing cloudinit, pickled datasource
+        # will not contain these new class attributes. So we need to recrawl
+        # metadata to discover that content.
         return {
             'v1': {
+                '_beta_keys': ['subplatform'],
                 'availability-zone': availability_zone,
                 'availability_zone': availability_zone,
-                'cloud-name': cloud_name,
-                'cloud_name': cloud_name,
+                'cloud-name': self.cloud_name,
+                'cloud_name': self.cloud_name,
+                'platform': self.platform_type,
+                'public_ssh_keys': self.get_public_ssh_keys(),
                 'instance-id': instance_id,
                 'instance_id': instance_id,
                 'local-hostname': local_hostname,
                 'local_hostname': local_hostname,
-                'region': self.region}}
+                'region': self.region,
+                'subplatform': self.subplatform}}
 
     def clear_cached_attrs(self, attr_defaults=()):
         """Reset any cached metadata attributes to datasource defaults.
@@ -247,19 +259,27 @@ class DataSource(object):
 
         @return True on successful write, False otherwise.
         """
-        instance_data = {
-            'ds': {'_doc': EXPERIMENTAL_TEXT,
-                   'meta_data': self.metadata}}
-        if hasattr(self, 'network_json'):
-            network_json = getattr(self, 'network_json')
-            if network_json != UNSET:
-                instance_data['ds']['network_json'] = network_json
-        if hasattr(self, 'ec2_metadata'):
-            ec2_metadata = getattr(self, 'ec2_metadata')
-            if ec2_metadata != UNSET:
-                instance_data['ds']['ec2_metadata'] = ec2_metadata
+        if hasattr(self, '_crawled_metadata'):
+            # Any datasource with _crawled_metadata will best represent
+            # most recent, 'raw' metadata
+            crawled_metadata = copy.deepcopy(
+                getattr(self, '_crawled_metadata'))
+            crawled_metadata.pop('user-data', None)
+            crawled_metadata.pop('vendor-data', None)
+            instance_data = {'ds': crawled_metadata}
+        else:
+            instance_data = {'ds': {'meta_data': self.metadata}}
+            if hasattr(self, 'network_json'):
+                network_json = getattr(self, 'network_json')
+                if network_json != UNSET:
+                    instance_data['ds']['network_json'] = network_json
+            if hasattr(self, 'ec2_metadata'):
+                ec2_metadata = getattr(self, 'ec2_metadata')
+                if ec2_metadata != UNSET:
+                    instance_data['ds']['ec2_metadata'] = ec2_metadata
         instance_data.update(
             self._get_standardized_metadata())
+        instance_data['ds']['_doc'] = EXPERIMENTAL_TEXT
         try:
             # Process content base64encoding unserializable values
             content = util.json_dumps(instance_data)
@@ -347,6 +367,40 @@ class DataSource(object):
         return self._fallback_interface
 
     @property
+    def platform_type(self):
+        if not hasattr(self, '_platform_type'):
+            # Handle upgrade path where pickled datasource has no _platform.
+            self._platform_type = self.dsname.lower()
+        if not self._platform_type:
+            self._platform_type = self.dsname.lower()
+        return self._platform_type
+
+    @property
+    def subplatform(self):
+        """Return a string representing subplatform details for the datasource.
+
+        This should be guidance for where the metadata is sourced.
+        Examples of this on different clouds:
+            ec2:       metadata (http://169.254.169.254)
+            openstack: configdrive (/dev/path)
+            openstack: metadata (http://169.254.169.254)
+            nocloud:   seed-dir (/seed/dir/path)
+            lxd:   nocloud (/seed/dir/path)
+        """
+        if not hasattr(self, '_subplatform'):
+            # Handle upgrade path where pickled datasource has no _platform.
+            self._subplatform = self._get_subplatform()
+        if not self._subplatform:
+            self._subplatform = self._get_subplatform()
+        return self._subplatform
+
+    def _get_subplatform(self):
+        """Subclasses should implement to return a "slug (detail)" string."""
+        if hasattr(self, 'metadata_address'):
+            return 'metadata (%s)' % getattr(self, 'metadata_address')
+        return METADATA_UNKNOWN
+
+    @property
     def cloud_name(self):
         """Return lowercase cloud name as determined by the datasource.
 
@@ -359,9 +413,11 @@ class DataSource(object):
             cloud_name = self.metadata.get(METADATA_CLOUD_NAME_KEY)
             if isinstance(cloud_name, six.string_types):
                 self._cloud_name = cloud_name.lower()
-            LOG.debug(
-                'Ignoring metadata provided key %s: non-string type %s',
-                METADATA_CLOUD_NAME_KEY, type(cloud_name))
+            else:
+                self._cloud_name = self._get_cloud_name().lower()
+                LOG.debug(
+                    'Ignoring metadata provided key %s: non-string type %s',
+                    METADATA_CLOUD_NAME_KEY, type(cloud_name))
         else:
             self._cloud_name = self._get_cloud_name().lower()
         return self._cloud_name
diff --git a/cloudinit/sources/tests/test_init.py b/cloudinit/sources/tests/test_init.py
index 8082019e..391b3436 100644
--- a/cloudinit/sources/tests/test_init.py
+++ b/cloudinit/sources/tests/test_init.py
@@ -295,6 +295,7 @@ class TestDataSource(CiTestCase):
             'base64_encoded_keys': [],
             'sensitive_keys': [],
             'v1': {
+                '_beta_keys': ['subplatform'],
                 'availability-zone': 'myaz',
                 'availability_zone': 'myaz',
                 'cloud-name': 'subclasscloudname',
@@ -303,7 +304,10 @@ class TestDataSource(CiTestCase):
                 'instance_id': 'iid-datasource',
                 'local-hostname': 'test-subclass-hostname',
                 'local_hostname': 'test-subclass-hostname',
-                'region': 'myregion'},
+                'platform': 'mytestsubclass',
+                'public_ssh_keys': [],
+                'region': 'myregion',
+                'subplatform': 'unknown'},
             'ds': {
                 '_doc': EXPERIMENTAL_TEXT,
                 'meta_data': {'availability_zone': 'myaz',
@@ -339,6 +343,7 @@ class TestDataSource(CiTestCase):
             'base64_encoded_keys': [],
             'sensitive_keys': ['ds/meta_data/some/security-credentials'],
             'v1': {
+                '_beta_keys': ['subplatform'],
                 'availability-zone': 'myaz',
                 'availability_zone': 'myaz',
                 'cloud-name': 'subclasscloudname',
@@ -347,7 +352,10 @@ class TestDataSource(CiTestCase):
                 'instance_id': 'iid-datasource',
                 'local-hostname': 'test-subclass-hostname',
                 'local_hostname': 'test-subclass-hostname',
-                'region': 'myregion'},
+                'platform': 'mytestsubclass',
+                'public_ssh_keys': [],
+                'region': 'myregion',
+                'subplatform': 'unknown'},
             'ds': {
                 '_doc': EXPERIMENTAL_TEXT,
                 'meta_data': {
diff --git a/cloudinit/sources/tests/test_oracle.py b/cloudinit/sources/tests/test_oracle.py
index 7599126c..97d62947 100644
--- a/cloudinit/sources/tests/test_oracle.py
+++ b/cloudinit/sources/tests/test_oracle.py
@@ -71,6 +71,14 @@ class TestDataSourceOracle(test_helpers.CiTestCase):
         self.assertFalse(ds._get_data())
         mocks._is_platform_viable.assert_called_once_with()
 
+    def test_platform_info(self):
+        """Return platform-related information for Oracle Datasource."""
+        ds, _mocks = self._get_ds()
+        self.assertEqual('oracle', ds.cloud_name)
+        self.assertEqual('oracle', ds.platform_type)
+        self.assertEqual(
+            'metadata (http://169.254.169.254/openstack/)', ds.subplatform)
+
     @mock.patch(DS_PATH + "._is_iscsi_root", return_value=True)
     def test_without_userdata(self, m_is_iscsi_root):
         """If no user-data is provided, it should not be in return dict."""