Decouple Sahara templates test from vanilla plugin

The change introduces a list of template data for node templates and
clusters for each plugin. The first known plugin and its first known
version are used.

At least one default template is added for each plugin, so that all
the real deployment can be covered.

The tests are skipped if the plugins key in tempest configuration is empty;
they fail if no known plugin or no known version for known plugins are found
(which probably means that the test configuration should be updated).

The list must be maintained; on the other way, after some discussions,
the alternative solution - expose all the node group and cluster
templates parameters in tempest configuration - would have made the
testing more random and too dependent on the cloud operator in charge
of running tempest.
The list of plugins is not probably going to change massively for each
release, so it seems feasible.

Change-Id: I397616f7923e696082fe67da3ff9841f89e9a505

3 files changed, 336 insertions, 63 deletions

diff --git a/tempest/api/data_processing/base.py b/tempest/api/data_processing/base.py
index 5992921d7..7f3474b7d 100644
--- a/tempest/api/data_processing/base.py
+++ b/tempest/api/data_processing/base.py
@@ -12,14 +12,216 @@
 #    License for the specific language governing permissions and limitations
 #    under the License.
 
+from collections import OrderedDict
+
+import six
 from tempest_lib import exceptions as lib_exc
 
 from tempest import config
+from tempest import exceptions
 import tempest.test
 
 
 CONF = config.CONF
 
+"""Default templates.
+There should always be at least a master1 and a worker1 node
+group template."""
+DEFAULT_TEMPLATES = {
+    'vanilla': OrderedDict([
+        ('2.6.0', {
+            'NODES': {
+                'master1': {
+                    'count': 1,
+                    'node_processes': ['namenode', 'resourcemanager',
+                                       'hiveserver']
+                },
+                'master2': {
+                    'count': 1,
+                    'node_processes': ['oozie', 'historyserver',
+                                       'secondarynamenode']
+                },
+                'worker1': {
+                    'count': 1,
+                    'node_processes': ['datanode', 'nodemanager'],
+                    'node_configs': {
+                        'MapReduce': {
+                            'yarn.app.mapreduce.am.resource.mb': 256,
+                            'yarn.app.mapreduce.am.command-opts': '-Xmx256m'
+                        },
+                        'YARN': {
+                            'yarn.scheduler.minimum-allocation-mb': 256,
+                            'yarn.scheduler.maximum-allocation-mb': 1024,
+                            'yarn.nodemanager.vmem-check-enabled': False
+                        }
+                    }
+                }
+            },
+            'cluster_configs': {
+                'HDFS': {
+                    'dfs.replication': 1
+                }
+            }
+        }),
+        ('1.2.1', {
+            'NODES': {
+                'master1': {
+                    'count': 1,
+                    'node_processes': ['namenode', 'jobtracker']
+                },
+                'worker1': {
+                    'count': 1,
+                    'node_processes': ['datanode', 'tasktracker'],
+                    'node_configs': {
+                        'HDFS': {
+                            'Data Node Heap Size': 1024
+                        },
+                        'MapReduce': {
+                            'Task Tracker Heap Size': 1024
+                        }
+                    }
+                }
+            },
+            'cluster_configs': {
+                'HDFS': {
+                    'dfs.replication': 1
+                },
+                'MapReduce': {
+                    'mapred.map.tasks.speculative.execution': False,
+                    'mapred.child.java.opts': '-Xmx500m'
+                },
+                'general': {
+                    'Enable Swift': False
+                }
+            }
+        })
+    ]),
+    'hdp': OrderedDict([
+        ('2.0.6', {
+            'NODES': {
+                'master1': {
+                    'count': 1,
+                    'node_processes': ['NAMENODE', 'SECONDARY_NAMENODE',
+                                       'ZOOKEEPER_SERVER', 'AMBARI_SERVER',
+                                       'HISTORYSERVER', 'RESOURCEMANAGER',
+                                       'GANGLIA_SERVER', 'NAGIOS_SERVER',
+                                       'OOZIE_SERVER']
+                },
+                'worker1': {
+                    'count': 1,
+                    'node_processes': ['HDFS_CLIENT', 'DATANODE',
+                                       'YARN_CLIENT', 'ZOOKEEPER_CLIENT',
+                                       'MAPREDUCE2_CLIENT', 'NODEMANAGER',
+                                       'PIG', 'OOZIE_CLIENT']
+                }
+            },
+            'cluster_configs': {
+                'HDFS': {
+                    'dfs.replication': 1
+                }
+            }
+        })
+    ]),
+    'spark': OrderedDict([
+        ('1.0.0', {
+            'NODES': {
+                'master1': {
+                    'count': 1,
+                    'node_processes': ['namenode', 'master']
+                },
+                'worker1': {
+                    'count': 1,
+                    'node_processes': ['datanode', 'slave']
+                }
+            },
+            'cluster_configs': {
+                'HDFS': {
+                    'dfs.replication': 1
+                }
+            }
+        })
+    ]),
+    'cdh': OrderedDict([
+        ('5.3.0', {
+            'NODES': {
+                'master1': {
+                    'count': 1,
+                    'node_processes': ['CLOUDERA_MANAGER']
+                },
+                'master2': {
+                    'count': 1,
+                    'node_processes': ['HDFS_NAMENODE',
+                                       'YARN_RESOURCEMANAGER']
+                },
+                'master3': {
+                    'count': 1,
+                    'node_processes': ['OOZIE_SERVER', 'YARN_JOBHISTORY',
+                                       'HDFS_SECONDARYNAMENODE',
+                                       'HIVE_METASTORE', 'HIVE_SERVER2']
+                },
+                'worker1': {
+                    'count': 1,
+                    'node_processes': ['YARN_NODEMANAGER', 'HDFS_DATANODE']
+                }
+            },
+            'cluster_configs': {
+                'HDFS': {
+                    'dfs_replication': 1
+                }
+            }
+        }),
+        ('5', {
+            'NODES': {
+                'master1': {
+                    'count': 1,
+                    'node_processes': ['CLOUDERA_MANAGER']
+                },
+                'master2': {
+                    'count': 1,
+                    'node_processes': ['HDFS_NAMENODE',
+                                       'YARN_RESOURCEMANAGER']
+                },
+                'master3': {
+                    'count': 1,
+                    'node_processes': ['OOZIE_SERVER', 'YARN_JOBHISTORY',
+                                       'HDFS_SECONDARYNAMENODE',
+                                       'HIVE_METASTORE', 'HIVE_SERVER2']
+                },
+                'worker1': {
+                    'count': 1,
+                    'node_processes': ['YARN_NODEMANAGER', 'HDFS_DATANODE']
+                }
+            },
+            'cluster_configs': {
+                'HDFS': {
+                    'dfs_replication': 1
+                }
+            }
+        })
+    ]),
+    'mapr': OrderedDict([
+        ('4.0.1.mrv2', {
+            'NODES': {
+                'master1': {
+                    'count': 1,
+                    'node_processes': ['CLDB', 'FileServer', 'ZooKeeper',
+                                       'NodeManager', 'ResourceManager',
+                                       'HistoryServer', 'Oozie']
+                },
+                'worker1': {
+                    'count': 1,
+                    'node_processes': ['FileServer', 'NodeManager', 'Pig']
+                }
+            },
+            'cluster_configs': {
+                'Hive': {
+                    'Hive Version': '0.13',
+                }
+            }
+        })
+    ]),
+}
+
 
 class BaseDataProcessingTest(tempest.test.BaseTestCase):
 
@@ -28,6 +230,7 @@ class BaseDataProcessingTest(tempest.test.BaseTestCase):
         super(BaseDataProcessingTest, cls).skip_checks()
         if not CONF.service_available.sahara:
             raise cls.skipException('Sahara support is required')
+        cls.default_plugin = cls._get_default_plugin()
 
     @classmethod
     def setup_credentials(cls):
@@ -43,6 +246,10 @@ class BaseDataProcessingTest(tempest.test.BaseTestCase):
     def resource_setup(cls):
         super(BaseDataProcessingTest, cls).resource_setup()
 
+        cls.default_version = cls._get_default_version()
+        if cls.default_plugin is not None and cls.default_version is None:
+            raise exceptions.InvalidConfiguration(
+                message="No known Sahara plugin version was found")
         cls.flavor_ref = CONF.compute.flavor_ref
 
         # add lists for watched resources
@@ -173,3 +380,100 @@ class BaseDataProcessingTest(tempest.test.BaseTestCase):
         cls._jobs.append(resp_body['id'])
 
         return resp_body
+
+    @classmethod
+    def _get_default_plugin(cls):
+        """Returns the default plugin used for testing."""
+        if len(CONF.data_processing_feature_enabled.plugins) == 0:
+            return None
+
+        for plugin in CONF.data_processing_feature_enabled.plugins:
+            if plugin in DEFAULT_TEMPLATES.keys():
+                break
+        else:
+            plugin = ''
+        return plugin
+
+    @classmethod
+    def _get_default_version(cls):
+        """Returns the default plugin version used for testing.
+        This is gathered separately from the plugin to allow
+        the usage of plugin name in skip_checks. This method is
+        rather invoked into resource_setup, which allows API calls
+        and exceptions.
+        """
+        if not cls.default_plugin:
+            return None
+        plugin = cls.client.get_plugin(cls.default_plugin)
+
+        for version in DEFAULT_TEMPLATES[cls.default_plugin].keys():
+            if version in plugin['versions']:
+                break
+        else:
+            version = None
+
+        return version
+
+    @classmethod
+    def get_node_group_template(cls, nodegroup='worker1'):
+        """Returns a node group template for the default plugin."""
+        try:
+            plugin_data = (
+                DEFAULT_TEMPLATES[cls.default_plugin][cls.default_version]
+            )
+            nodegroup_data = plugin_data['NODES'][nodegroup]
+            node_group_template = {
+                'description': 'Test node group template',
+                'plugin_name': cls.default_plugin,
+                'hadoop_version': cls.default_version,
+                'node_processes': nodegroup_data['node_processes'],
+                'flavor_id': cls.flavor_ref,
+                'node_configs': nodegroup_data.get('node_configs', {}),
+            }
+            return node_group_template
+        except (IndexError, KeyError):
+            return None
+
+    @classmethod
+    def get_cluster_template(cls, node_group_template_ids=None):
+        """Returns a cluster template for the default plugin.
+        node_group_template_defined contains the type and ID of pre-defined
+        node group templates that have to be used in the cluster template
+        (instead of dynamically defining them with 'node_processes').
+        """
+        if node_group_template_ids is None:
+            node_group_template_ids = {}
+        try:
+            plugin_data = (
+                DEFAULT_TEMPLATES[cls.default_plugin][cls.default_version]
+            )
+
+            all_node_groups = []
+            for ng_name, ng_data in six.iteritems(plugin_data['NODES']):
+                node_group = {
+                    'name': '%s-node' % (ng_name),
+                    'flavor_id': cls.flavor_ref,
+                    'count': ng_data['count']
+                }
+                if ng_name in node_group_template_ids.keys():
+                    # node group already defined, use it
+                    node_group['node_group_template_id'] = (
+                        node_group_template_ids[ng_name]
+                    )
+                else:
+                    # node_processes list defined on-the-fly
+                    node_group['node_processes'] = ng_data['node_processes']
+                if 'node_configs' in ng_data:
+                    node_group['node_configs'] = ng_data['node_configs']
+                all_node_groups.append(node_group)
+
+            cluster_template = {
+                'description': 'Test cluster template',
+                'plugin_name': cls.default_plugin,
+                'hadoop_version': cls.default_version,
+                'cluster_configs': plugin_data.get('cluster_configs', {}),
+                'node_groups': all_node_groups,
+            }
+            return cluster_template
+        except (IndexError, KeyError):
+            return None
diff --git a/tempest/api/data_processing/test_cluster_templates.py b/tempest/api/data_processing/test_cluster_templates.py
index 8a63c3f68..cebf493fb 100644
--- a/tempest/api/data_processing/test_cluster_templates.py
+++ b/tempest/api/data_processing/test_cluster_templates.py
@@ -15,6 +15,7 @@
 from tempest_lib.common.utils import data_utils
 
 from tempest.api.data_processing import base as dp_base
+from tempest import exceptions
 from tempest import test
 
 
@@ -23,55 +24,30 @@ class ClusterTemplateTest(dp_base.BaseDataProcessingTest):
     sahara/restapi/rest_api_v1.0.html#cluster-templates
     """
     @classmethod
+    def skip_checks(cls):
+        super(ClusterTemplateTest, cls).skip_checks()
+        if cls.default_plugin is None:
+            raise cls.skipException("No Sahara plugins configured")
+
+    @classmethod
     def resource_setup(cls):
         super(ClusterTemplateTest, cls).resource_setup()
-        # create node group template
-        node_group_template = {
-            'name': data_utils.rand_name('sahara-ng-template'),
-            'description': 'Test node group template',
-            'plugin_name': 'vanilla',
-            'hadoop_version': '1.2.1',
-            'node_processes': ['datanode'],
-            'flavor_id': cls.flavor_ref,
-            'node_configs': {
-                'HDFS': {
-                    'Data Node Heap Size': 1024
-                }
-            }
-        }
-        resp_body = cls.create_node_group_template(**node_group_template)
+
+        # pre-define a node group templates
+        node_group_template_w = cls.get_node_group_template('worker1')
+        if node_group_template_w is None:
+            raise exceptions.InvalidConfiguration(
+                message="No known Sahara plugin was found")
+
+        node_group_template_w['name'] = data_utils.rand_name(
+            'sahara-ng-template')
+        resp_body = cls.create_node_group_template(**node_group_template_w)
         node_group_template_id = resp_body['id']
+        configured_node_group_templates = {'worker1': node_group_template_id}
+
+        cls.full_cluster_template = cls.get_cluster_template(
+            configured_node_group_templates)
 
-        cls.full_cluster_template = {
-            'description': 'Test cluster template',
-            'plugin_name': 'vanilla',
-            'hadoop_version': '1.2.1',
-            'cluster_configs': {
-                'HDFS': {
-                    'dfs.replication': 2
-                },
-                'MapReduce': {
-                    'mapred.map.tasks.speculative.execution': False,
-                    'mapred.child.java.opts': '-Xmx500m'
-                },
-                'general': {
-                    'Enable Swift': False
-                }
-            },
-            'node_groups': [
-                {
-                    'name': 'master-node',
-                    'flavor_id': cls.flavor_ref,
-                    'node_processes': ['namenode'],
-                    'count': 1
-                },
-                {
-                    'name': 'worker-node',
-                    'node_group_template_id': node_group_template_id,
-                    'count': 3
-                }
-            ]
-        }
         # create cls.cluster_template variable to use for comparison to cluster
         # template response body. The 'node_groups' field in the response body
         # has some extra info that post body does not have. The 'node_groups'
diff --git a/tempest/api/data_processing/test_node_group_templates.py b/tempest/api/data_processing/test_node_group_templates.py
index d7381f4d0..4068027e9 100644
--- a/tempest/api/data_processing/test_node_group_templates.py
+++ b/tempest/api/data_processing/test_node_group_templates.py
@@ -19,27 +19,16 @@ from tempest import test
 
 
 class NodeGroupTemplateTest(dp_base.BaseDataProcessingTest):
+
+    @classmethod
+    def skip_checks(cls):
+        super(NodeGroupTemplateTest, cls).skip_checks()
+        if cls.default_plugin is None:
+            raise cls.skipException("No Sahara plugins configured")
+
     @classmethod
     def resource_setup(cls):
         super(NodeGroupTemplateTest, cls).resource_setup()
-        cls.node_group_template = {
-            'description': 'Test node group template',
-            'plugin_name': 'vanilla',
-            'hadoop_version': '1.2.1',
-            'node_processes': [
-                'datanode',
-                'tasktracker'
-            ],
-            'flavor_id': cls.flavor_ref,
-            'node_configs': {
-                'HDFS': {
-                    'Data Node Heap Size': 1024
-                },
-                'MapReduce': {
-                    'Task Tracker Heap Size': 1024
-                }
-            }
-        }
 
     def _create_node_group_template(self, template_name=None):
         """Creates Node Group Template with optional name specified.
@@ -47,6 +36,10 @@ class NodeGroupTemplateTest(dp_base.BaseDataProcessingTest):
         It creates template, ensures template name and response body.
         Returns id and name of created template.
         """
+        self.node_group_template = self.get_node_group_template()
+        self.assertIsNotNone(self.node_group_template,
+                             "No known Sahara plugin was found")
+
         if not template_name:
             # generate random name if it's not specified
             template_name = data_utils.rand_name('sahara-ng-template')