Decouple Sahara templates test from vanilla plugin
The change introduces a list of template data for node templates and
clusters for each plugin. The first known plugin and its first known
version are used.
At least one default template is added for each plugin, so that all
the real deployment can be covered.
The tests are skipped if the plugins key in tempest configuration is empty;
they fail if no known plugin or no known version for known plugins are found
(which probably means that the test configuration should be updated).
The list must be maintained; on the other way, after some discussions,
the alternative solution - expose all the node group and cluster
templates parameters in tempest configuration - would have made the
testing more random and too dependent on the cloud operator in charge
of running tempest.
The list of plugins is not probably going to change massively for each
release, so it seems feasible.
Change-Id: I397616f7923e696082fe67da3ff9841f89e9a505
diff --git a/tempest/api/data_processing/base.py b/tempest/api/data_processing/base.py
index 5992921..7f3474b 100644
--- a/tempest/api/data_processing/base.py
+++ b/tempest/api/data_processing/base.py
@@ -12,14 +12,216 @@
# License for the specific language governing permissions and limitations
# under the License.
+from collections import OrderedDict
+
+import six
from tempest_lib import exceptions as lib_exc
from tempest import config
+from tempest import exceptions
import tempest.test
CONF = config.CONF
+"""Default templates.
+There should always be at least a master1 and a worker1 node
+group template."""
+DEFAULT_TEMPLATES = {
+ 'vanilla': OrderedDict([
+ ('2.6.0', {
+ 'NODES': {
+ 'master1': {
+ 'count': 1,
+ 'node_processes': ['namenode', 'resourcemanager',
+ 'hiveserver']
+ },
+ 'master2': {
+ 'count': 1,
+ 'node_processes': ['oozie', 'historyserver',
+ 'secondarynamenode']
+ },
+ 'worker1': {
+ 'count': 1,
+ 'node_processes': ['datanode', 'nodemanager'],
+ 'node_configs': {
+ 'MapReduce': {
+ 'yarn.app.mapreduce.am.resource.mb': 256,
+ 'yarn.app.mapreduce.am.command-opts': '-Xmx256m'
+ },
+ 'YARN': {
+ 'yarn.scheduler.minimum-allocation-mb': 256,
+ 'yarn.scheduler.maximum-allocation-mb': 1024,
+ 'yarn.nodemanager.vmem-check-enabled': False
+ }
+ }
+ }
+ },
+ 'cluster_configs': {
+ 'HDFS': {
+ 'dfs.replication': 1
+ }
+ }
+ }),
+ ('1.2.1', {
+ 'NODES': {
+ 'master1': {
+ 'count': 1,
+ 'node_processes': ['namenode', 'jobtracker']
+ },
+ 'worker1': {
+ 'count': 1,
+ 'node_processes': ['datanode', 'tasktracker'],
+ 'node_configs': {
+ 'HDFS': {
+ 'Data Node Heap Size': 1024
+ },
+ 'MapReduce': {
+ 'Task Tracker Heap Size': 1024
+ }
+ }
+ }
+ },
+ 'cluster_configs': {
+ 'HDFS': {
+ 'dfs.replication': 1
+ },
+ 'MapReduce': {
+ 'mapred.map.tasks.speculative.execution': False,
+ 'mapred.child.java.opts': '-Xmx500m'
+ },
+ 'general': {
+ 'Enable Swift': False
+ }
+ }
+ })
+ ]),
+ 'hdp': OrderedDict([
+ ('2.0.6', {
+ 'NODES': {
+ 'master1': {
+ 'count': 1,
+ 'node_processes': ['NAMENODE', 'SECONDARY_NAMENODE',
+ 'ZOOKEEPER_SERVER', 'AMBARI_SERVER',
+ 'HISTORYSERVER', 'RESOURCEMANAGER',
+ 'GANGLIA_SERVER', 'NAGIOS_SERVER',
+ 'OOZIE_SERVER']
+ },
+ 'worker1': {
+ 'count': 1,
+ 'node_processes': ['HDFS_CLIENT', 'DATANODE',
+ 'YARN_CLIENT', 'ZOOKEEPER_CLIENT',
+ 'MAPREDUCE2_CLIENT', 'NODEMANAGER',
+ 'PIG', 'OOZIE_CLIENT']
+ }
+ },
+ 'cluster_configs': {
+ 'HDFS': {
+ 'dfs.replication': 1
+ }
+ }
+ })
+ ]),
+ 'spark': OrderedDict([
+ ('1.0.0', {
+ 'NODES': {
+ 'master1': {
+ 'count': 1,
+ 'node_processes': ['namenode', 'master']
+ },
+ 'worker1': {
+ 'count': 1,
+ 'node_processes': ['datanode', 'slave']
+ }
+ },
+ 'cluster_configs': {
+ 'HDFS': {
+ 'dfs.replication': 1
+ }
+ }
+ })
+ ]),
+ 'cdh': OrderedDict([
+ ('5.3.0', {
+ 'NODES': {
+ 'master1': {
+ 'count': 1,
+ 'node_processes': ['CLOUDERA_MANAGER']
+ },
+ 'master2': {
+ 'count': 1,
+ 'node_processes': ['HDFS_NAMENODE',
+ 'YARN_RESOURCEMANAGER']
+ },
+ 'master3': {
+ 'count': 1,
+ 'node_processes': ['OOZIE_SERVER', 'YARN_JOBHISTORY',
+ 'HDFS_SECONDARYNAMENODE',
+ 'HIVE_METASTORE', 'HIVE_SERVER2']
+ },
+ 'worker1': {
+ 'count': 1,
+ 'node_processes': ['YARN_NODEMANAGER', 'HDFS_DATANODE']
+ }
+ },
+ 'cluster_configs': {
+ 'HDFS': {
+ 'dfs_replication': 1
+ }
+ }
+ }),
+ ('5', {
+ 'NODES': {
+ 'master1': {
+ 'count': 1,
+ 'node_processes': ['CLOUDERA_MANAGER']
+ },
+ 'master2': {
+ 'count': 1,
+ 'node_processes': ['HDFS_NAMENODE',
+ 'YARN_RESOURCEMANAGER']
+ },
+ 'master3': {
+ 'count': 1,
+ 'node_processes': ['OOZIE_SERVER', 'YARN_JOBHISTORY',
+ 'HDFS_SECONDARYNAMENODE',
+ 'HIVE_METASTORE', 'HIVE_SERVER2']
+ },
+ 'worker1': {
+ 'count': 1,
+ 'node_processes': ['YARN_NODEMANAGER', 'HDFS_DATANODE']
+ }
+ },
+ 'cluster_configs': {
+ 'HDFS': {
+ 'dfs_replication': 1
+ }
+ }
+ })
+ ]),
+ 'mapr': OrderedDict([
+ ('4.0.1.mrv2', {
+ 'NODES': {
+ 'master1': {
+ 'count': 1,
+ 'node_processes': ['CLDB', 'FileServer', 'ZooKeeper',
+ 'NodeManager', 'ResourceManager',
+ 'HistoryServer', 'Oozie']
+ },
+ 'worker1': {
+ 'count': 1,
+ 'node_processes': ['FileServer', 'NodeManager', 'Pig']
+ }
+ },
+ 'cluster_configs': {
+ 'Hive': {
+ 'Hive Version': '0.13',
+ }
+ }
+ })
+ ]),
+}
+
class BaseDataProcessingTest(tempest.test.BaseTestCase):
@@ -28,6 +230,7 @@
super(BaseDataProcessingTest, cls).skip_checks()
if not CONF.service_available.sahara:
raise cls.skipException('Sahara support is required')
+ cls.default_plugin = cls._get_default_plugin()
@classmethod
def setup_credentials(cls):
@@ -43,6 +246,10 @@
def resource_setup(cls):
super(BaseDataProcessingTest, cls).resource_setup()
+ cls.default_version = cls._get_default_version()
+ if cls.default_plugin is not None and cls.default_version is None:
+ raise exceptions.InvalidConfiguration(
+ message="No known Sahara plugin version was found")
cls.flavor_ref = CONF.compute.flavor_ref
# add lists for watched resources
@@ -173,3 +380,100 @@
cls._jobs.append(resp_body['id'])
return resp_body
+
+ @classmethod
+ def _get_default_plugin(cls):
+ """Returns the default plugin used for testing."""
+ if len(CONF.data_processing_feature_enabled.plugins) == 0:
+ return None
+
+ for plugin in CONF.data_processing_feature_enabled.plugins:
+ if plugin in DEFAULT_TEMPLATES.keys():
+ break
+ else:
+ plugin = ''
+ return plugin
+
+ @classmethod
+ def _get_default_version(cls):
+ """Returns the default plugin version used for testing.
+ This is gathered separately from the plugin to allow
+ the usage of plugin name in skip_checks. This method is
+ rather invoked into resource_setup, which allows API calls
+ and exceptions.
+ """
+ if not cls.default_plugin:
+ return None
+ plugin = cls.client.get_plugin(cls.default_plugin)
+
+ for version in DEFAULT_TEMPLATES[cls.default_plugin].keys():
+ if version in plugin['versions']:
+ break
+ else:
+ version = None
+
+ return version
+
+ @classmethod
+ def get_node_group_template(cls, nodegroup='worker1'):
+ """Returns a node group template for the default plugin."""
+ try:
+ plugin_data = (
+ DEFAULT_TEMPLATES[cls.default_plugin][cls.default_version]
+ )
+ nodegroup_data = plugin_data['NODES'][nodegroup]
+ node_group_template = {
+ 'description': 'Test node group template',
+ 'plugin_name': cls.default_plugin,
+ 'hadoop_version': cls.default_version,
+ 'node_processes': nodegroup_data['node_processes'],
+ 'flavor_id': cls.flavor_ref,
+ 'node_configs': nodegroup_data.get('node_configs', {}),
+ }
+ return node_group_template
+ except (IndexError, KeyError):
+ return None
+
+ @classmethod
+ def get_cluster_template(cls, node_group_template_ids=None):
+ """Returns a cluster template for the default plugin.
+ node_group_template_defined contains the type and ID of pre-defined
+ node group templates that have to be used in the cluster template
+ (instead of dynamically defining them with 'node_processes').
+ """
+ if node_group_template_ids is None:
+ node_group_template_ids = {}
+ try:
+ plugin_data = (
+ DEFAULT_TEMPLATES[cls.default_plugin][cls.default_version]
+ )
+
+ all_node_groups = []
+ for ng_name, ng_data in six.iteritems(plugin_data['NODES']):
+ node_group = {
+ 'name': '%s-node' % (ng_name),
+ 'flavor_id': cls.flavor_ref,
+ 'count': ng_data['count']
+ }
+ if ng_name in node_group_template_ids.keys():
+ # node group already defined, use it
+ node_group['node_group_template_id'] = (
+ node_group_template_ids[ng_name]
+ )
+ else:
+ # node_processes list defined on-the-fly
+ node_group['node_processes'] = ng_data['node_processes']
+ if 'node_configs' in ng_data:
+ node_group['node_configs'] = ng_data['node_configs']
+ all_node_groups.append(node_group)
+
+ cluster_template = {
+ 'description': 'Test cluster template',
+ 'plugin_name': cls.default_plugin,
+ 'hadoop_version': cls.default_version,
+ 'cluster_configs': plugin_data.get('cluster_configs', {}),
+ 'node_groups': all_node_groups,
+ }
+ return cluster_template
+ except (IndexError, KeyError):
+ return None
diff --git a/tempest/api/data_processing/test_cluster_templates.py b/tempest/api/data_processing/test_cluster_templates.py
index 8a63c3f..cebf493 100644
--- a/tempest/api/data_processing/test_cluster_templates.py
+++ b/tempest/api/data_processing/test_cluster_templates.py
@@ -15,6 +15,7 @@
from tempest_lib.common.utils import data_utils
from tempest.api.data_processing import base as dp_base
+from tempest import exceptions
from tempest import test
@@ -23,55 +24,30 @@
sahara/restapi/rest_api_v1.0.html#cluster-templates
"""
@classmethod
+ def skip_checks(cls):
+ super(ClusterTemplateTest, cls).skip_checks()
+ if cls.default_plugin is None:
+ raise cls.skipException("No Sahara plugins configured")
+
+ @classmethod
def resource_setup(cls):
super(ClusterTemplateTest, cls).resource_setup()
- # create node group template
- node_group_template = {
- 'name': data_utils.rand_name('sahara-ng-template'),
- 'description': 'Test node group template',
- 'plugin_name': 'vanilla',
- 'hadoop_version': '1.2.1',
- 'node_processes': ['datanode'],
- 'flavor_id': cls.flavor_ref,
- 'node_configs': {
- 'HDFS': {
- 'Data Node Heap Size': 1024
- }
- }
- }
- resp_body = cls.create_node_group_template(**node_group_template)
- node_group_template_id = resp_body['id']
- cls.full_cluster_template = {
- 'description': 'Test cluster template',
- 'plugin_name': 'vanilla',
- 'hadoop_version': '1.2.1',
- 'cluster_configs': {
- 'HDFS': {
- 'dfs.replication': 2
- },
- 'MapReduce': {
- 'mapred.map.tasks.speculative.execution': False,
- 'mapred.child.java.opts': '-Xmx500m'
- },
- 'general': {
- 'Enable Swift': False
- }
- },
- 'node_groups': [
- {
- 'name': 'master-node',
- 'flavor_id': cls.flavor_ref,
- 'node_processes': ['namenode'],
- 'count': 1
- },
- {
- 'name': 'worker-node',
- 'node_group_template_id': node_group_template_id,
- 'count': 3
- }
- ]
- }
+ # pre-define a node group templates
+ node_group_template_w = cls.get_node_group_template('worker1')
+ if node_group_template_w is None:
+ raise exceptions.InvalidConfiguration(
+ message="No known Sahara plugin was found")
+
+ node_group_template_w['name'] = data_utils.rand_name(
+ 'sahara-ng-template')
+ resp_body = cls.create_node_group_template(**node_group_template_w)
+ node_group_template_id = resp_body['id']
+ configured_node_group_templates = {'worker1': node_group_template_id}
+
+ cls.full_cluster_template = cls.get_cluster_template(
+ configured_node_group_templates)
+
# create cls.cluster_template variable to use for comparison to cluster
# template response body. The 'node_groups' field in the response body
# has some extra info that post body does not have. The 'node_groups'
diff --git a/tempest/api/data_processing/test_node_group_templates.py b/tempest/api/data_processing/test_node_group_templates.py
index d7381f4..4068027 100644
--- a/tempest/api/data_processing/test_node_group_templates.py
+++ b/tempest/api/data_processing/test_node_group_templates.py
@@ -19,27 +19,16 @@
class NodeGroupTemplateTest(dp_base.BaseDataProcessingTest):
+
+ @classmethod
+ def skip_checks(cls):
+ super(NodeGroupTemplateTest, cls).skip_checks()
+ if cls.default_plugin is None:
+ raise cls.skipException("No Sahara plugins configured")
+
@classmethod
def resource_setup(cls):
super(NodeGroupTemplateTest, cls).resource_setup()
- cls.node_group_template = {
- 'description': 'Test node group template',
- 'plugin_name': 'vanilla',
- 'hadoop_version': '1.2.1',
- 'node_processes': [
- 'datanode',
- 'tasktracker'
- ],
- 'flavor_id': cls.flavor_ref,
- 'node_configs': {
- 'HDFS': {
- 'Data Node Heap Size': 1024
- },
- 'MapReduce': {
- 'Task Tracker Heap Size': 1024
- }
- }
- }
def _create_node_group_template(self, template_name=None):
"""Creates Node Group Template with optional name specified.
@@ -47,6 +36,10 @@
It creates template, ensures template name and response body.
Returns id and name of created template.
"""
+ self.node_group_template = self.get_node_group_template()
+ self.assertIsNotNone(self.node_group_template,
+ "No known Sahara plugin was found")
+
if not template_name:
# generate random name if it's not specified
template_name = data_utils.rand_name('sahara-ng-template')