Merge "Add multiple negative test generator support"
diff --git a/etc/tempest.conf.sample b/etc/tempest.conf.sample
index b0c7826..eb2340a 100644
--- a/etc/tempest.conf.sample
+++ b/etc/tempest.conf.sample
@@ -562,6 +562,16 @@
#ssh_user_regex=[["^.*[Cc]irros.*$", "root"]]
+[negative]
+
+#
+# Options defined in tempest.config
+#
+
+# Test generator class for all negative tests (string value)
+#test_generator=tempest.common.generator.negative_generator.NegativeTestGenerator
+
+
[network]
#
diff --git a/tempest/common/generate_json.py b/tempest/common/generate_json.py
deleted file mode 100644
index c8e86dc..0000000
--- a/tempest/common/generate_json.py
+++ /dev/null
@@ -1,265 +0,0 @@
-# Copyright 2014 Red Hat, Inc. & Deutsche Telekom AG
-# All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License"); you may
-# not use this file except in compliance with the License. You may obtain
-# a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
-# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
-# License for the specific language governing permissions and limitations
-# under the License.
-
-import copy
-import jsonschema
-
-from tempest.openstack.common import log as logging
-
-LOG = logging.getLogger(__name__)
-
-
-def generate_valid(schema):
- """
- Create a valid dictionary based on the types in a json schema.
- """
- LOG.debug("generate_valid: %s" % schema)
- schema_type = schema["type"]
- if isinstance(schema_type, list):
- # Just choose the first one since all are valid.
- schema_type = schema_type[0]
- return type_map_valid[schema_type](schema)
-
-
-def generate_valid_string(schema):
- size = schema.get("minLength", 0)
- # TODO(dkr mko): handle format and pattern
- return "x" * size
-
-
-def generate_valid_integer(schema):
- # TODO(dkr mko): handle multipleOf
- if "minimum" in schema:
- minimum = schema["minimum"]
- if "exclusiveMinimum" not in schema:
- return minimum
- else:
- return minimum + 1
- if "maximum" in schema:
- maximum = schema["maximum"]
- if "exclusiveMaximum" not in schema:
- return maximum
- else:
- return maximum - 1
- return 0
-
-
-def generate_valid_object(schema):
- obj = {}
- for k, v in schema["properties"].iteritems():
- obj[k] = generate_valid(v)
- return obj
-
-
-def generate_invalid(schema):
- """
- Generate an invalid json dictionary based on a schema.
- Only one value is mis-generated for each dictionary created.
-
- Any generator must return a list of tuples or a single tuple.
- The values of this tuple are:
- result[0]: Name of the test
- result[1]: json schema for the test
- result[2]: expected result of the test (can be None)
- """
- LOG.debug("generate_invalid: %s" % schema)
- schema_type = schema["type"]
- if isinstance(schema_type, list):
- if "integer" in schema_type:
- schema_type = "integer"
- else:
- raise Exception("non-integer list types not supported")
- result = []
- for generator in type_map_invalid[schema_type]:
- ret = generator(schema)
- if ret is not None:
- if isinstance(ret, list):
- result.extend(ret)
- elif isinstance(ret, tuple):
- result.append(ret)
- else:
- raise Exception("generator (%s) returns invalid result"
- % generator)
- LOG.debug("result: %s" % result)
- return result
-
-
-def _check_for_expected_result(name, schema):
- expected_result = None
- if "results" in schema:
- if name in schema["results"]:
- expected_result = schema["results"][name]
- return expected_result
-
-
-def generator(fn):
- """
- Decorator for simple generators that simply return one value
- """
- def wrapped(schema):
- result = fn(schema)
- if result is not None:
- expected_result = _check_for_expected_result(fn.__name__, schema)
- return (fn.__name__, result, expected_result)
- return
- return wrapped
-
-
-@generator
-def gen_int(_):
- return 4
-
-
-@generator
-def gen_string(_):
- return "XXXXXX"
-
-
-def gen_none(schema):
- # Note(mkoderer): it's not using the decorator otherwise it'd be filtered
- expected_result = _check_for_expected_result('gen_none', schema)
- return ('gen_none', None, expected_result)
-
-
-@generator
-def gen_str_min_length(schema):
- min_length = schema.get("minLength", 0)
- if min_length > 0:
- return "x" * (min_length - 1)
-
-
-@generator
-def gen_str_max_length(schema):
- max_length = schema.get("maxLength", -1)
- if max_length > -1:
- return "x" * (max_length + 1)
-
-
-@generator
-def gen_int_min(schema):
- if "minimum" in schema:
- minimum = schema["minimum"]
- if "exclusiveMinimum" not in schema:
- minimum -= 1
- return minimum
-
-
-@generator
-def gen_int_max(schema):
- if "maximum" in schema:
- maximum = schema["maximum"]
- if "exclusiveMaximum" not in schema:
- maximum += 1
- return maximum
-
-
-def gen_obj_remove_attr(schema):
- invalids = []
- valid = generate_valid(schema)
- required = schema.get("required", [])
- for r in required:
- new_valid = copy.deepcopy(valid)
- del new_valid[r]
- invalids.append(("gen_obj_remove_attr", new_valid, None))
- return invalids
-
-
-@generator
-def gen_obj_add_attr(schema):
- valid = generate_valid(schema)
- if not schema.get("additionalProperties", True):
- new_valid = copy.deepcopy(valid)
- new_valid["$$$$$$$$$$"] = "xxx"
- return new_valid
-
-
-def gen_inv_prop_obj(schema):
- LOG.debug("generate_invalid_object: %s" % schema)
- valid = generate_valid(schema)
- invalids = []
- properties = schema["properties"]
-
- for k, v in properties.iteritems():
- for invalid in generate_invalid(v):
- LOG.debug(v)
- new_valid = copy.deepcopy(valid)
- new_valid[k] = invalid[1]
- name = "prop_%s_%s" % (k, invalid[0])
- invalids.append((name, new_valid, invalid[2]))
-
- LOG.debug("generate_invalid_object return: %s" % invalids)
- return invalids
-
-
-type_map_valid = {
- "string": generate_valid_string,
- "integer": generate_valid_integer,
- "object": generate_valid_object
-}
-
-type_map_invalid = {
- "string": [
- gen_int,
- gen_none,
- gen_str_min_length,
- gen_str_max_length],
- "integer": [
- gen_string,
- gen_none,
- gen_int_min,
- gen_int_max],
- "object": [
- gen_obj_remove_attr,
- gen_obj_add_attr,
- gen_inv_prop_obj]
-}
-
-schema = {
- "type": "object",
- "properties": {
- "name": {"type": "string"},
- "http-method": {
- "enum": ["GET", "PUT", "HEAD",
- "POST", "PATCH", "DELETE", 'COPY']
- },
- "url": {"type": "string"},
- "json-schema": jsonschema._utils.load_schema("draft4"),
- "resources": {
- "type": "array",
- "items": {
- "oneOf": [
- {"type": "string"},
- {
- "type": "object",
- "properties": {
- "name": {"type": "string"},
- "expected_result": {"type": "integer"}
- }
- }
- ]
- }
- },
- "results": {
- "type": "object",
- "properties": {}
- }
- },
- "required": ["name", "http-method", "url"],
- "additionalProperties": False,
-}
-
-
-def validate_negative_test_schema(nts):
- jsonschema.validate(nts, schema)
diff --git a/tempest/common/generator/__init__.py b/tempest/common/generator/__init__.py
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/tempest/common/generator/__init__.py
diff --git a/tempest/common/generator/base_generator.py b/tempest/common/generator/base_generator.py
new file mode 100644
index 0000000..35f8158
--- /dev/null
+++ b/tempest/common/generator/base_generator.py
@@ -0,0 +1,141 @@
+# Copyright 2014 Deutsche Telekom AG
+# All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+import jsonschema
+
+from tempest.openstack.common import log as logging
+
+LOG = logging.getLogger(__name__)
+
+
+def _check_for_expected_result(name, schema):
+ expected_result = None
+ if "results" in schema:
+ if name in schema["results"]:
+ expected_result = schema["results"][name]
+ return expected_result
+
+
+def generator_type(*args):
+ def wrapper(func):
+ func.types = args
+ return func
+ return wrapper
+
+
+def simple_generator(fn):
+ """
+ Decorator for simple generators that return one value
+ """
+ def wrapped(self, schema):
+ result = fn(self, schema)
+ if result is not None:
+ expected_result = _check_for_expected_result(fn.__name__, schema)
+ return (fn.__name__, result, expected_result)
+ return
+ return wrapped
+
+
+class BasicGeneratorSet(object):
+ _instance = None
+
+ schema = {
+ "type": "object",
+ "properties": {
+ "name": {"type": "string"},
+ "http-method": {
+ "enum": ["GET", "PUT", "HEAD",
+ "POST", "PATCH", "DELETE", 'COPY']
+ },
+ "url": {"type": "string"},
+ "json-schema": jsonschema._utils.load_schema("draft4"),
+ "resources": {
+ "type": "array",
+ "items": {
+ "oneOf": [
+ {"type": "string"},
+ {
+ "type": "object",
+ "properties": {
+ "name": {"type": "string"},
+ "expected_result": {"type": "integer"}
+ }
+ }
+ ]
+ }
+ },
+ "results": {
+ "type": "object",
+ "properties": {}
+ }
+ },
+ "required": ["name", "http-method", "url"],
+ "additionalProperties": False,
+ }
+
+ def __new__(cls, *args, **kwargs):
+ if not cls._instance:
+ cls._instance = super(BasicGeneratorSet, cls).__new__(cls, *args,
+ **kwargs)
+ return cls._instance
+
+ def __init__(self):
+ self.types_dict = {}
+ for m in dir(self):
+ if callable(getattr(self, m)) and not'__' in m:
+ method = getattr(self, m)
+ if hasattr(method, "types"):
+ for type in method.types:
+ if type not in self.types_dict:
+ self.types_dict[type] = []
+ self.types_dict[type].append(method)
+
+ def validate_schema(self, schema):
+ jsonschema.validate(schema, self.schema)
+
+ def generate(self, schema):
+ """
+ Generate an json dictionary based on a schema.
+ Only one value is mis-generated for each dictionary created.
+
+ Any generator must return a list of tuples or a single tuple.
+ The values of this tuple are:
+ result[0]: Name of the test
+ result[1]: json schema for the test
+ result[2]: expected result of the test (can be None)
+ """
+ LOG.debug("generate_invalid: %s" % schema)
+ schema_type = schema["type"]
+ if isinstance(schema_type, list):
+ if "integer" in schema_type:
+ schema_type = "integer"
+ else:
+ raise Exception("non-integer list types not supported")
+ result = []
+ if schema_type not in self.types_dict:
+ raise Exception("generator (%s) doesn't support type: %s"
+ % (self.__class__.__name__, schema_type))
+ for generator in self.types_dict[schema_type]:
+ ret = generator(schema)
+ if ret is not None:
+ if isinstance(ret, list):
+ result.extend(ret)
+ elif isinstance(ret, tuple):
+ result.append(ret)
+ else:
+ raise Exception("generator (%s) returns invalid result: %s"
+ % (generator, ret))
+ LOG.debug("result: %s" % result)
+ return result
diff --git a/tempest/common/generator/negative_generator.py b/tempest/common/generator/negative_generator.py
new file mode 100644
index 0000000..4f3d2cd
--- /dev/null
+++ b/tempest/common/generator/negative_generator.py
@@ -0,0 +1,111 @@
+# Copyright 2014 Deutsche Telekom AG
+# All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+import copy
+
+import tempest.common.generator.base_generator as base
+import tempest.common.generator.valid_generator as valid
+from tempest.openstack.common import log as logging
+
+LOG = logging.getLogger(__name__)
+
+
+class NegativeTestGenerator(base.BasicGeneratorSet):
+ @base.generator_type("string")
+ @base.simple_generator
+ def gen_int(self, _):
+ return 4
+
+ @base.generator_type("integer")
+ @base.simple_generator
+ def gen_string(self, _):
+ return "XXXXXX"
+
+ @base.generator_type("integer", "string")
+ def gen_none(self, schema):
+ # Note(mkoderer): it's not using the decorator otherwise it'd be
+ # filtered
+ expected_result = base._check_for_expected_result('gen_none', schema)
+ return ('gen_none', None, expected_result)
+
+ @base.generator_type("string")
+ @base.simple_generator
+ def gen_str_min_length(self, schema):
+ min_length = schema.get("minLength", 0)
+ if min_length > 0:
+ return "x" * (min_length - 1)
+
+ @base.generator_type("string")
+ @base.simple_generator
+ def gen_str_max_length(self, schema):
+ max_length = schema.get("maxLength", -1)
+ if max_length > -1:
+ return "x" * (max_length + 1)
+
+ @base.generator_type("integer")
+ @base.simple_generator
+ def gen_int_min(self, schema):
+ if "minimum" in schema:
+ minimum = schema["minimum"]
+ if "exclusiveMinimum" not in schema:
+ minimum -= 1
+ return minimum
+
+ @base.generator_type("integer")
+ @base.simple_generator
+ def gen_int_max(self, schema):
+ if "maximum" in schema:
+ maximum = schema["maximum"]
+ if "exclusiveMaximum" not in schema:
+ maximum += 1
+ return maximum
+
+ @base.generator_type("object")
+ def gen_obj_remove_attr(self, schema):
+ invalids = []
+ valid_schema = valid.ValidTestGenerator().generate_valid(schema)
+ required = schema.get("required", [])
+ for r in required:
+ new_valid = copy.deepcopy(valid_schema)
+ del new_valid[r]
+ invalids.append(("gen_obj_remove_attr", new_valid, None))
+ return invalids
+
+ @base.generator_type("object")
+ @base.simple_generator
+ def gen_obj_add_attr(self, schema):
+ valid_schema = valid.ValidTestGenerator().generate_valid(schema)
+ if not schema.get("additionalProperties", True):
+ new_valid = copy.deepcopy(valid_schema)
+ new_valid["$$$$$$$$$$"] = "xxx"
+ return new_valid
+
+ @base.generator_type("object")
+ def gen_inv_prop_obj(self, schema):
+ LOG.debug("generate_invalid_object: %s" % schema)
+ valid_schema = valid.ValidTestGenerator().generate_valid(schema)
+ invalids = []
+ properties = schema["properties"]
+
+ for k, v in properties.iteritems():
+ for invalid in self.generate(v):
+ LOG.debug(v)
+ new_valid = copy.deepcopy(valid_schema)
+ new_valid[k] = invalid[1]
+ name = "prop_%s_%s" % (k, invalid[0])
+ invalids.append((name, new_valid, invalid[2]))
+
+ LOG.debug("generate_invalid_object return: %s" % invalids)
+ return invalids
diff --git a/tempest/common/generator/valid_generator.py b/tempest/common/generator/valid_generator.py
new file mode 100644
index 0000000..a99bbc0
--- /dev/null
+++ b/tempest/common/generator/valid_generator.py
@@ -0,0 +1,58 @@
+# Copyright 2014 Deutsche Telekom AG
+# All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+import tempest.common.generator.base_generator as base
+from tempest.openstack.common import log as logging
+
+
+LOG = logging.getLogger(__name__)
+
+
+class ValidTestGenerator(base.BasicGeneratorSet):
+ @base.generator_type("string")
+ @base.simple_generator
+ def generate_valid_string(self, schema):
+ size = schema.get("minLength", 0)
+ # TODO(dkr mko): handle format and pattern
+ return "x" * size
+
+ @base.generator_type("integer")
+ @base.simple_generator
+ def generate_valid_integer(self, schema):
+ # TODO(dkr mko): handle multipleOf
+ if "minimum" in schema:
+ minimum = schema["minimum"]
+ if "exclusiveMinimum" not in schema:
+ return minimum
+ else:
+ return minimum + 1
+ if "maximum" in schema:
+ maximum = schema["maximum"]
+ if "exclusiveMaximum" not in schema:
+ return maximum
+ else:
+ return maximum - 1
+ return 0
+
+ @base.generator_type("object")
+ @base.simple_generator
+ def generate_valid_object(self, schema):
+ obj = {}
+ for k, v in schema["properties"].iteritems():
+ obj[k] = self.generate_valid(v)
+ return obj
+
+ def generate_valid(self, schema):
+ return self.generate(schema)[0][1]
diff --git a/tempest/config.py b/tempest/config.py
index 0c33233..db81f6e 100644
--- a/tempest/config.py
+++ b/tempest/config.py
@@ -820,6 +820,15 @@
help="Number of seconds to wait on a CLI timeout"),
]
+negative_group = cfg.OptGroup(name='negative', title="Negative Test Options")
+
+NegativeGroup = [
+ cfg.StrOpt('test_generator',
+ default='tempest.common.' +
+ 'generator.negative_generator.NegativeTestGenerator',
+ help="Test generator class for all negative tests"),
+]
+
def register_opts():
register_opt_group(cfg.CONF, compute_group, ComputeGroup)
@@ -855,6 +864,7 @@
register_opt_group(cfg.CONF, baremetal_group, BaremetalGroup)
register_opt_group(cfg.CONF, input_scenario_group, InputScenarioGroup)
register_opt_group(cfg.CONF, cli_group, CLIGroup)
+ register_opt_group(cfg.CONF, negative_group, NegativeGroup)
# this should never be called outside of this class
@@ -895,6 +905,7 @@
self.baremetal = cfg.CONF.baremetal
self.input_scenario = cfg.CONF['input-scenario']
self.cli = cfg.CONF.cli
+ self.negative = cfg.CONF.negative
if not self.compute_admin.username:
self.compute_admin.username = self.identity.admin_username
self.compute_admin.password = self.identity.admin_password
diff --git a/tempest/test.py b/tempest/test.py
index c6e3d6e..2125047 100644
--- a/tempest/test.py
+++ b/tempest/test.py
@@ -27,10 +27,11 @@
import testtools
from tempest import clients
-from tempest.common import generate_json
+import tempest.common.generator.valid_generator as valid
from tempest.common import isolated_creds
from tempest import config
from tempest import exceptions
+from tempest.openstack.common import importutils
from tempest.openstack.common import log as logging
LOG = logging.getLogger(__name__)
@@ -400,7 +401,8 @@
"""
description = NegativeAutoTest.load_schema(description_file)
LOG.debug(description)
- generate_json.validate_negative_test_schema(description)
+ generator = importutils.import_class(CONF.negative.test_generator)()
+ generator.validate_schema(description)
schema = description.get("json-schema", None)
resources = description.get("resources", [])
scenario_list = []
@@ -416,7 +418,7 @@
"expected_result": expected_result
}))
if schema is not None:
- for invalid in generate_json.generate_invalid(schema):
+ for invalid in generator.generate(schema):
scenario_list.append((invalid[0],
{"schema": invalid[1],
"expected_result": invalid[2]}))
@@ -459,11 +461,12 @@
# Note(mkoderer): The resources list already contains an invalid
# entry (see get_resource).
# We just send a valid json-schema with it
- valid = None
+ valid_schema = None
schema = description.get("json-schema", None)
if schema:
- valid = generate_json.generate_valid(schema)
- new_url, body = self._http_arguments(valid, url, method)
+ valid_schema = \
+ valid.ValidTestGenerator().generate_valid(schema)
+ new_url, body = self._http_arguments(valid_schema, url, method)
elif hasattr(self, "schema"):
new_url, body = self._http_arguments(self.schema, url, method)
diff --git a/tempest/tests/fake_config.py b/tempest/tests/fake_config.py
index 41b0558..e941606 100644
--- a/tempest/tests/fake_config.py
+++ b/tempest/tests/fake_config.py
@@ -43,6 +43,10 @@
swift = True
horizon = True
+ class fake_negative(object):
+ test_generator = 'tempest.common.' \
+ 'generator.negative_generator.NegativeTestGenerator'
+
compute_feature_enabled = fake_compute_feature_enabled()
volume_feature_enabled = fake_default_feature_enabled()
network_feature_enabled = fake_default_feature_enabled()
@@ -52,3 +56,5 @@
compute = fake_compute()
identity = fake_identity()
+
+ negative = fake_negative()
diff --git a/tempest/tests/negative/test_generate_json.py b/tempest/tests/negative/test_generate_json.py
index a0aa088..e09fcdf 100644
--- a/tempest/tests/negative/test_generate_json.py
+++ b/tempest/tests/negative/test_generate_json.py
@@ -13,11 +13,11 @@
# License for the specific language governing permissions and limitations
# under the License.
-from tempest.common import generate_json as gen
+from tempest.common.generator import negative_generator
import tempest.test
-class TestGenerateJson(tempest.test.BaseTestCase):
+class TestNegativeGenerator(tempest.test.BaseTestCase):
fake_input_str = {"type": "string",
"minLength": 2,
@@ -35,19 +35,23 @@
}
}
+ def setUp(self):
+ super(TestNegativeGenerator, self).setUp()
+ self.negative = negative_generator.NegativeTestGenerator()
+
def _validate_result(self, data):
self.assertTrue(isinstance(data, list))
for t in data:
self.assertTrue(isinstance(t, tuple))
def test_generate_invalid_string(self):
- result = gen.generate_invalid(self.fake_input_str)
+ result = self.negative.generate(self.fake_input_str)
self._validate_result(result)
def test_generate_invalid_integer(self):
- result = gen.generate_invalid(self.fake_input_int)
+ result = self.negative.generate(self.fake_input_int)
self._validate_result(result)
def test_generate_invalid_obj(self):
- result = gen.generate_invalid(self.fake_input_obj)
+ result = self.negative.generate(self.fake_input_obj)
self._validate_result(result)
diff --git a/tempest/tests/negative/test_negative_auto_test.py b/tempest/tests/negative/test_negative_auto_test.py
index 4c59383..27ddc95 100644
--- a/tempest/tests/negative/test_negative_auto_test.py
+++ b/tempest/tests/negative/test_negative_auto_test.py
@@ -16,9 +16,11 @@
import mock
import tempest.test as test
+from tempest.tests import base
+from tempest.tests import fake_config
-class TestNegativeAutoTest(test.BaseTestCase):
+class TestNegativeAutoTest(base.TestCase):
# Fake entries
_interface = 'json'
_service = 'compute'
@@ -34,6 +36,10 @@
"resources": ["flavor", "volume", "image"]
}
+ def setUp(self):
+ super(TestNegativeAutoTest, self).setUp()
+ self.stubs.Set(test, 'CONF', fake_config.FakeConfig)
+
def _check_prop_entries(self, result, entry):
entries = [a for a in result if entry in a[0]]
self.assertIsNotNone(entries)