Add multiple negative test generator support
In order to support different generator styles and sets of tests
a new configuration parameter is introduced to define the negative
test generator class. This can be used to define generators that create
only random values (random fuzzy test) or pattern based values.
With this functionality it is also possible to reduce the amount
of negative tests that are automatically produced.
Change-Id: Icfad55d1eea92dc2a42642b37d34c253c26c0846
Partially-implements: bp fuzzy-test
diff --git a/etc/tempest.conf.sample b/etc/tempest.conf.sample
index ee2da40..e46b813 100644
--- a/etc/tempest.conf.sample
+++ b/etc/tempest.conf.sample
@@ -548,6 +548,16 @@
#ssh_user_regex=[["^.*[Cc]irros.*$", "root"]]
+[negative]
+
+#
+# Options defined in tempest.config
+#
+
+# Test generator class for all negative tests (string value)
+#test_generator=tempest.common.generator.negative_generator.NegativeTestGenerator
+
+
[network]
#
diff --git a/tempest/common/generate_json.py b/tempest/common/generate_json.py
deleted file mode 100644
index c8e86dc..0000000
--- a/tempest/common/generate_json.py
+++ /dev/null
@@ -1,265 +0,0 @@
-# Copyright 2014 Red Hat, Inc. & Deutsche Telekom AG
-# All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License"); you may
-# not use this file except in compliance with the License. You may obtain
-# a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
-# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
-# License for the specific language governing permissions and limitations
-# under the License.
-
-import copy
-import jsonschema
-
-from tempest.openstack.common import log as logging
-
-LOG = logging.getLogger(__name__)
-
-
-def generate_valid(schema):
- """
- Create a valid dictionary based on the types in a json schema.
- """
- LOG.debug("generate_valid: %s" % schema)
- schema_type = schema["type"]
- if isinstance(schema_type, list):
- # Just choose the first one since all are valid.
- schema_type = schema_type[0]
- return type_map_valid[schema_type](schema)
-
-
-def generate_valid_string(schema):
- size = schema.get("minLength", 0)
- # TODO(dkr mko): handle format and pattern
- return "x" * size
-
-
-def generate_valid_integer(schema):
- # TODO(dkr mko): handle multipleOf
- if "minimum" in schema:
- minimum = schema["minimum"]
- if "exclusiveMinimum" not in schema:
- return minimum
- else:
- return minimum + 1
- if "maximum" in schema:
- maximum = schema["maximum"]
- if "exclusiveMaximum" not in schema:
- return maximum
- else:
- return maximum - 1
- return 0
-
-
-def generate_valid_object(schema):
- obj = {}
- for k, v in schema["properties"].iteritems():
- obj[k] = generate_valid(v)
- return obj
-
-
-def generate_invalid(schema):
- """
- Generate an invalid json dictionary based on a schema.
- Only one value is mis-generated for each dictionary created.
-
- Any generator must return a list of tuples or a single tuple.
- The values of this tuple are:
- result[0]: Name of the test
- result[1]: json schema for the test
- result[2]: expected result of the test (can be None)
- """
- LOG.debug("generate_invalid: %s" % schema)
- schema_type = schema["type"]
- if isinstance(schema_type, list):
- if "integer" in schema_type:
- schema_type = "integer"
- else:
- raise Exception("non-integer list types not supported")
- result = []
- for generator in type_map_invalid[schema_type]:
- ret = generator(schema)
- if ret is not None:
- if isinstance(ret, list):
- result.extend(ret)
- elif isinstance(ret, tuple):
- result.append(ret)
- else:
- raise Exception("generator (%s) returns invalid result"
- % generator)
- LOG.debug("result: %s" % result)
- return result
-
-
-def _check_for_expected_result(name, schema):
- expected_result = None
- if "results" in schema:
- if name in schema["results"]:
- expected_result = schema["results"][name]
- return expected_result
-
-
-def generator(fn):
- """
- Decorator for simple generators that simply return one value
- """
- def wrapped(schema):
- result = fn(schema)
- if result is not None:
- expected_result = _check_for_expected_result(fn.__name__, schema)
- return (fn.__name__, result, expected_result)
- return
- return wrapped
-
-
-@generator
-def gen_int(_):
- return 4
-
-
-@generator
-def gen_string(_):
- return "XXXXXX"
-
-
-def gen_none(schema):
- # Note(mkoderer): it's not using the decorator otherwise it'd be filtered
- expected_result = _check_for_expected_result('gen_none', schema)
- return ('gen_none', None, expected_result)
-
-
-@generator
-def gen_str_min_length(schema):
- min_length = schema.get("minLength", 0)
- if min_length > 0:
- return "x" * (min_length - 1)
-
-
-@generator
-def gen_str_max_length(schema):
- max_length = schema.get("maxLength", -1)
- if max_length > -1:
- return "x" * (max_length + 1)
-
-
-@generator
-def gen_int_min(schema):
- if "minimum" in schema:
- minimum = schema["minimum"]
- if "exclusiveMinimum" not in schema:
- minimum -= 1
- return minimum
-
-
-@generator
-def gen_int_max(schema):
- if "maximum" in schema:
- maximum = schema["maximum"]
- if "exclusiveMaximum" not in schema:
- maximum += 1
- return maximum
-
-
-def gen_obj_remove_attr(schema):
- invalids = []
- valid = generate_valid(schema)
- required = schema.get("required", [])
- for r in required:
- new_valid = copy.deepcopy(valid)
- del new_valid[r]
- invalids.append(("gen_obj_remove_attr", new_valid, None))
- return invalids
-
-
-@generator
-def gen_obj_add_attr(schema):
- valid = generate_valid(schema)
- if not schema.get("additionalProperties", True):
- new_valid = copy.deepcopy(valid)
- new_valid["$$$$$$$$$$"] = "xxx"
- return new_valid
-
-
-def gen_inv_prop_obj(schema):
- LOG.debug("generate_invalid_object: %s" % schema)
- valid = generate_valid(schema)
- invalids = []
- properties = schema["properties"]
-
- for k, v in properties.iteritems():
- for invalid in generate_invalid(v):
- LOG.debug(v)
- new_valid = copy.deepcopy(valid)
- new_valid[k] = invalid[1]
- name = "prop_%s_%s" % (k, invalid[0])
- invalids.append((name, new_valid, invalid[2]))
-
- LOG.debug("generate_invalid_object return: %s" % invalids)
- return invalids
-
-
-type_map_valid = {
- "string": generate_valid_string,
- "integer": generate_valid_integer,
- "object": generate_valid_object
-}
-
-type_map_invalid = {
- "string": [
- gen_int,
- gen_none,
- gen_str_min_length,
- gen_str_max_length],
- "integer": [
- gen_string,
- gen_none,
- gen_int_min,
- gen_int_max],
- "object": [
- gen_obj_remove_attr,
- gen_obj_add_attr,
- gen_inv_prop_obj]
-}
-
-schema = {
- "type": "object",
- "properties": {
- "name": {"type": "string"},
- "http-method": {
- "enum": ["GET", "PUT", "HEAD",
- "POST", "PATCH", "DELETE", 'COPY']
- },
- "url": {"type": "string"},
- "json-schema": jsonschema._utils.load_schema("draft4"),
- "resources": {
- "type": "array",
- "items": {
- "oneOf": [
- {"type": "string"},
- {
- "type": "object",
- "properties": {
- "name": {"type": "string"},
- "expected_result": {"type": "integer"}
- }
- }
- ]
- }
- },
- "results": {
- "type": "object",
- "properties": {}
- }
- },
- "required": ["name", "http-method", "url"],
- "additionalProperties": False,
-}
-
-
-def validate_negative_test_schema(nts):
- jsonschema.validate(nts, schema)
diff --git a/tempest/common/generator/__init__.py b/tempest/common/generator/__init__.py
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/tempest/common/generator/__init__.py
diff --git a/tempest/common/generator/base_generator.py b/tempest/common/generator/base_generator.py
new file mode 100644
index 0000000..35f8158
--- /dev/null
+++ b/tempest/common/generator/base_generator.py
@@ -0,0 +1,141 @@
+# Copyright 2014 Deutsche Telekom AG
+# All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+import jsonschema
+
+from tempest.openstack.common import log as logging
+
+LOG = logging.getLogger(__name__)
+
+
+def _check_for_expected_result(name, schema):
+ expected_result = None
+ if "results" in schema:
+ if name in schema["results"]:
+ expected_result = schema["results"][name]
+ return expected_result
+
+
+def generator_type(*args):
+ def wrapper(func):
+ func.types = args
+ return func
+ return wrapper
+
+
+def simple_generator(fn):
+ """
+ Decorator for simple generators that return one value
+ """
+ def wrapped(self, schema):
+ result = fn(self, schema)
+ if result is not None:
+ expected_result = _check_for_expected_result(fn.__name__, schema)
+ return (fn.__name__, result, expected_result)
+ return
+ return wrapped
+
+
+class BasicGeneratorSet(object):
+ _instance = None
+
+ schema = {
+ "type": "object",
+ "properties": {
+ "name": {"type": "string"},
+ "http-method": {
+ "enum": ["GET", "PUT", "HEAD",
+ "POST", "PATCH", "DELETE", 'COPY']
+ },
+ "url": {"type": "string"},
+ "json-schema": jsonschema._utils.load_schema("draft4"),
+ "resources": {
+ "type": "array",
+ "items": {
+ "oneOf": [
+ {"type": "string"},
+ {
+ "type": "object",
+ "properties": {
+ "name": {"type": "string"},
+ "expected_result": {"type": "integer"}
+ }
+ }
+ ]
+ }
+ },
+ "results": {
+ "type": "object",
+ "properties": {}
+ }
+ },
+ "required": ["name", "http-method", "url"],
+ "additionalProperties": False,
+ }
+
+ def __new__(cls, *args, **kwargs):
+ if not cls._instance:
+ cls._instance = super(BasicGeneratorSet, cls).__new__(cls, *args,
+ **kwargs)
+ return cls._instance
+
+ def __init__(self):
+ self.types_dict = {}
+ for m in dir(self):
+ if callable(getattr(self, m)) and not'__' in m:
+ method = getattr(self, m)
+ if hasattr(method, "types"):
+ for type in method.types:
+ if type not in self.types_dict:
+ self.types_dict[type] = []
+ self.types_dict[type].append(method)
+
+ def validate_schema(self, schema):
+ jsonschema.validate(schema, self.schema)
+
+ def generate(self, schema):
+ """
+ Generate an json dictionary based on a schema.
+ Only one value is mis-generated for each dictionary created.
+
+ Any generator must return a list of tuples or a single tuple.
+ The values of this tuple are:
+ result[0]: Name of the test
+ result[1]: json schema for the test
+ result[2]: expected result of the test (can be None)
+ """
+ LOG.debug("generate_invalid: %s" % schema)
+ schema_type = schema["type"]
+ if isinstance(schema_type, list):
+ if "integer" in schema_type:
+ schema_type = "integer"
+ else:
+ raise Exception("non-integer list types not supported")
+ result = []
+ if schema_type not in self.types_dict:
+ raise Exception("generator (%s) doesn't support type: %s"
+ % (self.__class__.__name__, schema_type))
+ for generator in self.types_dict[schema_type]:
+ ret = generator(schema)
+ if ret is not None:
+ if isinstance(ret, list):
+ result.extend(ret)
+ elif isinstance(ret, tuple):
+ result.append(ret)
+ else:
+ raise Exception("generator (%s) returns invalid result: %s"
+ % (generator, ret))
+ LOG.debug("result: %s" % result)
+ return result
diff --git a/tempest/common/generator/negative_generator.py b/tempest/common/generator/negative_generator.py
new file mode 100644
index 0000000..4f3d2cd
--- /dev/null
+++ b/tempest/common/generator/negative_generator.py
@@ -0,0 +1,111 @@
+# Copyright 2014 Deutsche Telekom AG
+# All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+import copy
+
+import tempest.common.generator.base_generator as base
+import tempest.common.generator.valid_generator as valid
+from tempest.openstack.common import log as logging
+
+LOG = logging.getLogger(__name__)
+
+
+class NegativeTestGenerator(base.BasicGeneratorSet):
+ @base.generator_type("string")
+ @base.simple_generator
+ def gen_int(self, _):
+ return 4
+
+ @base.generator_type("integer")
+ @base.simple_generator
+ def gen_string(self, _):
+ return "XXXXXX"
+
+ @base.generator_type("integer", "string")
+ def gen_none(self, schema):
+ # Note(mkoderer): it's not using the decorator otherwise it'd be
+ # filtered
+ expected_result = base._check_for_expected_result('gen_none', schema)
+ return ('gen_none', None, expected_result)
+
+ @base.generator_type("string")
+ @base.simple_generator
+ def gen_str_min_length(self, schema):
+ min_length = schema.get("minLength", 0)
+ if min_length > 0:
+ return "x" * (min_length - 1)
+
+ @base.generator_type("string")
+ @base.simple_generator
+ def gen_str_max_length(self, schema):
+ max_length = schema.get("maxLength", -1)
+ if max_length > -1:
+ return "x" * (max_length + 1)
+
+ @base.generator_type("integer")
+ @base.simple_generator
+ def gen_int_min(self, schema):
+ if "minimum" in schema:
+ minimum = schema["minimum"]
+ if "exclusiveMinimum" not in schema:
+ minimum -= 1
+ return minimum
+
+ @base.generator_type("integer")
+ @base.simple_generator
+ def gen_int_max(self, schema):
+ if "maximum" in schema:
+ maximum = schema["maximum"]
+ if "exclusiveMaximum" not in schema:
+ maximum += 1
+ return maximum
+
+ @base.generator_type("object")
+ def gen_obj_remove_attr(self, schema):
+ invalids = []
+ valid_schema = valid.ValidTestGenerator().generate_valid(schema)
+ required = schema.get("required", [])
+ for r in required:
+ new_valid = copy.deepcopy(valid_schema)
+ del new_valid[r]
+ invalids.append(("gen_obj_remove_attr", new_valid, None))
+ return invalids
+
+ @base.generator_type("object")
+ @base.simple_generator
+ def gen_obj_add_attr(self, schema):
+ valid_schema = valid.ValidTestGenerator().generate_valid(schema)
+ if not schema.get("additionalProperties", True):
+ new_valid = copy.deepcopy(valid_schema)
+ new_valid["$$$$$$$$$$"] = "xxx"
+ return new_valid
+
+ @base.generator_type("object")
+ def gen_inv_prop_obj(self, schema):
+ LOG.debug("generate_invalid_object: %s" % schema)
+ valid_schema = valid.ValidTestGenerator().generate_valid(schema)
+ invalids = []
+ properties = schema["properties"]
+
+ for k, v in properties.iteritems():
+ for invalid in self.generate(v):
+ LOG.debug(v)
+ new_valid = copy.deepcopy(valid_schema)
+ new_valid[k] = invalid[1]
+ name = "prop_%s_%s" % (k, invalid[0])
+ invalids.append((name, new_valid, invalid[2]))
+
+ LOG.debug("generate_invalid_object return: %s" % invalids)
+ return invalids
diff --git a/tempest/common/generator/valid_generator.py b/tempest/common/generator/valid_generator.py
new file mode 100644
index 0000000..a99bbc0
--- /dev/null
+++ b/tempest/common/generator/valid_generator.py
@@ -0,0 +1,58 @@
+# Copyright 2014 Deutsche Telekom AG
+# All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+import tempest.common.generator.base_generator as base
+from tempest.openstack.common import log as logging
+
+
+LOG = logging.getLogger(__name__)
+
+
+class ValidTestGenerator(base.BasicGeneratorSet):
+ @base.generator_type("string")
+ @base.simple_generator
+ def generate_valid_string(self, schema):
+ size = schema.get("minLength", 0)
+ # TODO(dkr mko): handle format and pattern
+ return "x" * size
+
+ @base.generator_type("integer")
+ @base.simple_generator
+ def generate_valid_integer(self, schema):
+ # TODO(dkr mko): handle multipleOf
+ if "minimum" in schema:
+ minimum = schema["minimum"]
+ if "exclusiveMinimum" not in schema:
+ return minimum
+ else:
+ return minimum + 1
+ if "maximum" in schema:
+ maximum = schema["maximum"]
+ if "exclusiveMaximum" not in schema:
+ return maximum
+ else:
+ return maximum - 1
+ return 0
+
+ @base.generator_type("object")
+ @base.simple_generator
+ def generate_valid_object(self, schema):
+ obj = {}
+ for k, v in schema["properties"].iteritems():
+ obj[k] = self.generate_valid(v)
+ return obj
+
+ def generate_valid(self, schema):
+ return self.generate(schema)[0][1]
diff --git a/tempest/config.py b/tempest/config.py
index 0f5e23c..26b3bf1 100644
--- a/tempest/config.py
+++ b/tempest/config.py
@@ -806,6 +806,15 @@
help="Number of seconds to wait on a CLI timeout"),
]
+negative_group = cfg.OptGroup(name='negative', title="Negative Test Options")
+
+NegativeGroup = [
+ cfg.StrOpt('test_generator',
+ default='tempest.common.' +
+ 'generator.negative_generator.NegativeTestGenerator',
+ help="Test generator class for all negative tests"),
+]
+
def register_opts():
register_opt_group(cfg.CONF, compute_group, ComputeGroup)
@@ -840,6 +849,7 @@
register_opt_group(cfg.CONF, baremetal_group, BaremetalGroup)
register_opt_group(cfg.CONF, input_scenario_group, InputScenarioGroup)
register_opt_group(cfg.CONF, cli_group, CLIGroup)
+ register_opt_group(cfg.CONF, negative_group, NegativeGroup)
# this should never be called outside of this class
@@ -879,6 +889,7 @@
self.baremetal = cfg.CONF.baremetal
self.input_scenario = cfg.CONF['input-scenario']
self.cli = cfg.CONF.cli
+ self.negative = cfg.CONF.negative
if not self.compute_admin.username:
self.compute_admin.username = self.identity.admin_username
self.compute_admin.password = self.identity.admin_password
diff --git a/tempest/test.py b/tempest/test.py
index c6e3d6e..2125047 100644
--- a/tempest/test.py
+++ b/tempest/test.py
@@ -27,10 +27,11 @@
import testtools
from tempest import clients
-from tempest.common import generate_json
+import tempest.common.generator.valid_generator as valid
from tempest.common import isolated_creds
from tempest import config
from tempest import exceptions
+from tempest.openstack.common import importutils
from tempest.openstack.common import log as logging
LOG = logging.getLogger(__name__)
@@ -400,7 +401,8 @@
"""
description = NegativeAutoTest.load_schema(description_file)
LOG.debug(description)
- generate_json.validate_negative_test_schema(description)
+ generator = importutils.import_class(CONF.negative.test_generator)()
+ generator.validate_schema(description)
schema = description.get("json-schema", None)
resources = description.get("resources", [])
scenario_list = []
@@ -416,7 +418,7 @@
"expected_result": expected_result
}))
if schema is not None:
- for invalid in generate_json.generate_invalid(schema):
+ for invalid in generator.generate(schema):
scenario_list.append((invalid[0],
{"schema": invalid[1],
"expected_result": invalid[2]}))
@@ -459,11 +461,12 @@
# Note(mkoderer): The resources list already contains an invalid
# entry (see get_resource).
# We just send a valid json-schema with it
- valid = None
+ valid_schema = None
schema = description.get("json-schema", None)
if schema:
- valid = generate_json.generate_valid(schema)
- new_url, body = self._http_arguments(valid, url, method)
+ valid_schema = \
+ valid.ValidTestGenerator().generate_valid(schema)
+ new_url, body = self._http_arguments(valid_schema, url, method)
elif hasattr(self, "schema"):
new_url, body = self._http_arguments(self.schema, url, method)
diff --git a/tempest/tests/fake_config.py b/tempest/tests/fake_config.py
index 41b0558..e941606 100644
--- a/tempest/tests/fake_config.py
+++ b/tempest/tests/fake_config.py
@@ -43,6 +43,10 @@
swift = True
horizon = True
+ class fake_negative(object):
+ test_generator = 'tempest.common.' \
+ 'generator.negative_generator.NegativeTestGenerator'
+
compute_feature_enabled = fake_compute_feature_enabled()
volume_feature_enabled = fake_default_feature_enabled()
network_feature_enabled = fake_default_feature_enabled()
@@ -52,3 +56,5 @@
compute = fake_compute()
identity = fake_identity()
+
+ negative = fake_negative()
diff --git a/tempest/tests/negative/test_generate_json.py b/tempest/tests/negative/test_generate_json.py
index a0aa088..e09fcdf 100644
--- a/tempest/tests/negative/test_generate_json.py
+++ b/tempest/tests/negative/test_generate_json.py
@@ -13,11 +13,11 @@
# License for the specific language governing permissions and limitations
# under the License.
-from tempest.common import generate_json as gen
+from tempest.common.generator import negative_generator
import tempest.test
-class TestGenerateJson(tempest.test.BaseTestCase):
+class TestNegativeGenerator(tempest.test.BaseTestCase):
fake_input_str = {"type": "string",
"minLength": 2,
@@ -35,19 +35,23 @@
}
}
+ def setUp(self):
+ super(TestNegativeGenerator, self).setUp()
+ self.negative = negative_generator.NegativeTestGenerator()
+
def _validate_result(self, data):
self.assertTrue(isinstance(data, list))
for t in data:
self.assertTrue(isinstance(t, tuple))
def test_generate_invalid_string(self):
- result = gen.generate_invalid(self.fake_input_str)
+ result = self.negative.generate(self.fake_input_str)
self._validate_result(result)
def test_generate_invalid_integer(self):
- result = gen.generate_invalid(self.fake_input_int)
+ result = self.negative.generate(self.fake_input_int)
self._validate_result(result)
def test_generate_invalid_obj(self):
- result = gen.generate_invalid(self.fake_input_obj)
+ result = self.negative.generate(self.fake_input_obj)
self._validate_result(result)
diff --git a/tempest/tests/negative/test_negative_auto_test.py b/tempest/tests/negative/test_negative_auto_test.py
index 4c59383..27ddc95 100644
--- a/tempest/tests/negative/test_negative_auto_test.py
+++ b/tempest/tests/negative/test_negative_auto_test.py
@@ -16,9 +16,11 @@
import mock
import tempest.test as test
+from tempest.tests import base
+from tempest.tests import fake_config
-class TestNegativeAutoTest(test.BaseTestCase):
+class TestNegativeAutoTest(base.TestCase):
# Fake entries
_interface = 'json'
_service = 'compute'
@@ -34,6 +36,10 @@
"resources": ["flavor", "volume", "image"]
}
+ def setUp(self):
+ super(TestNegativeAutoTest, self).setUp()
+ self.stubs.Set(test, 'CONF', fake_config.FakeConfig)
+
def _check_prop_entries(self, result, entry):
entries = [a for a in result if entry in a[0]]
self.assertIsNotNone(entries)