Add support for negative tests with admin client
This adds the missing support for admin clients and implements an
example based on flavor creation. Instead of listing all result code
checks it is now possible to define on default.
Change-Id: I9512c1b91eb227e35faf24e3e88ed73a6ed3b734
Partially-implements: bp autogen-negative-tests
diff --git a/etc/schemas/compute/admin/flavor_create.json b/etc/schemas/compute/admin/flavor_create.json
new file mode 100644
index 0000000..0a3e7b3
--- /dev/null
+++ b/etc/schemas/compute/admin/flavor_create.json
@@ -0,0 +1,20 @@
+{
+ "name": "flavor-create",
+ "http-method": "POST",
+ "admin_client": true,
+ "url": "flavors",
+ "default_result_code": 400,
+ "json-schema": {
+ "type": "object",
+ "properties": {
+ "name": { "type": "string"},
+ "ram": { "type": "integer", "minimum": 1},
+ "vcpus": { "type": "integer", "minimum": 1},
+ "disk": { "type": "integer"},
+ "id": { "type": "integer"},
+ "swap": { "type": "integer"},
+ "rxtx_factor": { "type": "integer"},
+ "OS-FLV-EXT-DATA:ephemeral": { "type": "integer"}
+ }
+ }
+}
diff --git a/tempest/api/compute/admin/test_flavors_negative.py b/tempest/api/compute/admin/test_flavors_negative.py
index 49d49ef..b882ff4 100644
--- a/tempest/api/compute/admin/test_flavors_negative.py
+++ b/tempest/api/compute/admin/test_flavors_negative.py
@@ -13,6 +13,7 @@
# License for the specific language governing permissions and limitations
# under the License.
+import testscenarios
import uuid
from tempest.api.compute import base
@@ -20,6 +21,8 @@
from tempest import exceptions
from tempest import test
+load_tests = testscenarios.load_tests_apply_scenarios
+
class FlavorsAdminNegativeTestJSON(base.BaseV2ComputeAdminTest):
@@ -44,11 +47,6 @@
cls.swap = 1024
cls.rxtx = 2
- def flavor_clean_up(self, flavor_id):
- resp, body = self.client.delete_flavor(flavor_id)
- self.assertEqual(resp.status, 202)
- self.client.wait_for_resource_deletion(flavor_id)
-
@test.attr(type=['negative', 'gate'])
def test_get_flavor_details_for_deleted_flavor(self):
# Delete a flavor and ensure it is not listed
@@ -85,13 +83,6 @@
self.assertTrue(flag)
@test.attr(type=['negative', 'gate'])
- def test_invalid_is_public_string(self):
- # the 'is_public' parameter can be 'none/true/false' if it exists
- self.assertRaises(exceptions.BadRequest,
- self.client.list_flavors_with_detail,
- {'is_public': 'invalid'})
-
- @test.attr(type=['negative', 'gate'])
def test_create_flavor_as_user(self):
# only admin user can create a flavor
flavor_name = data_utils.rand_name(self.flavor_name_prefix)
@@ -110,231 +101,16 @@
self.user_client.delete_flavor,
self.flavor_ref_alt)
- @test.attr(type=['negative', 'gate'])
- def test_create_flavor_using_invalid_ram(self):
- # the 'ram' attribute must be positive integer
- flavor_name = data_utils.rand_name(self.flavor_name_prefix)
- new_flavor_id = str(uuid.uuid4())
- self.assertRaises(exceptions.BadRequest,
- self.client.create_flavor,
- flavor_name, -1, self.vcpus,
- self.disk, new_flavor_id)
+class FlavorCreateNegativeTestJSON(base.BaseV2ComputeAdminTest,
+ test.NegativeAutoTest):
+ _interface = 'json'
+ _service = 'compute'
+ _schema_file = 'compute/admin/flavor_create.json'
+
+ scenarios = test.NegativeAutoTest.generate_scenario(_schema_file)
@test.attr(type=['negative', 'gate'])
- def test_create_flavor_using_invalid_vcpus(self):
- # the 'vcpu' attribute must be positive integer
- flavor_name = data_utils.rand_name(self.flavor_name_prefix)
- new_flavor_id = str(uuid.uuid4())
-
- self.assertRaises(exceptions.BadRequest,
- self.client.create_flavor,
- flavor_name, self.ram, -1,
- self.disk, new_flavor_id)
-
- @test.attr(type=['negative', 'gate'])
- def test_create_flavor_with_name_length_less_than_1(self):
- # ensure name length >= 1
- new_flavor_id = str(uuid.uuid4())
-
- self.assertRaises(exceptions.BadRequest,
- self.client.create_flavor,
- '',
- self.ram, self.vcpus,
- self.disk,
- new_flavor_id,
- ephemeral=self.ephemeral,
- swap=self.swap,
- rxtx=self.rxtx,
- is_public='False')
-
- @test.attr(type=['negative', 'gate'])
- def test_create_flavor_with_name_length_exceeds_255(self):
- # ensure name do not exceed 255 characters
- new_flavor_name = 'a' * 256
- new_flavor_id = str(uuid.uuid4())
-
- self.assertRaises(exceptions.BadRequest,
- self.client.create_flavor,
- new_flavor_name,
- self.ram, self.vcpus,
- self.disk,
- new_flavor_id,
- ephemeral=self.ephemeral,
- swap=self.swap,
- rxtx=self.rxtx,
- is_public='False')
-
- @test.attr(type=['negative', 'gate'])
- def test_create_flavor_with_invalid_name(self):
- # the regex of flavor_name is '^[\w\.\- ]*$'
- invalid_flavor_name = data_utils.rand_name('invalid-!@#$%-')
- new_flavor_id = str(uuid.uuid4())
-
- self.assertRaises(exceptions.BadRequest,
- self.client.create_flavor,
- invalid_flavor_name,
- self.ram, self.vcpus,
- self.disk,
- new_flavor_id,
- ephemeral=self.ephemeral,
- swap=self.swap,
- rxtx=self.rxtx,
- is_public='False')
-
- @test.attr(type=['negative', 'gate'])
- def test_create_flavor_with_invalid_flavor_id(self):
- # the regex of flavor_id is '^[\w\.\- ]*$', and it cannot contain
- # leading and/or trailing whitespace
- new_flavor_name = data_utils.rand_name(self.flavor_name_prefix)
- invalid_flavor_id = '!@#$%'
-
- self.assertRaises(exceptions.BadRequest,
- self.client.create_flavor,
- new_flavor_name,
- self.ram, self.vcpus,
- self.disk,
- invalid_flavor_id,
- ephemeral=self.ephemeral,
- swap=self.swap,
- rxtx=self.rxtx,
- is_public='False')
-
- @test.attr(type=['negative', 'gate'])
- def test_create_flavor_with_id_length_exceeds_255(self):
- # the length of flavor_id should not exceed 255 characters
- new_flavor_name = data_utils.rand_name(self.flavor_name_prefix)
- invalid_flavor_id = 'a' * 256
-
- self.assertRaises(exceptions.BadRequest,
- self.client.create_flavor,
- new_flavor_name,
- self.ram, self.vcpus,
- self.disk,
- invalid_flavor_id,
- ephemeral=self.ephemeral,
- swap=self.swap,
- rxtx=self.rxtx,
- is_public='False')
-
- @test.attr(type=['negative', 'gate'])
- def test_create_flavor_with_invalid_root_gb(self):
- # root_gb attribute should be non-negative ( >= 0) integer
- new_flavor_name = data_utils.rand_name(self.flavor_name_prefix)
- new_flavor_id = str(uuid.uuid4())
-
- self.assertRaises(exceptions.BadRequest,
- self.client.create_flavor,
- new_flavor_name,
- self.ram, self.vcpus,
- -1,
- new_flavor_id,
- ephemeral=self.ephemeral,
- swap=self.swap,
- rxtx=self.rxtx,
- is_public='False')
-
- @test.attr(type=['negative', 'gate'])
- def test_create_flavor_with_invalid_ephemeral_gb(self):
- # ephemeral_gb attribute should be non-negative ( >= 0) integer
- new_flavor_name = data_utils.rand_name(self.flavor_name_prefix)
- new_flavor_id = str(uuid.uuid4())
-
- self.assertRaises(exceptions.BadRequest,
- self.client.create_flavor,
- new_flavor_name,
- self.ram, self.vcpus,
- self.disk,
- new_flavor_id,
- ephemeral=-1,
- swap=self.swap,
- rxtx=self.rxtx,
- is_public='False')
-
- @test.attr(type=['negative', 'gate'])
- def test_create_flavor_with_invalid_swap(self):
- # swap attribute should be non-negative ( >= 0) integer
- new_flavor_name = data_utils.rand_name(self.flavor_name_prefix)
- new_flavor_id = str(uuid.uuid4())
-
- self.assertRaises(exceptions.BadRequest,
- self.client.create_flavor,
- new_flavor_name,
- self.ram, self.vcpus,
- self.disk,
- new_flavor_id,
- ephemeral=self.ephemeral,
- swap=-1,
- rxtx=self.rxtx,
- is_public='False')
-
- @test.attr(type=['negative', 'gate'])
- def test_create_flavor_with_invalid_rxtx_factor(self):
- # rxtx_factor attribute should be a positive float
- new_flavor_name = data_utils.rand_name(self.flavor_name_prefix)
- new_flavor_id = str(uuid.uuid4())
-
- self.assertRaises(exceptions.BadRequest,
- self.client.create_flavor,
- new_flavor_name,
- self.ram, self.vcpus,
- self.disk,
- new_flavor_id,
- ephemeral=self.ephemeral,
- swap=self.swap,
- rxtx=-1.5,
- is_public='False')
-
- @test.attr(type=['negative', 'gate'])
- def test_create_flavor_with_invalid_is_public(self):
- # is_public attribute should be boolean
- new_flavor_name = data_utils.rand_name(self.flavor_name_prefix)
- new_flavor_id = str(uuid.uuid4())
-
- self.assertRaises(exceptions.BadRequest,
- self.client.create_flavor,
- new_flavor_name,
- self.ram, self.vcpus,
- self.disk,
- new_flavor_id,
- ephemeral=self.ephemeral,
- swap=self.swap,
- rxtx=self.rxtx,
- is_public='Invalid')
-
- @test.attr(type=['negative', 'gate'])
- def test_create_flavor_already_exists(self):
- flavor_name = data_utils.rand_name(self.flavor_name_prefix)
- new_flavor_id = str(uuid.uuid4())
-
- resp, flavor = self.client.create_flavor(flavor_name,
- self.ram, self.vcpus,
- self.disk,
- new_flavor_id,
- ephemeral=self.ephemeral,
- swap=self.swap,
- rxtx=self.rxtx)
- self.assertEqual(200, resp.status)
- self.addCleanup(self.flavor_clean_up, flavor['id'])
-
- self.assertRaises(exceptions.Conflict,
- self.client.create_flavor,
- flavor_name,
- self.ram, self.vcpus,
- self.disk,
- new_flavor_id,
- ephemeral=self.ephemeral,
- swap=self.swap,
- rxtx=self.rxtx)
-
- @test.attr(type=['negative', 'gate'])
- def test_delete_nonexistent_flavor(self):
- nonexistent_flavor_id = str(uuid.uuid4())
-
- self.assertRaises(exceptions.NotFound,
- self.client.delete_flavor,
- nonexistent_flavor_id)
-
-
-class FlavorsAdminNegativeTestXML(FlavorsAdminNegativeTestJSON):
- _interface = 'xml'
+ def test_create_flavor(self):
+ # flavor details are not returned for non-existent flavors
+ self.execute(self._schema_file)
diff --git a/tempest/api/compute/admin/test_flavors_negative_xml.py b/tempest/api/compute/admin/test_flavors_negative_xml.py
new file mode 100644
index 0000000..a06b0e6
--- /dev/null
+++ b/tempest/api/compute/admin/test_flavors_negative_xml.py
@@ -0,0 +1,268 @@
+# Copyright 2012 OpenStack Foundation
+# All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+import uuid
+
+from tempest.api.compute.admin import test_flavors_negative
+from tempest.common.utils import data_utils
+from tempest import exceptions
+from tempest import test
+
+
+class FlavorsAdminNegativeTestXML(test_flavors_negative.
+ FlavorsAdminNegativeTestJSON):
+
+ """
+ Tests Flavors API Create and Delete that require admin privileges
+ """
+
+ _interface = 'xml'
+
+ def flavor_clean_up(self, flavor_id):
+ resp, body = self.client.delete_flavor(flavor_id)
+ self.assertEqual(resp.status, 202)
+ self.client.wait_for_resource_deletion(flavor_id)
+
+ @test.attr(type=['negative', 'gate'])
+ def test_invalid_is_public_string(self):
+ # the 'is_public' parameter can be 'none/true/false' if it exists
+ self.assertRaises(exceptions.BadRequest,
+ self.client.list_flavors_with_detail,
+ {'is_public': 'invalid'})
+
+ @test.attr(type=['negative', 'gate'])
+ def test_create_flavor_using_invalid_ram(self):
+ # the 'ram' attribute must be positive integer
+ flavor_name = data_utils.rand_name(self.flavor_name_prefix)
+ new_flavor_id = str(uuid.uuid4())
+
+ self.assertRaises(exceptions.BadRequest,
+ self.client.create_flavor,
+ flavor_name, -1, self.vcpus,
+ self.disk, new_flavor_id)
+
+ @test.attr(type=['negative', 'gate'])
+ def test_create_flavor_using_invalid_vcpus(self):
+ # the 'vcpu' attribute must be positive integer
+ flavor_name = data_utils.rand_name(self.flavor_name_prefix)
+ new_flavor_id = str(uuid.uuid4())
+
+ self.assertRaises(exceptions.BadRequest,
+ self.client.create_flavor,
+ flavor_name, self.ram, -1,
+ self.disk, new_flavor_id)
+
+ @test.attr(type=['negative', 'gate'])
+ def test_create_flavor_with_name_length_less_than_1(self):
+ # ensure name length >= 1
+ new_flavor_id = str(uuid.uuid4())
+
+ self.assertRaises(exceptions.BadRequest,
+ self.client.create_flavor,
+ '',
+ self.ram, self.vcpus,
+ self.disk,
+ new_flavor_id,
+ ephemeral=self.ephemeral,
+ swap=self.swap,
+ rxtx=self.rxtx,
+ is_public='False')
+
+ @test.attr(type=['negative', 'gate'])
+ def test_create_flavor_with_name_length_exceeds_255(self):
+ # ensure name do not exceed 255 characters
+ new_flavor_name = 'a' * 256
+ new_flavor_id = str(uuid.uuid4())
+
+ self.assertRaises(exceptions.BadRequest,
+ self.client.create_flavor,
+ new_flavor_name,
+ self.ram, self.vcpus,
+ self.disk,
+ new_flavor_id,
+ ephemeral=self.ephemeral,
+ swap=self.swap,
+ rxtx=self.rxtx,
+ is_public='False')
+
+ @test.attr(type=['negative', 'gate'])
+ def test_create_flavor_with_invalid_name(self):
+ # the regex of flavor_name is '^[\w\.\- ]*$'
+ invalid_flavor_name = data_utils.rand_name('invalid-!@#$%-')
+ new_flavor_id = str(uuid.uuid4())
+
+ self.assertRaises(exceptions.BadRequest,
+ self.client.create_flavor,
+ invalid_flavor_name,
+ self.ram, self.vcpus,
+ self.disk,
+ new_flavor_id,
+ ephemeral=self.ephemeral,
+ swap=self.swap,
+ rxtx=self.rxtx,
+ is_public='False')
+
+ @test.attr(type=['negative', 'gate'])
+ def test_create_flavor_with_invalid_flavor_id(self):
+ # the regex of flavor_id is '^[\w\.\- ]*$', and it cannot contain
+ # leading and/or trailing whitespace
+ new_flavor_name = data_utils.rand_name(self.flavor_name_prefix)
+ invalid_flavor_id = '!@#$%'
+
+ self.assertRaises(exceptions.BadRequest,
+ self.client.create_flavor,
+ new_flavor_name,
+ self.ram, self.vcpus,
+ self.disk,
+ invalid_flavor_id,
+ ephemeral=self.ephemeral,
+ swap=self.swap,
+ rxtx=self.rxtx,
+ is_public='False')
+
+ @test.attr(type=['negative', 'gate'])
+ def test_create_flavor_with_id_length_exceeds_255(self):
+ # the length of flavor_id should not exceed 255 characters
+ new_flavor_name = data_utils.rand_name(self.flavor_name_prefix)
+ invalid_flavor_id = 'a' * 256
+
+ self.assertRaises(exceptions.BadRequest,
+ self.client.create_flavor,
+ new_flavor_name,
+ self.ram, self.vcpus,
+ self.disk,
+ invalid_flavor_id,
+ ephemeral=self.ephemeral,
+ swap=self.swap,
+ rxtx=self.rxtx,
+ is_public='False')
+
+ @test.attr(type=['negative', 'gate'])
+ def test_create_flavor_with_invalid_root_gb(self):
+ # root_gb attribute should be non-negative ( >= 0) integer
+ new_flavor_name = data_utils.rand_name(self.flavor_name_prefix)
+ new_flavor_id = str(uuid.uuid4())
+
+ self.assertRaises(exceptions.BadRequest,
+ self.client.create_flavor,
+ new_flavor_name,
+ self.ram, self.vcpus,
+ -1,
+ new_flavor_id,
+ ephemeral=self.ephemeral,
+ swap=self.swap,
+ rxtx=self.rxtx,
+ is_public='False')
+
+ @test.attr(type=['negative', 'gate'])
+ def test_create_flavor_with_invalid_ephemeral_gb(self):
+ # ephemeral_gb attribute should be non-negative ( >= 0) integer
+ new_flavor_name = data_utils.rand_name(self.flavor_name_prefix)
+ new_flavor_id = str(uuid.uuid4())
+
+ self.assertRaises(exceptions.BadRequest,
+ self.client.create_flavor,
+ new_flavor_name,
+ self.ram, self.vcpus,
+ self.disk,
+ new_flavor_id,
+ ephemeral=-1,
+ swap=self.swap,
+ rxtx=self.rxtx,
+ is_public='False')
+
+ @test.attr(type=['negative', 'gate'])
+ def test_create_flavor_with_invalid_swap(self):
+ # swap attribute should be non-negative ( >= 0) integer
+ new_flavor_name = data_utils.rand_name(self.flavor_name_prefix)
+ new_flavor_id = str(uuid.uuid4())
+
+ self.assertRaises(exceptions.BadRequest,
+ self.client.create_flavor,
+ new_flavor_name,
+ self.ram, self.vcpus,
+ self.disk,
+ new_flavor_id,
+ ephemeral=self.ephemeral,
+ swap=-1,
+ rxtx=self.rxtx,
+ is_public='False')
+
+ @test.attr(type=['negative', 'gate'])
+ def test_create_flavor_with_invalid_rxtx_factor(self):
+ # rxtx_factor attribute should be a positive float
+ new_flavor_name = data_utils.rand_name(self.flavor_name_prefix)
+ new_flavor_id = str(uuid.uuid4())
+
+ self.assertRaises(exceptions.BadRequest,
+ self.client.create_flavor,
+ new_flavor_name,
+ self.ram, self.vcpus,
+ self.disk,
+ new_flavor_id,
+ ephemeral=self.ephemeral,
+ swap=self.swap,
+ rxtx=-1.5,
+ is_public='False')
+
+ @test.attr(type=['negative', 'gate'])
+ def test_create_flavor_with_invalid_is_public(self):
+ # is_public attribute should be boolean
+ new_flavor_name = data_utils.rand_name(self.flavor_name_prefix)
+ new_flavor_id = str(uuid.uuid4())
+
+ self.assertRaises(exceptions.BadRequest,
+ self.client.create_flavor,
+ new_flavor_name,
+ self.ram, self.vcpus,
+ self.disk,
+ new_flavor_id,
+ ephemeral=self.ephemeral,
+ swap=self.swap,
+ rxtx=self.rxtx,
+ is_public='Invalid')
+
+ @test.attr(type=['negative', 'gate'])
+ def test_create_flavor_already_exists(self):
+ flavor_name = data_utils.rand_name(self.flavor_name_prefix)
+ new_flavor_id = str(uuid.uuid4())
+
+ resp, flavor = self.client.create_flavor(flavor_name,
+ self.ram, self.vcpus,
+ self.disk,
+ new_flavor_id,
+ ephemeral=self.ephemeral,
+ swap=self.swap,
+ rxtx=self.rxtx)
+ self.assertEqual(200, resp.status)
+ self.addCleanup(self.flavor_clean_up, flavor['id'])
+
+ self.assertRaises(exceptions.Conflict,
+ self.client.create_flavor,
+ flavor_name,
+ self.ram, self.vcpus,
+ self.disk,
+ new_flavor_id,
+ ephemeral=self.ephemeral,
+ swap=self.swap,
+ rxtx=self.rxtx)
+
+ @test.attr(type=['negative', 'gate'])
+ def test_delete_nonexistent_flavor(self):
+ nonexistent_flavor_id = str(uuid.uuid4())
+
+ self.assertRaises(exceptions.NotFound,
+ self.client.delete_flavor,
+ nonexistent_flavor_id)
diff --git a/tempest/common/generator/base_generator.py b/tempest/common/generator/base_generator.py
index 35f8158..7e7a2d6 100644
--- a/tempest/common/generator/base_generator.py
+++ b/tempest/common/generator/base_generator.py
@@ -59,7 +59,9 @@
"enum": ["GET", "PUT", "HEAD",
"POST", "PATCH", "DELETE", 'COPY']
},
+ "admin_client": {"type": "boolean"},
"url": {"type": "string"},
+ "default_result_code": {"type": "integer"},
"json-schema": jsonschema._utils.load_schema("draft4"),
"resources": {
"type": "array",
diff --git a/tempest/test.py b/tempest/test.py
index 2125047..20581ab 100644
--- a/tempest/test.py
+++ b/tempest/test.py
@@ -363,6 +363,9 @@
super(NegativeAutoTest, cls).setUpClass()
os = cls.get_client_manager()
cls.client = os.negative_client
+ os_admin = clients.AdminManager(interface=cls._interface,
+ service=cls._service)
+ cls.admin_client = os_admin.negative_client
@staticmethod
def load_schema(file):
@@ -418,10 +421,13 @@
"expected_result": expected_result
}))
if schema is not None:
- for invalid in generator.generate(schema):
- scenario_list.append((invalid[0],
- {"schema": invalid[1],
- "expected_result": invalid[2]}))
+ for name, schema, expected_result in generator.generate(schema):
+ if (expected_result is None and
+ "default_result_code" in description):
+ expected_result = description["default_result_code"]
+ scenario_list.append((name,
+ {"schema": schema,
+ "expected_result": expected_result}))
LOG.debug(scenario_list)
return scenario_list
@@ -470,8 +476,12 @@
elif hasattr(self, "schema"):
new_url, body = self._http_arguments(self.schema, url, method)
- resp, resp_body = self.client.send_request(method, new_url,
- resources, body=body)
+ if "admin_client" in description and description["admin_client"]:
+ client = self.admin_client
+ else:
+ client = self.client
+ resp, resp_body = client.send_request(method, new_url,
+ resources, body=body)
self._check_negative_response(resp.status, resp_body)
def _http_arguments(self, json_dict, url, method):