Merge "part-2 expect badRequest in server metadata test"
diff --git a/HACKING.rst b/HACKING.rst
index e920634..7363e7f 100644
--- a/HACKING.rst
+++ b/HACKING.rst
@@ -111,9 +111,9 @@
Negative Tests
--------------
Newly added negative tests should use the negative test framework. First step
-is to create an interface description in a json file under `etc/schemas`.
-These descriptions consists of two important sections for the test
-(one of those is mandatory):
+is to create an interface description in a python file under
+`tempest/api_schema/request/`. These descriptions consists of two important
+sections for the test (one of those is mandatory):
- A resource (part of the URL of the request): Resources needed for a test
must be created in `setUpClass` and registered with `set_resource` e.g.:
@@ -126,21 +126,17 @@
load_tests = test.NegativeAutoTest.load_tests
- class SampeTestNegativeTestJSON(<your base class>, test.NegativeAutoTest):
- _interface = 'json'
+ @test.SimpleNegativeAutoTest
+ class SampleTestNegativeTestJSON(<your base class>, test.NegativeAutoTest):
_service = 'compute'
- _schema_file = <your Schema file>
+ _schema = <your schema file>
-Negative tests must be marked with a negative attribute::
-
- @test.attr(type=['negative', 'gate'])
- def test_get_console_output(self):
- self.execute(self._schema_file)
+The class decorator `SimpleNegativeAutoTest` will automatically generate test
+cases out of the given schema in the attribute `_schema`.
All negative tests should be added into a separate negative test file.
If such a file doesn't exist for the particular resource being tested a new
-test file should be added. Old XML based negative tests can be kept but should
-be renamed to `_xml.py`.
+test file should be added.
Test skips because of Known Bugs
--------------------------------
diff --git a/tempest/api/compute/admin/test_flavors_negative.py b/tempest/api/compute/admin/test_flavors_negative.py
index 5bc3d10..fb27360 100644
--- a/tempest/api/compute/admin/test_flavors_negative.py
+++ b/tempest/api/compute/admin/test_flavors_negative.py
@@ -18,9 +18,13 @@
from tempest.api.compute import base
from tempest.api_schema.request.compute.v2 import flavors
from tempest.common.utils import data_utils
+from tempest import config
from tempest import exceptions
from tempest import test
+
+CONF = config.CONF
+
load_tests = test.NegativeAutoTest.load_tests
@@ -106,5 +110,5 @@
class FlavorCreateNegativeTestJSON(base.BaseV2ComputeAdminTest,
test.NegativeAutoTest):
_interface = 'json'
- _service = 'compute'
+ _service = CONF.compute.catalog_type
_schema = flavors.flavor_create
diff --git a/tempest/api/compute/flavors/test_flavors_negative.py b/tempest/api/compute/flavors/test_flavors_negative.py
index cae1ac4..83f8e19 100644
--- a/tempest/api/compute/flavors/test_flavors_negative.py
+++ b/tempest/api/compute/flavors/test_flavors_negative.py
@@ -15,23 +15,26 @@
from tempest.api.compute import base
from tempest.api_schema.request.compute.v2 import flavors
+from tempest import config
from tempest import test
+CONF = config.CONF
+
load_tests = test.NegativeAutoTest.load_tests
@test.SimpleNegativeAutoTest
class FlavorsListWithDetailsNegativeTestJSON(base.BaseV2ComputeTest,
test.NegativeAutoTest):
- _service = 'compute'
+ _service = CONF.compute.catalog_type
_schema = flavors.flavor_list
@test.SimpleNegativeAutoTest
class FlavorDetailsNegativeTestJSON(base.BaseV2ComputeTest,
test.NegativeAutoTest):
- _service = 'compute'
+ _service = CONF.compute.catalog_type
_schema = flavors.flavors_details
@classmethod
diff --git a/tempest/api_schema/response/compute/services.py b/tempest/api_schema/response/compute/services.py
index fc42b89..6f361ef 100644
--- a/tempest/api_schema/response/compute/services.py
+++ b/tempest/api_schema/response/compute/services.py
@@ -22,7 +22,8 @@
'items': {
'type': 'object',
'properties': {
- 'id': {'type': 'integer'},
+ 'id': {'type': ['integer', 'string'],
+ 'pattern': '^[a-zA-Z!]*@[0-9]+$'},
'zone': {'type': 'string'},
'host': {'type': 'string'},
'state': {'type': 'string'},
diff --git a/tempest/cli/simple_read_only/network/test_neutron.py b/tempest/cli/simple_read_only/network/test_neutron.py
index 6090882..6cf0640 100644
--- a/tempest/cli/simple_read_only/network/test_neutron.py
+++ b/tempest/cli/simple_read_only/network/test_neutron.py
@@ -198,6 +198,31 @@
'auth_mode', 'status'])
@test.attr(type='smoke')
+ @test.requires_ext(extension='fwaas', service='network')
+ def test_neutron_firewall_list(self):
+ firewall_list = self.parser.listing(self.neutron
+ ('firewall-list'))
+ self.assertTableStruct(firewall_list, ['id', 'name',
+ 'firewall_policy_id'])
+
+ @test.attr(type='smoke')
+ @test.requires_ext(extension='fwaas', service='network')
+ def test_neutron_firewall_policy_list(self):
+ firewall_policy = self.parser.listing(self.neutron
+ ('firewall-policy-list'))
+ self.assertTableStruct(firewall_policy, ['id', 'name',
+ 'firewall_rules'])
+
+ @test.attr(type='smoke')
+ @test.requires_ext(extension='fwaas', service='network')
+ def test_neutron_firewall_rule_list(self):
+ firewall_rule = self.parser.listing(self.neutron
+ ('firewall-rule-list'))
+ self.assertTableStruct(firewall_rule, ['id', 'name',
+ 'firewall_policy_id',
+ 'summary', 'enabled'])
+
+ @test.attr(type='smoke')
def test_neutron_help(self):
help_text = self.neutron('help')
lines = help_text.split('\n')
diff --git a/tempest/cmd/cleanup.py b/tempest/cmd/cleanup.py
index a305e42..f36ef56 100755
--- a/tempest/cmd/cleanup.py
+++ b/tempest/cmd/cleanup.py
@@ -9,7 +9,7 @@
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
-# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
@@ -36,14 +36,14 @@
**NOTE**: The _tenants_to_clean array in dry-run.json lists the
tenants that cleanup will loop through and delete child objects, not
delete the tenant itself. This may differ from the tenants array as you
-can clean the tempest and alternate tempest tenants but not delete the
-tenants themselves. This is actually the default behavior.
+can clean the tempest and alternate tempest tenants but by default,
+cleanup deletes the objects in the tempest and alternate tempest tenants
+but does not delete those tenants unless the --delete-tempest-conf-objects
+flag is used to force their deletion.
**Normal mode**: running with no arguments, will query your deployment and
-build a list of objects to delete after filtering out out the objects
-found in saved_state.json and based on the
---preserve-tempest-conf-objects and
---delete-tempest-conf-objects flags.
+build a list of objects to delete after filtering out the objects found in
+saved_state.json and based on the --delete-tempest-conf-objects flag.
By default the tempest and alternate tempest users and tenants are not
deleted and the admin user specified in tempest.conf is never deleted.
@@ -84,7 +84,6 @@
# available services
self.tenant_services = cleanup_service.get_tenant_cleanup_services()
self.global_services = cleanup_service.get_global_cleanup_services()
- cleanup_service.init_conf()
def run(self):
opts = self.options
@@ -98,7 +97,7 @@
def _cleanup(self):
LOG.debug("Begin cleanup")
is_dry_run = self.options.dry_run
- is_preserve = self.options.preserve_tempest_conf_objects
+ is_preserve = not self.options.delete_tempest_conf_objects
is_save_state = False
if is_dry_run:
@@ -149,7 +148,7 @@
LOG.debug("Cleaning tenant: %s " % tenant['name'])
is_dry_run = self.options.dry_run
dry_run_data = self.dry_run_data
- is_preserve = self.options.preserve_tempest_conf_objects
+ is_preserve = not self.options.delete_tempest_conf_objects
tenant_id = tenant['id']
tenant_name = tenant['name']
tenant_data = None
@@ -194,23 +193,16 @@
dest='init_saved_state', default=False,
help="Creates JSON file: " + SAVED_STATE_JSON +
", representing the current state of your "
- "deployment, specifically objects types "
- "Tempest creates and destroys during a run. "
+ "deployment, specifically object types "
+ "tempest creates and destroys during a run. "
"You must run with this flag prior to "
- "executing cleanup.")
- parser.add_argument('--preserve-tempest-conf-objects',
- action="store_true",
- dest='preserve_tempest_conf_objects',
- default=True, help="Do not delete the "
- "tempest and alternate tempest users and "
- "tenants, so they may be used for future "
- "tempest runs. By default this is argument "
- "is true.")
+ "executing cleanup in normal mode, which is with "
+ "no arguments.")
parser.add_argument('--delete-tempest-conf-objects',
- action="store_false",
- dest='preserve_tempest_conf_objects',
+ action="store_true",
+ dest='delete_tempest_conf_objects',
default=False,
- help="Delete the tempest and "
+ help="Force deletion of the tempest and "
"alternate tempest users and tenants.")
parser.add_argument('--dry-run', action="store_true",
dest='dry_run', default=False,
@@ -291,6 +283,7 @@
def main():
+ cleanup_service.init_conf()
cleanup = Cleanup()
cleanup.run()
LOG.info('Cleanup finished!')
diff --git a/tempest/cmd/cleanup_service.py b/tempest/cmd/cleanup_service.py
index 8adfbef..67843e6 100644
--- a/tempest/cmd/cleanup_service.py
+++ b/tempest/cmd/cleanup_service.py
@@ -1,3 +1,5 @@
+#!/usr/bin/env python
+
# Copyright 2014 Dell Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
@@ -12,6 +14,7 @@
# License for the specific language governing permissions and limitations
# under the License.
+from tempest import clients
from tempest import config
from tempest.openstack.common import log as logging
from tempest import test
@@ -19,13 +22,14 @@
LOG = logging.getLogger(__name__)
CONF = config.CONF
-CONF_USERS = None
-CONF_TENANTS = None
-CONF_PUB_NETWORK = None
-CONF_PRIV_NETWORK_NAME = None
-CONF_PUB_ROUTER = None
CONF_FLAVORS = None
CONF_IMAGES = None
+CONF_NETWORKS = []
+CONF_PRIV_NETWORK_NAME = None
+CONF_PUB_NETWORK = None
+CONF_PUB_ROUTER = None
+CONF_TENANTS = None
+CONF_USERS = None
IS_CEILOMETER = None
IS_CINDER = None
@@ -36,14 +40,15 @@
def init_conf():
- global CONF_USERS
- global CONF_TENANTS
- global CONF_PUB_NETWORK
- global CONF_PRIV_NETWORK_NAME
- global CONF_PUB_ROUTER
global CONF_FLAVORS
global CONF_IMAGES
-
+ global CONF_NETWORKS
+ global CONF_PRIV_NETWORK
+ global CONF_PRIV_NETWORK_NAME
+ global CONF_PUB_NETWORK
+ global CONF_PUB_ROUTER
+ global CONF_TENANTS
+ global CONF_USERS
global IS_CEILOMETER
global IS_CINDER
global IS_GLANCE
@@ -51,17 +56,6 @@
global IS_NEUTRON
global IS_NOVA
- CONF_USERS = [CONF.identity.admin_username, CONF.identity.username,
- CONF.identity.alt_username]
- CONF_TENANTS = [CONF.identity.admin_tenant_name,
- CONF.identity.tenant_name,
- CONF.identity.alt_tenant_name]
- CONF_PUB_NETWORK = CONF.network.public_network_id
- CONF_PRIV_NETWORK_NAME = CONF.compute.fixed_network_name
- CONF_PUB_ROUTER = CONF.network.public_router_id
- CONF_FLAVORS = [CONF.compute.flavor_ref, CONF.compute.flavor_ref_alt]
- CONF_IMAGES = [CONF.compute.image_ref, CONF.compute.image_ref_alt]
-
IS_CEILOMETER = CONF.service_available.ceilometer
IS_CINDER = CONF.service_available.cinder
IS_GLANCE = CONF.service_available.glance
@@ -69,6 +63,38 @@
IS_NEUTRON = CONF.service_available.neutron
IS_NOVA = CONF.service_available.nova
+ CONF_FLAVORS = [CONF.compute.flavor_ref, CONF.compute.flavor_ref_alt]
+ CONF_IMAGES = [CONF.compute.image_ref, CONF.compute.image_ref_alt]
+ CONF_PRIV_NETWORK_NAME = CONF.compute.fixed_network_name
+ CONF_PUB_NETWORK = CONF.network.public_network_id
+ CONF_PUB_ROUTER = CONF.network.public_router_id
+ CONF_TENANTS = [CONF.identity.admin_tenant_name,
+ CONF.identity.tenant_name,
+ CONF.identity.alt_tenant_name]
+ CONF_USERS = [CONF.identity.admin_username, CONF.identity.username,
+ CONF.identity.alt_username]
+
+ if IS_NEUTRON:
+ CONF_PRIV_NETWORK = _get_priv_net_id(CONF.compute.fixed_network_name,
+ CONF.identity.tenant_name)
+ CONF_NETWORKS = [CONF_PUB_NETWORK, CONF_PRIV_NETWORK]
+
+
+def _get_priv_net_id(prv_net_name, tenant_name):
+ am = clients.AdminManager()
+ net_cl = am.network_client
+ id_cl = am.identity_client
+
+ _, networks = net_cl.list_networks()
+ tenant = id_cl.get_tenant_by_name(tenant_name)
+ t_id = tenant['id']
+ n_id = None
+ for net in networks['networks']:
+ if (net['tenant_id'] == t_id and net['name'] == prv_net_name):
+ n_id = net['id']
+ break
+ return n_id
+
class BaseService(object):
def __init__(self, kwargs):
@@ -84,11 +110,8 @@
or 'tenant_id' not in item_list[0]):
return item_list
- _filtered_list = []
- for item in item_list:
- if item['tenant_id'] == self.tenant_id:
- _filtered_list.append(item)
- return _filtered_list
+ return [item for item in item_list
+ if item['tenant_id'] == self.tenant_id]
def list(self):
pass
@@ -325,6 +348,13 @@
super(NetworkService, self).__init__(kwargs)
self.client = manager.network_client
+ def _filter_by_conf_networks(self, item_list):
+ if not item_list or not all(('network_id' in i for i in item_list)):
+ return item_list
+
+ return [item for item in item_list if item['network_id']
+ not in CONF_NETWORKS]
+
def list(self):
client = self.client
_, networks = client.list_networks()
@@ -332,8 +362,7 @@
# filter out networks declared in tempest.conf
if self.is_preserve:
networks = [network for network in networks
- if (network['name'] != CONF_PRIV_NETWORK_NAME
- and network['id'] != CONF_PUB_NETWORK)]
+ if network['id'] not in CONF_NETWORKS]
LOG.debug("List count, %s Networks" % networks)
return networks
@@ -527,7 +556,7 @@
for port in ports:
subid = port['fixed_ips'][0]['subnet_id']
client.remove_router_interface_with_subnet_id(rid, subid)
- client.delete_router(rid)
+ client.delete_router(rid)
except Exception as e:
LOG.exception("Delete Router exception: %s" % e)
pass
@@ -694,6 +723,8 @@
_, ports = client.list_ports()
ports = ports['ports']
ports = self._filter_by_tenant_id(ports)
+ if self.is_preserve:
+ ports = self._filter_by_conf_networks(ports)
LOG.debug("List count, %s Ports" % len(ports))
return ports
@@ -719,6 +750,8 @@
_, subnets = client.list_subnets()
subnets = subnets['subnets']
subnets = self._filter_by_tenant_id(subnets)
+ if self.is_preserve:
+ subnets = self._filter_by_conf_networks(subnets)
LOG.debug("List count, %s Subnets" % len(subnets))
return subnets
diff --git a/tempest/scenario/test_dashboard_basic_ops.py b/tempest/scenario/test_dashboard_basic_ops.py
index 1a10b79..2014293 100644
--- a/tempest/scenario/test_dashboard_basic_ops.py
+++ b/tempest/scenario/test_dashboard_basic_ops.py
@@ -65,7 +65,7 @@
def check_login_page(self):
response = urllib2.urlopen(CONF.dashboard.dashboard_url)
- self.assertIn("<h3>Log In</h3>", response.read())
+ self.assertIn("Log In", response.read())
def user_login(self):
self.opener = urllib2.build_opener(urllib2.HTTPCookieProcessor())
diff --git a/tempest/scenario/test_large_ops.py b/tempest/scenario/test_large_ops.py
index 91b95a8..e9fa960 100644
--- a/tempest/scenario/test_large_ops.py
+++ b/tempest/scenario/test_large_ops.py
@@ -12,6 +12,7 @@
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
+from tempest_lib import exceptions
from tempest.common.utils import data_utils
from tempest import config
@@ -44,6 +45,22 @@
"instances")
cls.set_network_resources()
super(TestLargeOpsScenario, cls).resource_setup()
+ # list of cleanup calls to be executed in reverse order
+ cls._cleanup_resources = []
+
+ @classmethod
+ def resource_cleanup(cls):
+ while cls._cleanup_resources:
+ function, args, kwargs = cls._cleanup_resources.pop(-1)
+ try:
+ function(*args, **kwargs)
+ except exceptions.NotFound:
+ pass
+ super(TestLargeOpsScenario, cls).resource_cleanup()
+
+ @classmethod
+ def addCleanupClass(cls, function, *arguments, **keywordArguments):
+ cls._cleanup_resources.append((function, arguments, keywordArguments))
def _wait_for_server_status(self, status):
for server in self.servers:
@@ -54,7 +71,14 @@
def nova_boot(self):
name = data_utils.rand_name('scenario-server-')
flavor_id = CONF.compute.flavor_ref
- secgroup = self._create_security_group()
+ # Explicitly create secgroup to avoid cleanup at the end of testcases.
+ # Since no traffic is tested, we don't need to actually add rules to
+ # secgroup
+ _, secgroup = self.security_groups_client.create_security_group(
+ 'secgroup-%s' % name, 'secgroup-desc-%s' % name)
+ self.addCleanupClass(self.security_groups_client.delete_security_group,
+ secgroup['id'])
+
self.servers_client.create_server(
name,
self.image,
@@ -68,15 +92,12 @@
for server in self.servers:
# after deleting all servers - wait for all servers to clear
# before cleanup continues
- self.addCleanup(self.servers_client.wait_for_server_termination,
- server['id'])
+ self.addCleanupClass(self.servers_client.
+ wait_for_server_termination,
+ server['id'])
for server in self.servers:
- self.addCleanup_with_wait(
- waiter_callable=(self.servers_client.
- wait_for_server_termination),
- thing_id=server['id'], thing_id_param='server_id',
- cleanup_callable=self.delete_wrapper,
- cleanup_args=[self.servers_client.delete_server, server['id']])
+ self.addCleanupClass(self.servers_client.delete_server,
+ server['id'])
self._wait_for_server_status('ACTIVE')
def _large_ops_scenario(self):
diff --git a/tempest/test.py b/tempest/test.py
index 14cf3bb..7db0376 100644
--- a/tempest/test.py
+++ b/tempest/test.py
@@ -414,12 +414,8 @@
else:
standard_tests, module, loader = args
for test in testtools.iterate_tests(standard_tests):
- schema_file = getattr(test, '_schema_file', None)
schema = getattr(test, '_schema', None)
- if schema_file is not None:
- setattr(test, 'scenarios',
- NegativeAutoTest.generate_scenario(schema_file))
- elif schema is not None:
+ if schema is not None:
setattr(test, 'scenarios',
NegativeAutoTest.generate_scenario(schema))
return testscenarios.load_tests_apply_scenarios(*args)