Merge "Fix T104 for scenario test subdirs"
diff --git a/README.rst b/README.rst
index 4393ae9..ea36619 100644
--- a/README.rst
+++ b/README.rst
@@ -130,3 +130,57 @@
unittest2.TestSuite instead. See:
https://code.google.com/p/unittest-ext/issues/detail?id=79
+
+Branchless Tempest Considerations
+---------------------------------
+
+Starting with the OpenStack Icehouse release Tempest no longer has any stable
+branches. This is to better ensure API consistency between releases because
+the API behavior should not change between releases. This means that the stable
+branches are also gated by the Tempest master branch, which also means that
+proposed commits to Tempest must work against both the master and all the
+currently supported stable branches of the projects. As such there are a few
+special considerations that have to be accounted for when pushing new changes
+to tempest.
+
+1. New Tests for new features
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+When adding tests for new features that were not in previous releases of the
+projects the new test has to be properly skipped with a feature flag. Whether
+this is just as simple as using the @test.requires_ext() decorator to check
+if the required extension (or discoverable optional API) is enabled or adding
+a new config option to the appropriate section. If there isn't a method of
+selecting the new **feature** from the config file then there won't be a
+mechanism to disable the test with older stable releases and the new test won't
+be able to merge.
+
+2. Bug fix on core project needing Tempest changes
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+When trying to land a bug fix which changes a tested API you'll have to use the
+following procedure::
+
+ - Propose change to the project, get a +2 on the change even with failing
+ - Propose skip on Tempest which will only be approved after the
+ corresponding change in the project has a +2 on change
+ - Land project change in master and all open stable branches (if required)
+ - Land changed test in Tempest
+
+Otherwise the bug fix won't be able to land in the project.
+
+3. New Tests for existing features
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+If a test is being added for a feature that exists in all the current releases
+of the projects then the only concern is that the API behavior is the same
+across all the versions of the project being tested. If the behavior is not
+consistent the test will not be able to merge.
+
+API Stability
+-------------
+
+For new tests being added to Tempest the assumption is that the API being
+tested is considered stable and adheres to the OpenStack API stability
+guidelines. If an API is still considered experimental or in development then
+it should not be tested by Tempest until it is considered stable.
diff --git a/etc/tempest.conf.sample b/etc/tempest.conf.sample
index 9051310..f1712ac 100644
--- a/etc/tempest.conf.sample
+++ b/etc/tempest.conf.sample
@@ -424,6 +424,10 @@
# as [nova.rdp]->enabled in nova.conf (boolean value)
#rdp_console=false
+# Does the test environment support instance rescue mode?
+# (boolean value)
+#rescue=true
+
[dashboard]
diff --git a/requirements.txt b/requirements.txt
index f907e7d..ab2903a 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -8,7 +8,7 @@
paramiko>=1.13.0
netaddr>=0.7.6
python-glanceclient>=0.9.0
-python-keystoneclient>=0.8.0
+python-keystoneclient>=0.9.0
python-novaclient>=2.17.0
python-neutronclient>=2.3.4,<3
python-cinderclient>=1.0.6
@@ -18,8 +18,8 @@
python-swiftclient>=2.0.2
testresources>=0.2.4
testrepository>=0.0.18
-oslo.config>=1.2.0
-six>=1.6.0
+oslo.config>=1.2.1
+six>=1.7.0
iso8601>=0.1.9
fixtures>=0.3.14
testscenarios>=0.4
diff --git a/tempest/api/compute/admin/test_floating_ips_bulk.py b/tempest/api/compute/admin/test_floating_ips_bulk.py
index 8f875d2..208b032 100644
--- a/tempest/api/compute/admin/test_floating_ips_bulk.py
+++ b/tempest/api/compute/admin/test_floating_ips_bulk.py
@@ -74,7 +74,7 @@
self.assertEqual(self.ip_range, body['ip_range'])
resp, ips_list = self.client.list_floating_ips_bulk()
self.assertEqual(200, resp.status)
- self.assertNotEqual(0, len(body))
+ self.assertNotEqual(0, len(ips_list))
for ip in netaddr.IPNetwork(self.ip_range).iter_hosts():
self.assertIn(str(ip), map(lambda x: x['address'], ips_list))
resp, body = self.client.delete_floating_ips_bulk(self.ip_range)
diff --git a/tempest/api/compute/servers/test_server_group.py b/tempest/api/compute/servers/test_server_group.py
index 0cd23fd..f1ef5d5 100644
--- a/tempest/api/compute/servers/test_server_group.py
+++ b/tempest/api/compute/servers/test_server_group.py
@@ -75,6 +75,7 @@
policy = ['anti-affinity']
self._create_delete_server_group(policy)
+ @test.skip_because(bug="1324348")
@test.attr(type='gate')
def test_create_delete_server_group_with_multiple_policies(self):
# Create and Delete the server-group with multiple policies
diff --git a/tempest/api/compute/servers/test_server_rescue.py b/tempest/api/compute/servers/test_server_rescue.py
index 093e9e2..ab98d88 100644
--- a/tempest/api/compute/servers/test_server_rescue.py
+++ b/tempest/api/compute/servers/test_server_rescue.py
@@ -15,14 +15,21 @@
from tempest.api.compute import base
from tempest.common.utils import data_utils
+from tempest import config
from tempest import test
+CONF = config.CONF
+
class ServerRescueTestJSON(base.BaseV2ComputeTest):
@classmethod
@test.safe_setup
def setUpClass(cls):
+ if not CONF.compute_feature_enabled.rescue:
+ msg = "Server rescue not available."
+ raise cls.skipException(msg)
+
cls.set_network_resources(network=True, subnet=True, router=True)
super(ServerRescueTestJSON, cls).setUpClass()
diff --git a/tempest/api/compute/servers/test_server_rescue_negative.py b/tempest/api/compute/servers/test_server_rescue_negative.py
index dae4709..b35e55c 100644
--- a/tempest/api/compute/servers/test_server_rescue_negative.py
+++ b/tempest/api/compute/servers/test_server_rescue_negative.py
@@ -28,6 +28,10 @@
@classmethod
@test.safe_setup
def setUpClass(cls):
+ if not CONF.compute_feature_enabled.rescue:
+ msg = "Server rescue not available."
+ raise cls.skipException(msg)
+
cls.set_network_resources(network=True, subnet=True, router=True)
super(ServerRescueNegativeTestJSON, cls).setUpClass()
cls.device = 'vdf'
diff --git a/tempest/api/compute/v3/admin/test_servers_negative.py b/tempest/api/compute/v3/admin/test_servers_negative.py
index a971463..5eb6395 100644
--- a/tempest/api/compute/v3/admin/test_servers_negative.py
+++ b/tempest/api/compute/v3/admin/test_servers_negative.py
@@ -54,6 +54,7 @@
flavor_id = data_utils.rand_int_id(start=1000)
return flavor_id
+ @test.skip_because(bug="1298131")
@test.attr(type=['negative', 'gate'])
def test_resize_server_using_overlimit_ram(self):
flavor_name = data_utils.rand_name("flavor-")
@@ -67,11 +68,12 @@
ram, vcpus, disk,
flavor_id)
self.addCleanup(self.flavors_client.delete_flavor, flavor_id)
- self.assertRaises(exceptions.OverLimit,
+ self.assertRaises(exceptions.Unauthorized,
self.client.resize,
self.servers[0]['id'],
flavor_ref['id'])
+ @test.skip_because(bug="1298131")
@test.attr(type=['negative', 'gate'])
def test_resize_server_using_overlimit_vcpus(self):
flavor_name = data_utils.rand_name("flavor-")
@@ -85,7 +87,7 @@
ram, vcpus, disk,
flavor_id)
self.addCleanup(self.flavors_client.delete_flavor, flavor_id)
- self.assertRaises(exceptions.OverLimit,
+ self.assertRaises(exceptions.Unauthorized,
self.client.resize,
self.servers[0]['id'],
flavor_ref['id'])
diff --git a/tempest/api/compute/v3/servers/test_server_rescue.py b/tempest/api/compute/v3/servers/test_server_rescue.py
index b3dcb51..da58f26 100644
--- a/tempest/api/compute/v3/servers/test_server_rescue.py
+++ b/tempest/api/compute/v3/servers/test_server_rescue.py
@@ -14,13 +14,19 @@
# under the License.
from tempest.api.compute import base
+from tempest import config
from tempest import test
+CONF = config.CONF
+
class ServerRescueV3Test(base.BaseV3ComputeTest):
@classmethod
def setUpClass(cls):
+ if not CONF.compute_feature_enabled.rescue:
+ msg = "Server rescue not available."
+ raise cls.skipException(msg)
super(ServerRescueV3Test, cls).setUpClass()
# Server for positive tests
diff --git a/tempest/api/compute/v3/servers/test_server_rescue_negative.py b/tempest/api/compute/v3/servers/test_server_rescue_negative.py
index eb6bcdd..5eb6c9a 100644
--- a/tempest/api/compute/v3/servers/test_server_rescue_negative.py
+++ b/tempest/api/compute/v3/servers/test_server_rescue_negative.py
@@ -28,6 +28,10 @@
@classmethod
@test.safe_setup
def setUpClass(cls):
+ if not CONF.compute_feature_enabled.rescue:
+ msg = "Server rescue not available."
+ raise cls.skipException(msg)
+
super(ServerRescueNegativeV3Test, cls).setUpClass()
cls.device = 'vdf'
diff --git a/tempest/api/image/v1/test_images.py b/tempest/api/image/v1/test_images.py
index 2df3f7f..2cc2009 100644
--- a/tempest/api/image/v1/test_images.py
+++ b/tempest/api/image/v1/test_images.py
@@ -55,8 +55,7 @@
resp, body = self.create_image(name='New Remote Image',
container_format='bare',
disk_format='raw', is_public=False,
- location='http://example.com'
- '/someimage.iso',
+ location=CONF.image.http_image,
properties={'key1': 'value1',
'key2': 'value2'})
self.assertIn('id', body)
@@ -143,7 +142,7 @@
image
"""
name = 'New Remote Image %s' % name
- location = 'http://example.com/someimage_%s.iso' % name
+ location = CONF.image.http_image
resp, image = cls.create_image(name=name,
container_format=container_format,
disk_format=disk_format,
diff --git a/tempest/api_schema/compute/v2/floating_ips.py b/tempest/api_schema/compute/v2/floating_ips.py
index 3ea6320..03e6aef 100644
--- a/tempest/api_schema/compute/v2/floating_ips.py
+++ b/tempest/api_schema/compute/v2/floating_ips.py
@@ -98,3 +98,22 @@
add_remove_floating_ip = {
'status_code': [202]
}
+
+create_floating_ips_bulk = {
+ 'status_code': [200],
+ 'response_body': {
+ 'type': 'object',
+ 'properties': {
+ 'floating_ips_bulk_create': {
+ 'type': 'object',
+ 'properties': {
+ 'interface': {'type': ['string', 'null']},
+ 'ip_range': {'type': 'string'},
+ 'pool': {'type': ['string', 'null']},
+ },
+ 'required': ['interface', 'ip_range', 'pool']
+ }
+ },
+ 'required': ['floating_ips_bulk_create']
+ }
+}
diff --git a/tempest/clients.py b/tempest/clients.py
index 7532bf2..4050a20 100644
--- a/tempest/clients.py
+++ b/tempest/clients.py
@@ -456,6 +456,7 @@
HEATCLIENT_VERSION = '1'
IRONICCLIENT_VERSION = '1'
SAHARACLIENT_VERSION = '1.1'
+ CEILOMETERCLIENT_VERSION = '2'
def __init__(self, credentials):
# FIXME(andreaf) Auth provider for client_type 'official' is
@@ -476,6 +477,8 @@
credentials)
self.data_processing_client = self._get_data_processing_client(
credentials)
+ self.ceilometer_client = self._get_ceilometer_client(
+ credentials)
def _get_roles(self):
admin_credentials = auth.get_default_credentials('identity_admin')
@@ -696,3 +699,34 @@
auth_url=auth_url)
return client
+
+ def _get_ceilometer_client(self, credentials):
+ if not CONF.service_available.ceilometer:
+ return None
+
+ import ceilometerclient.client
+
+ keystone = self._get_identity_client(credentials)
+ region = CONF.identity.region
+
+ endpoint_type = CONF.telemetry.endpoint_type
+ service_type = CONF.telemetry.catalog_type
+ auth_url = CONF.identity.uri
+
+ try:
+ keystone.service_catalog.url_for(
+ attr='region',
+ filter_value=region,
+ service_type=service_type,
+ endpoint_type=endpoint_type)
+ except keystoneclient.exceptions.EndpointNotFound:
+ return None
+ else:
+ return ceilometerclient.client.get_client(
+ self.CEILOMETERCLIENT_VERSION,
+ os_username=credentials.username,
+ os_password=credentials.password,
+ os_tenant_name=credentials.tenant_name,
+ os_auth_url=auth_url,
+ os_service_type=service_type,
+ os_endpoint_type=endpoint_type)
diff --git a/tempest/common/rest_client.py b/tempest/common/rest_client.py
index 33128a9..9273437 100644
--- a/tempest/common/rest_client.py
+++ b/tempest/common/rest_client.py
@@ -18,6 +18,7 @@
import json
from lxml import etree
import re
+import string
import time
import jsonschema
@@ -295,9 +296,11 @@
req_url,
secs,
str(req_headers),
- str(req_body)[:2048],
+ filter(lambda x: x in string.printable,
+ str(req_body)[:2048]),
str(resp),
- str(resp_body)[:2048]),
+ filter(lambda x: x in string.printable,
+ str(resp_body)[:2048])),
extra=extra)
def _parse_resp(self, body):
diff --git a/tempest/config.py b/tempest/config.py
index e3f0f2a..4ea0702 100644
--- a/tempest/config.py
+++ b/tempest/config.py
@@ -310,7 +310,11 @@
cfg.BoolOpt('rdp_console',
default=False,
help='Enable RDP console. This configuration value should '
- 'be same as [nova.rdp]->enabled in nova.conf')
+ 'be same as [nova.rdp]->enabled in nova.conf'),
+ cfg.BoolOpt('rescue',
+ default=True,
+ help='Does the test environment support instance rescue '
+ 'mode?')
]
diff --git a/tempest/scenario/manager.py b/tempest/scenario/manager.py
index db26bda..07d8828 100644
--- a/tempest/scenario/manager.py
+++ b/tempest/scenario/manager.py
@@ -82,6 +82,7 @@
cls.object_storage_client = cls.manager.object_storage_client
cls.orchestration_client = cls.manager.orchestration_client
cls.data_processing_client = cls.manager.data_processing_client
+ cls.ceilometer_client = cls.manager.ceilometer_client
cls.resource_keys = {}
cls.os_resources = []
@@ -406,7 +407,16 @@
username = CONF.scenario.ssh_user
if private_key is None:
private_key = self.keypair.private_key
- return remote_client.RemoteClient(ip, username, pkey=private_key)
+ linux_client = remote_client.RemoteClient(ip, username,
+ pkey=private_key)
+ try:
+ linux_client.validate_authentication()
+ except exceptions.SSHTimeout:
+ LOG.exception('ssh connection to %s failed' % ip)
+ debug.log_net_debug()
+ raise
+
+ return linux_client
def _log_console_output(self, servers=None):
if not servers:
@@ -868,9 +878,7 @@
msg=msg)
if should_connect:
# no need to check ssh for negative connectivity
- linux_client = self.get_remote_client(ip_address, username,
- private_key)
- linux_client.validate_authentication()
+ self.get_remote_client(ip_address, username, private_key)
def _check_public_network_connectivity(self, ip_address, username,
private_key, should_connect=True,
@@ -884,13 +892,15 @@
username,
private_key,
should_connect=should_connect)
- except Exception:
+ except Exception as e:
ex_msg = 'Public network connectivity check failed'
if msg:
ex_msg += ": " + msg
LOG.exception(ex_msg)
self._log_console_output(servers)
- debug.log_net_debug()
+ # network debug is called as part of ssh init
+ if not isinstance(e, exceptions.SSHTimeout):
+ debug.log_net_debug()
raise
def _check_tenant_network_connectivity(self, server,
@@ -911,10 +921,12 @@
username,
private_key,
should_connect=should_connect)
- except Exception:
+ except Exception as e:
LOG.exception('Tenant network connectivity check failed')
self._log_console_output(servers_for_debug)
- debug.log_net_debug()
+ # network debug is called as part of ssh init
+ if not isinstance(e, exceptions.SSHTimeout):
+ debug.log_net_debug()
raise
def _check_remote_connectivity(self, source, dest, should_succeed=True):
diff --git a/tempest/scenario/test_load_balancer_basic.py b/tempest/scenario/test_load_balancer_basic.py
index 1c24b5c..03cfef5 100644
--- a/tempest/scenario/test_load_balancer_basic.py
+++ b/tempest/scenario/test_load_balancer_basic.py
@@ -148,7 +148,6 @@
ssh_client = self.get_remote_client(
server_or_ip=ip,
private_key=private_key)
- ssh_client.validate_authentication()
# Write a backend's responce into a file
resp = """HTTP/1.0 200 OK\r\nContent-Length: 8\r\n\r\n%s"""
diff --git a/tempest/scenario/test_minimum_basic.py b/tempest/scenario/test_minimum_basic.py
index 58f0dbf..0406217 100644
--- a/tempest/scenario/test_minimum_basic.py
+++ b/tempest/scenario/test_minimum_basic.py
@@ -93,11 +93,12 @@
def ssh_to_server(self):
try:
self.linux_client = self.get_remote_client(self.floating_ip.ip)
- self.linux_client.validate_authentication()
- except Exception:
+ except Exception as e:
LOG.exception('ssh to server failed')
self._log_console_output()
- debug.log_net_debug()
+ # network debug is called as part of ssh init
+ if not isinstance(e, test.exceptions.SSHTimeout):
+ debug.log_net_debug()
raise
def check_partitions(self):
diff --git a/tempest/scenario/test_security_groups_basic_ops.py b/tempest/scenario/test_security_groups_basic_ops.py
index b4e509a..dd89dc0 100644
--- a/tempest/scenario/test_security_groups_basic_ops.py
+++ b/tempest/scenario/test_security_groups_basic_ops.py
@@ -336,6 +336,8 @@
self.assertTrue(self._check_remote_connectivity(access_point, ip,
should_succeed),
msg)
+ except test.exceptions.SSHTimeout:
+ raise
except Exception:
debug.log_net_debug()
raise
diff --git a/tempest/scenario/test_server_basic_ops.py b/tempest/scenario/test_server_basic_ops.py
index dc7a092..54f1d9e 100644
--- a/tempest/scenario/test_server_basic_ops.py
+++ b/tempest/scenario/test_server_basic_ops.py
@@ -93,11 +93,10 @@
instance.add_floating_ip(floating_ip)
# Check ssh
try:
- linux_client = self.get_remote_client(
+ self.get_remote_client(
server_or_ip=floating_ip.ip,
username=self.image_utils.ssh_user(self.image_ref),
private_key=self.keypair.private_key)
- linux_client.validate_authentication()
except Exception:
LOG.exception('ssh to server failed')
self._log_console_output()
diff --git a/tempest/scenario/test_snapshot_pattern.py b/tempest/scenario/test_snapshot_pattern.py
index ab335e2..d41490a 100644
--- a/tempest/scenario/test_snapshot_pattern.py
+++ b/tempest/scenario/test_snapshot_pattern.py
@@ -49,8 +49,9 @@
try:
return self.get_remote_client(server_or_ip)
except Exception:
- LOG.exception()
+ LOG.exception('Initializing SSH connection failed')
self._log_console_output()
+ raise
def _write_timestamp(self, server_or_ip):
ssh_client = self._ssh_to_server(server_or_ip)
diff --git a/tempest/services/compute/json/floating_ips_client.py b/tempest/services/compute/json/floating_ips_client.py
index 6fc2bc2..92b4ddf 100644
--- a/tempest/services/compute/json/floating_ips_client.py
+++ b/tempest/services/compute/json/floating_ips_client.py
@@ -123,6 +123,7 @@
post_body = json.dumps({'floating_ips_bulk_create': post_body})
resp, body = self.post('os-floating-ips-bulk', post_body)
body = json.loads(body)
+ self.validate_response(schema.create_floating_ips_bulk, resp, body)
return resp, body['floating_ips_bulk_create']
def list_floating_ips_bulk(self):
diff --git a/tempest/test.py b/tempest/test.py
index bc94bcd..650fad7 100644
--- a/tempest/test.py
+++ b/tempest/test.py
@@ -107,6 +107,7 @@
'identity': True,
'object_storage': CONF.service_available.swift,
'dashboard': CONF.service_available.horizon,
+ 'ceilometer': CONF.service_available.ceilometer,
}
def decorator(f):
@@ -256,6 +257,12 @@
network_resources = {}
+ # NOTE(sdague): log_format is defined inline here instead of using the oslo
+ # default because going through the config path recouples config to the
+ # stress tests too early, and depending on testr order will fail unit tests
+ log_format = ('%(asctime)s %(process)d %(levelname)-8s '
+ '[%(name)s] %(message)s')
+
@classmethod
def setUpClass(cls):
if hasattr(super(BaseTestCase, cls), 'setUpClass'):
@@ -293,9 +300,8 @@
self.useFixture(fixtures.MonkeyPatch('sys.stderr', stderr))
if (os.environ.get('OS_LOG_CAPTURE') != 'False' and
os.environ.get('OS_LOG_CAPTURE') != '0'):
- log_format = '%(asctime)-15s %(message)s'
self.useFixture(fixtures.LoggerFixture(nuke_handlers=False,
- format=log_format,
+ format=self.log_format,
level=None))
@classmethod
diff --git a/tempest/tests/test_tenant_isolation.py b/tempest/tests/test_tenant_isolation.py
index 7a9b6be..485beff 100644
--- a/tempest/tests/test_tenant_isolation.py
+++ b/tempest/tests/test_tenant_isolation.py
@@ -59,6 +59,8 @@
'_get_object_storage_client'))
self.useFixture(mockpatch.PatchObject(clients.OfficialClientManager,
'_get_orchestration_client'))
+ self.useFixture(mockpatch.PatchObject(clients.OfficialClientManager,
+ '_get_ceilometer_client'))
iso_creds = isolated_creds.IsolatedCreds('test class',
tempest_client=False)
self.assertTrue(isinstance(iso_creds.identity_admin_client,
diff --git a/test-requirements.txt b/test-requirements.txt
index f63d34e..215f28b 100644
--- a/test-requirements.txt
+++ b/test-requirements.txt
@@ -1,7 +1,7 @@
hacking>=0.9.2,<0.10
# needed for doc build
docutils==0.9.1
-sphinx>=1.2.1,<1.3
+sphinx>=1.1.2,!=1.2.0,<1.3
python-subunit>=0.0.18
oslosphinx
mox>=0.5.3