Fix and simplify reboot stress test. Add new env stuff.
Added stress environment var for max_instances.
Added tempest var for log_level of rest_client.
Change-Id: Ia92536a547cdab4d2496bd9ba9067be3595b79cd
xxxxx
Change-Id: I8c2f499a41f74c2fe6fb08c80ab6fc31f6b93426
diff --git a/stress/README.rst b/stress/README.rst
index bf44f13..1667e31 100644
--- a/stress/README.rst
+++ b/stress/README.rst
@@ -32,6 +32,7 @@
host_admin_user=<name of user for ssh command>
nova_logdir=<value of --logdir in nova.conf>
controller=<hostname for calling nova-manage>
+ max_instances=<limit on instances that will be created>
The stress test needs the top-level tempest directory to be on PYTHONPATH
if you are not using nosetests to run.
diff --git a/stress/config.py b/stress/config.py
index 0dce816..16972d8 100755
--- a/stress/config.py
+++ b/stress/config.py
@@ -46,3 +46,8 @@
def controller(self):
"""Controller host"""
return self.get("controller", None)
+
+ @property
+ def max_instances(self):
+ """Maximum number of instances to create during test"""
+ return self.get("max_instances", 16)
diff --git a/stress/driver.py b/stress/driver.py
index b68825e..e8c8f12 100644
--- a/stress/driver.py
+++ b/stress/driver.py
@@ -114,14 +114,14 @@
(default: 32)
`seed` = random seed (default: None)
"""
+ stress_config = StressConfig(manager.config._conf)
# get keyword arguments
duration = kwargs.get('duration', datetime.timedelta(seconds=10))
seed = kwargs.get('seed', None)
sleep_time = float(kwargs.get('sleep_time', 3000)) / 1000
- max_vms = int(kwargs.get('max_vms', 32))
+ max_vms = int(kwargs.get('max_vms', stress_config.max_instances))
test_name = kwargs.get('test_name', 'unamed test')
- stress_config = StressConfig(manager.config._conf)
keypath = stress_config.host_private_key_path
user = stress_config.host_admin_user
logdir = stress_config.nova_logdir
@@ -194,7 +194,8 @@
break
i += 1
if i > 60:
- raise
+ _error_in_logs(keypath, logdir, user, computes)
+ raise Exception("Cleanup timed out")
time.sleep(1)
logging.info('killed %s' % kill_id)
state.delete_instance_state(kill_id)
diff --git a/stress/test_server_actions.py b/stress/test_server_actions.py
index 3cf3698..7080630 100644
--- a/stress/test_server_actions.py
+++ b/stress/test_server_actions.py
@@ -25,7 +25,7 @@
# local imports
import test_case
import pending_action
-from tempest.exceptions import TimeoutException
+from tempest.exceptions import TimeoutException, Duplicate
from utils.util import *
@@ -52,51 +52,35 @@
self._logger.info('no ACTIVE instances to reboot')
return
- _reboot_type = kwargs.get('type', 'SOFT')
+ _reboot_arg = kwargs.get('type', 'SOFT')
# select active vm to reboot and then send request to nova controller
target = random.choice(active_vms)
reboot_target = target[0]
+ # It seems that doing a reboot when in reboot is an error.
+ try:
+ response, body = manager.servers_client.reboot(
+ reboot_target['id'],
+ _reboot_arg)
+ except Duplicate:
+ return
- response, body = manager.servers_client.reboot(
- reboot_target['id'],
- _reboot_type)
if (response.status != 202):
self._logger.error("response: %s" % response)
raise Exception
- if _reboot_type == 'SOFT':
- state_name = 'REBOOT'
+ if _reboot_arg == 'SOFT':
+ reboot_state = 'REBOOT'
else:
- state_name = 'REBOOT' # this is a bug, should be HARD_REBOOT
+ reboot_state = 'HARD_REBOOT'
self._logger.info('waiting for machine %s to change to %s' %
- (reboot_target['id'], state_name))
-
- # check for state transition
- _resp, body = manager.servers_client.get_server(reboot_target['id'])
- if body['status'] == state_name:
- state_string = state_name
- else:
- # grab the actual state as we think it is
- temp_obj = state.get_instances()[self._target['id']]
- self._logger.debug(
- "machine %s in state %s" %
- (reboot_target['id'], temp_obj[1])
- )
- state_string = temp_obj[1]
-
- if state_string == state_name:
- self._logger.info('machine %s ACTIVE -> %s' %
- (reboot_target['id'], state_name))
- state.set_instance_state(reboot_target['id'],
- (reboot_target, state_name))
+ (reboot_target['id'], reboot_state))
return VerifyRebootVM(manager,
state,
reboot_target,
- reboot_type=_reboot_type,
- state_name=state_string)
+ reboot_state=reboot_state)
class VerifyRebootVM(pending_action.PendingAction):
@@ -104,22 +88,13 @@
States = enum('REBOOT_CHECK', 'ACTIVE_CHECK')
def __init__(self, manager, state, target_server,
- reboot_type=None,
- state_name=None,
+ reboot_state=None,
ip_addr=None):
super(VerifyRebootVM, self).__init__(manager,
state,
target_server)
- # FIX ME: this is a nova bug
- if reboot_type == 'SOFT':
- self._reboot_state = 'REBOOT'
- else:
- self._reboot_state = 'REBOOT' # should be HARD REBOOT
-
- if state_name == 'ACTIVE': # was still active, check to see if REBOOT
- self._retry_state = self.States.REBOOT_CHECK
- else: # was REBOOT, so now check for ACTIVE
- self._retry_state = self.States.ACTIVE_CHECK
+ self._reboot_state = reboot_state
+ self._retry_state = self.States.REBOOT_CHECK
def retry(self):
"""
@@ -155,8 +130,9 @@
if not self._check_for_status('ACTIVE'):
return False
target = self._target
- self._logger.info('machine %s REBOOT -> ACTIVE [%.1f secs elapsed]' %
- (target['id'], time.time() - self._start_time))
+ self._logger.info('machine %s %s -> ACTIVE [%.1f secs elapsed]' %
+ (target['id'], reboot_state,
+ time.time() - self._start_time))
self._state.set_instance_state(target['id'],
(target, 'ACTIVE'))
diff --git a/stress/test_servers.py b/stress/test_servers.py
index 47d30b5..a71bea2 100644
--- a/stress/test_servers.py
+++ b/stress/test_servers.py
@@ -214,7 +214,7 @@
target = self._target
self._logger.info('machine %s: DELETED [%.1f secs elapsed]' %
(target['id'], time.time() - self._start_time))
- self._state.delete_machine_state(target['id'])
+ self._state.delete_instance_state(target['id'])
return True
return False
diff --git a/stress/tests/create_kill.py b/stress/tests/create_kill.py
index 752f72d..8169fc4 100644
--- a/stress/tests/create_kill.py
+++ b/stress/tests/create_kill.py
@@ -32,4 +32,4 @@
sleep_time=100, # in milliseconds
seed=int(time.time()),
test_name="create and delete",
- max_vms=32)
+ )
diff --git a/stress/tests/hard_reboots.py b/stress/tests/hard_reboots.py
index f38ef6f..1b5928e 100644
--- a/stress/tests/hard_reboots.py
+++ b/stress/tests/hard_reboots.py
@@ -34,4 +34,4 @@
sleep_time=500, # in milliseconds
seed=int(time.time()),
test_name="hard reboots",
- max_vms=32)
+ )
diff --git a/stress/tests/user_script_sample.py b/stress/tests/user_script_sample.py
index e4f53c4..3a6972c 100644
--- a/stress/tests/user_script_sample.py
+++ b/stress/tests/user_script_sample.py
@@ -35,4 +35,4 @@
sleep_time=1000, # in milliseconds
seed=None,
test_name="simple create and delete",
- max_vms=10)
+ max_vms=4)
diff --git a/stress/tools/nova_destroy_all.py b/stress/tools/nova_destroy_all.py
index 1fa0487..3816557 100755
--- a/stress/tools/nova_destroy_all.py
+++ b/stress/tools/nova_destroy_all.py
@@ -21,9 +21,10 @@
# get the environment variables for credentials
identity = tempest.config.TempestConfig().identity
+compute = tempest.config.TempestConfig().compute
-nt = client.Client(identity.username, identity.password,
- identity.tenant_name, identity.auth_url)
+nt = client.Client(compute.username, compute.password,
+ compute.tenant_name, identity.auth_url)
flavor_list = nt.flavors.list()
server_list = nt.servers.list()
diff --git a/stress/tools/nova_status.py b/stress/tools/nova_status.py
index 65dfc82..c0b7893 100755
--- a/stress/tools/nova_status.py
+++ b/stress/tools/nova_status.py
@@ -21,11 +21,12 @@
# get the environment variables for credentials
identity = tempest.config.TempestConfig().identity
-print identity.username, identity.password,\
- identity.tenant_name, identity.auth_url
+compute = tempest.config.TempestConfig().compute
+print compute.username, compute.password,\
+ compute.tenant_name, identity.auth_url
-nt = client.Client(identity.username, identity.password,
- identity.tenant_name, identity.auth_url)
+nt = client.Client(compute.username, compute.password,
+ compute.tenant_name, identity.auth_url)
flavor_list = nt.flavors.list()
server_list = nt.servers.list()