Rework stress to be more UnitTest like
This patch reworks the stress test framework and
the scripts to be a bit more flexible and to be
like how unit tests are written. Instead of
writing a module that contains functions, we now
write an object that can implement a setUp and tearDown
method. By default the object's 'run' method will be
called by the driver.
The setUp method will be called prior to calling run
and tearDown will be called after run is stopped.
Also added debug logging to the cleanup mechanism.
Also added removing snapshots prior to removing
volumes during cleanup
DocImpact
Implements: blueprint stress-tests
Change-Id: I39b8a1d38f70ddda4867126b58bb7053b45654d6
diff --git a/tempest/stress/actions/create_destroy_server.py b/tempest/stress/actions/create_destroy_server.py
index 44b149f..68dc148 100644
--- a/tempest/stress/actions/create_destroy_server.py
+++ b/tempest/stress/actions/create_destroy_server.py
@@ -13,22 +13,27 @@
# limitations under the License.
from tempest.common.utils.data_utils import rand_name
+import tempest.stress.stressaction as stressaction
-def create_destroy(manager, logger):
- image = manager.config.compute.image_ref
- flavor = manager.config.compute.flavor_ref
- while True:
+class CreateDestroyServerTest(stressaction.StressAction):
+
+ def setUp(self, **kwargs):
+ self.image = self.manager.config.compute.image_ref
+ self.flavor = self.manager.config.compute.flavor_ref
+
+ def run(self):
name = rand_name("instance")
- logger.info("creating %s" % name)
- resp, server = manager.servers_client.create_server(
- name, image, flavor)
+ self.logger.info("creating %s" % name)
+ resp, server = self.manager.servers_client.create_server(
+ name, self.image, self.flavor)
server_id = server['id']
assert(resp.status == 202)
- manager.servers_client.wait_for_server_status(server_id, 'ACTIVE')
- logger.info("created %s" % server_id)
- logger.info("deleting %s" % name)
- resp, _ = manager.servers_client.delete_server(server_id)
+ self.manager.servers_client.wait_for_server_status(server_id,
+ 'ACTIVE')
+ self.logger.info("created %s" % server_id)
+ self.logger.info("deleting %s" % name)
+ resp, _ = self.manager.servers_client.delete_server(server_id)
assert(resp.status == 204)
- manager.servers_client.wait_for_server_termination(server_id)
- logger.info("deleted %s" % server_id)
+ self.manager.servers_client.wait_for_server_termination(server_id)
+ self.logger.info("deleted %s" % server_id)
diff --git a/tempest/stress/actions/volume_create_delete.py b/tempest/stress/actions/volume_create_delete.py
index e0c95b5..184f870 100644
--- a/tempest/stress/actions/volume_create_delete.py
+++ b/tempest/stress/actions/volume_create_delete.py
@@ -11,20 +11,23 @@
# limitations under the License.
from tempest.common.utils.data_utils import rand_name
+import tempest.stress.stressaction as stressaction
-def create_delete(manager, logger):
- while True:
+class CreateDeleteTest(stressaction.StressAction):
+
+ def run(self):
name = rand_name("volume")
- logger.info("creating %s" % name)
- resp, volume = manager.volumes_client.create_volume(size=1,
- display_name=name)
+ self.logger.info("creating %s" % name)
+ resp, volume = self.manager.volumes_client.\
+ create_volume(size=1, display_name=name)
assert(resp.status == 200)
- manager.volumes_client.wait_for_volume_status(volume['id'],
- 'available')
- logger.info("created %s" % volume['id'])
- logger.info("deleting %s" % name)
- resp, _ = manager.volumes_client.delete_volume(volume['id'])
+ vol_id = volume['id']
+ status = 'available'
+ self.manager.volumes_client.wait_for_volume_status(vol_id, status)
+ self.logger.info("created %s" % volume['id'])
+ self.logger.info("deleting %s" % name)
+ resp, _ = self.manager.volumes_client.delete_volume(vol_id)
assert(resp.status == 202)
- manager.volumes_client.wait_for_resource_deletion(volume['id'])
- logger.info("deleted %s" % volume['id'])
+ self.manager.volumes_client.wait_for_resource_deletion(vol_id)
+ self.logger.info("deleted %s" % vol_id)
diff --git a/tempest/stress/cleanup.py b/tempest/stress/cleanup.py
index 3b1c871..bfcf34f 100644
--- a/tempest/stress/cleanup.py
+++ b/tempest/stress/cleanup.py
@@ -19,10 +19,11 @@
from tempest import clients
-def cleanup():
+def cleanup(logger):
admin_manager = clients.AdminManager()
_, body = admin_manager.servers_client.list_servers({"all_tenants": True})
+ logger.debug("Cleanup::remove %s servers" % len(body['servers']))
for s in body['servers']:
try:
admin_manager.servers_client.delete_server(s['id'])
@@ -36,6 +37,7 @@
pass
_, keypairs = admin_manager.keypairs_client.list_keypairs()
+ logger.debug("Cleanup::remove %s keypairs" % len(keypairs))
for k in keypairs:
try:
admin_manager.keypairs_client.delete_keypair(k['name'])
@@ -43,6 +45,7 @@
pass
_, floating_ips = admin_manager.floating_ips_client.list_floating_ips()
+ logger.debug("Cleanup::remove %s floating ips" % len(floating_ips))
for f in floating_ips:
try:
admin_manager.floating_ips_client.delete_floating_ip(f['id'])
@@ -50,18 +53,43 @@
pass
_, users = admin_manager.identity_client.get_users()
+ logger.debug("Cleanup::remove %s users" % len(users))
for user in users:
if user['name'].startswith("stress_user"):
admin_manager.identity_client.delete_user(user['id'])
_, tenants = admin_manager.identity_client.list_tenants()
+ logger.debug("Cleanup::remove %s tenants" % len(tenants))
for tenant in tenants:
if tenant['name'].startswith("stress_tenant"):
admin_manager.identity_client.delete_tenant(tenant['id'])
+ # We have to delete snapshots first or
+ # volume deletion may block
+
+ _, snaps = admin_manager.snapshots_client.\
+ list_snapshots({"all_tenants": True})
+ logger.debug("Cleanup::remove %s snapshots" % len(snaps))
+ for v in snaps:
+ try:
+ admin_manager.snapshots_client.\
+ wait_for_snapshot_status(v['id'], 'available')
+ admin_manager.snapshots_client.delete_snapshot(v['id'])
+ except Exception:
+ pass
+
+ for v in snaps:
+ try:
+ admin_manager.snapshots_client.wait_for_resource_deletion(v['id'])
+ except Exception:
+ pass
+
_, vols = admin_manager.volumes_client.list_volumes({"all_tenants": True})
+ logger.debug("Cleanup::remove %s volumes" % len(vols))
for v in vols:
try:
+ admin_manager.volumes_client.\
+ wait_for_volume_status(v['id'], 'available')
admin_manager.volumes_client.delete_volume(v['id'])
except Exception:
pass
diff --git a/tempest/stress/driver.py b/tempest/stress/driver.py
index 51f159d..785da7d 100644
--- a/tempest/stress/driver.py
+++ b/tempest/stress/driver.py
@@ -93,9 +93,9 @@
return None
-def get_action_function(path):
- (module_part, _, function) = path.rpartition('.')
- return getattr(importlib.import_module(module_part), function)
+def get_action_object(path):
+ (module_part, _, obj_name) = path.rpartition('.')
+ return getattr(importlib.import_module(module_part), obj_name)
def stress_openstack(tests, duration):
@@ -130,10 +130,18 @@
manager = clients.Manager(username=username,
password="pass",
tenant_name=tenant_name)
- target = get_action_function(test['action'])
- p = multiprocessing.Process(target=target,
- args=(manager, logger),
- kwargs=test.get('kwargs', {}))
+
+ test_obj = get_action_object(test['action'])
+ test_run = test_obj(manager, logger)
+
+ kwargs = test.get('kwargs', {})
+ test_run.setUp(**dict(kwargs.iteritems()))
+
+ logger.debug("calling Target Object %s" %
+ test_run.__class__.__name__)
+ p = multiprocessing.Process(target=test_run.execute,
+ args=())
+
processes.append(p)
p.start()
end_time = time.time() + duration
@@ -149,8 +157,11 @@
if errors:
had_errors = True
break
+
for p in processes:
p.terminate()
+ p.join()
+
if not had_errors:
logger.info("cleaning up")
- cleanup.cleanup()
+ cleanup.cleanup(logger)
diff --git a/tempest/stress/etc/sample-test.json b/tempest/stress/etc/sample-test.json
index 5a0189c..494c823 100644
--- a/tempest/stress/etc/sample-test.json
+++ b/tempest/stress/etc/sample-test.json
@@ -1,4 +1,4 @@
-[{"action": "tempest.stress.actions.create_destroy_server.create_destroy",
+[{"action": "tempest.stress.actions.create_destroy_server.CreateDestroyServerTest",
"threads": 8,
"use_admin": false,
"use_isolated_tenants": false,
diff --git a/tempest/stress/etc/volume-create-delete-test.json b/tempest/stress/etc/volume-create-delete-test.json
index ed0aaeb..6325bdc 100644
--- a/tempest/stress/etc/volume-create-delete-test.json
+++ b/tempest/stress/etc/volume-create-delete-test.json
@@ -1,4 +1,4 @@
-[{"action": "tempest.stress.actions.volume_create_delete.create_delete",
+[{"action": "tempest.stress.actions.volume_create_delete.CreateDeleteTest",
"threads": 4,
"use_admin": false,
"use_isolated_tenants": false,
diff --git a/tempest/stress/stressaction.py b/tempest/stress/stressaction.py
new file mode 100644
index 0000000..f45ef17
--- /dev/null
+++ b/tempest/stress/stressaction.py
@@ -0,0 +1,62 @@
+# vim: tabstop=4 shiftwidth=4 softtabstop=4
+
+# (c) Copyright 2013 Hewlett-Packard Development Company, L.P.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+import signal
+import sys
+
+
+class StressAction(object):
+
+ def __init__(self, manager, logger):
+ self.manager = manager
+ self.logger = logger
+ self.runs = 0
+
+ def _shutdown_handler(self, signal, frame):
+ self.tearDown()
+ sys.exit(0)
+
+ def setUp(self, **kwargs):
+ """This method is called before the run method
+ to help the test initiatlize any structures.
+ kwargs contains arguments passed in from the
+ configuration json file.
+
+ setUp doesn't count against the time duration.
+ """
+ self.logger.debug("setUp")
+
+ def tearDown(self):
+ """This method is called to do any cleanup
+ after the test is complete.
+ """
+ self.logger.debug("tearDown")
+
+ def execute(self):
+ """This is the main execution entry point called
+ by the driver. We register a signal handler to
+ allow us to gracefull tearDown, and then exit.
+ We also keep track of how many runs we do.
+ """
+ signal.signal(signal.SIGHUP, self._shutdown_handler)
+ signal.signal(signal.SIGTERM, self._shutdown_handler)
+ while True:
+ self.run()
+ self.runs = self.runs + 1
+
+ def run(self):
+ """This method is where the stress test code runs."""
+ raise NotImplemented()
diff --git a/tempest/stress/tools/cleanup.py b/tempest/stress/tools/cleanup.py
index 7139d6c..b6a26cd 100755
--- a/tempest/stress/tools/cleanup.py
+++ b/tempest/stress/tools/cleanup.py
@@ -14,7 +14,15 @@
# See the License for the specific language governing permissions and
# limitations under the License.
+import logging
+
from tempest.stress import cleanup
+_console = logging.StreamHandler()
+_console.setLevel(logging.DEBUG)
+# add the handler to the root logger
+logger = logging.getLogger('tempest.stress.cleanup')
+logger.addHandler(logging.StreamHandler())
+logger.setLevel(logging.DEBUG)
-cleanup.cleanup()
+cleanup.cleanup(logger)