blob: 6606d0432e048ce7940615b38a7546d83a5b7b01 [file] [log] [blame]
koder aka kdanilovcff7b2e2015-04-18 20:48:15 +03001from __future__ import print_function
2
gstepanov023c1e42015-04-08 15:50:19 +03003import os
koder aka kdanilov7acd6bd2015-02-12 14:28:30 -08004import sys
koder aka kdanilov57ce4db2015-04-25 21:25:51 +03005import time
koder aka kdanilov2c473092015-03-29 17:12:13 +03006import Queue
koder aka kdanilov7acd6bd2015-02-12 14:28:30 -08007import pprint
koder aka kdanilove21d7472015-02-14 19:02:04 -08008import logging
koder aka kdanilov7acd6bd2015-02-12 14:28:30 -08009import argparse
koder aka kdanilov168f6092015-04-19 02:33:38 +030010import functools
koder aka kdanilov2c473092015-03-29 17:12:13 +030011import threading
koder aka kdanilov168f6092015-04-19 02:33:38 +030012import contextlib
koder aka kdanilov7306c642015-04-23 15:29:45 +030013import subprocess
koder aka kdanilov2c473092015-03-29 17:12:13 +030014import collections
koder aka kdanilov7acd6bd2015-02-12 14:28:30 -080015
koder aka kdanilov66839a92015-04-11 13:22:31 +030016import yaml
koder aka kdanilov2c473092015-03-29 17:12:13 +030017from concurrent.futures import ThreadPoolExecutor
koder aka kdanilov6c491062015-04-09 22:33:13 +030018
koder aka kdanilovcff7b2e2015-04-18 20:48:15 +030019from wally import pretty_yaml
koder aka kdanilov63ad2062015-04-27 13:11:40 +030020from wally.timeseries import SensorDatastore
koder aka kdanilovcff7b2e2015-04-18 20:48:15 +030021from wally.discover import discover, Node, undiscover
22from wally import utils, report, ssh_utils, start_vms
23from wally.suits.itest import IOPerfTest, PgBenchTest
koder aka kdanilov63ad2062015-04-27 13:11:40 +030024from wally.sensors_utils import deploy_sensors_stage
koder aka kdanilovcff7b2e2015-04-18 20:48:15 +030025from wally.config import cfg_dict, load_config, setup_loggers
koder aka kdanilov63ad2062015-04-27 13:11:40 +030026
koder aka kdanilov57ce4db2015-04-25 21:25:51 +030027
28try:
29 from wally import webui
30except ImportError:
31 webui = None
koder aka kdanilov2c473092015-03-29 17:12:13 +030032
33
koder aka kdanilovcff7b2e2015-04-18 20:48:15 +030034logger = logging.getLogger("wally")
koder aka kdanilovcee43342015-04-14 22:52:53 +030035
koder aka kdanilov7acd6bd2015-02-12 14:28:30 -080036
Yulia Portnova7ddfa732015-02-24 17:32:58 +020037def format_result(res, formatter):
koder aka kdanilove21d7472015-02-14 19:02:04 -080038 data = "\n{0}\n".format("=" * 80)
39 data += pprint.pformat(res) + "\n"
40 data += "{0}\n".format("=" * 80)
koder aka kdanilovfe056622015-02-19 08:46:15 -080041 templ = "{0}\n\n====> {1}\n\n{2}\n\n"
Yulia Portnova7ddfa732015-02-24 17:32:58 +020042 return templ.format(data, formatter(res), "=" * 80)
koder aka kdanilove21d7472015-02-14 19:02:04 -080043
44
koder aka kdanilov1c2b5112015-04-10 16:53:51 +030045class Context(object):
46 def __init__(self):
47 self.build_meta = {}
48 self.nodes = []
49 self.clear_calls_stack = []
50 self.openstack_nodes_ids = []
koder aka kdanilov168f6092015-04-19 02:33:38 +030051 self.sensors_mon_q = None
koder aka kdanilov1c2b5112015-04-10 16:53:51 +030052
53
koder aka kdanilov168f6092015-04-19 02:33:38 +030054def connect_one(node, vm=False):
koder aka kdanilov0c598a12015-04-21 03:01:40 +030055 if node.conn_url == 'local':
56 node.connection = ssh_utils.connect(node.conn_url)
57 return
58
koder aka kdanilov5d589b42015-03-26 12:25:51 +020059 try:
koder aka kdanilov2c473092015-03-29 17:12:13 +030060 ssh_pref = "ssh://"
61 if node.conn_url.startswith(ssh_pref):
62 url = node.conn_url[len(ssh_pref):]
koder aka kdanilov168f6092015-04-19 02:33:38 +030063
64 if vm:
koder aka kdanilov6b1341a2015-04-21 22:44:21 +030065 conn_timeout = 240
koder aka kdanilov168f6092015-04-19 02:33:38 +030066 else:
koder aka kdanilov6b1341a2015-04-21 22:44:21 +030067 conn_timeout = 30
koder aka kdanilov168f6092015-04-19 02:33:38 +030068
69 node.connection = ssh_utils.connect(url,
koder aka kdanilov6b1341a2015-04-21 22:44:21 +030070 conn_timeout=conn_timeout)
koder aka kdanilov2c473092015-03-29 17:12:13 +030071 else:
72 raise ValueError("Unknown url type {0}".format(node.conn_url))
koder aka kdanilove87ae652015-04-20 02:14:35 +030073 except Exception as exc:
74 # logger.exception("During connect to " + node.get_conn_id())
koder aka kdanilovec1b9732015-04-23 20:43:29 +030075 msg = "During connect to {0}: {1!s}".format(node.get_conn_id(),
76 exc)
koder aka kdanilove87ae652015-04-20 02:14:35 +030077 logger.error(msg)
koder aka kdanilov168f6092015-04-19 02:33:38 +030078 node.connection = None
koder aka kdanilov5d589b42015-03-26 12:25:51 +020079
80
koder aka kdanilov168f6092015-04-19 02:33:38 +030081def connect_all(nodes, vm=False):
koder aka kdanilov2c473092015-03-29 17:12:13 +030082 logger.info("Connecting to nodes")
83 with ThreadPoolExecutor(32) as pool:
koder aka kdanilov168f6092015-04-19 02:33:38 +030084 connect_one_f = functools.partial(connect_one, vm=vm)
85 list(pool.map(connect_one_f, nodes))
koder aka kdanilov2c473092015-03-29 17:12:13 +030086
87
koder aka kdanilov652cd802015-04-13 12:21:07 +030088def test_thread(test, node, barrier, res_q):
koder aka kdanilov4d4771c2015-04-23 01:32:02 +030089 exc = None
koder aka kdanilov2c473092015-03-29 17:12:13 +030090 try:
koder aka kdanilova047e1b2015-04-21 23:16:59 +030091 logger.debug("Run preparation for {0}".format(node.get_conn_id()))
koder aka kdanilov4d4771c2015-04-23 01:32:02 +030092 test.pre_run()
koder aka kdanilova047e1b2015-04-21 23:16:59 +030093 logger.debug("Run test for {0}".format(node.get_conn_id()))
koder aka kdanilov4d4771c2015-04-23 01:32:02 +030094 test.run(barrier)
koder aka kdanilove2de58c2015-04-24 22:59:36 +030095 except utils.StopTestError as exc:
96 pass
koder aka kdanilov652cd802015-04-13 12:21:07 +030097 except Exception as exc:
koder aka kdanilove2de58c2015-04-24 22:59:36 +030098 msg = "In test {0} for node {1}"
99 msg = msg.format(test, node.get_conn_id())
100 logger.exception(msg)
101 exc = utils.StopTestError(msg, exc)
koder aka kdanilov2c473092015-03-29 17:12:13 +0300102
koder aka kdanilov4500a5f2015-04-17 16:55:17 +0300103 try:
koder aka kdanilov4d4771c2015-04-23 01:32:02 +0300104 test.cleanup()
koder aka kdanilove2de58c2015-04-24 22:59:36 +0300105 except utils.StopTestError as exc1:
106 if exc is None:
107 exc = exc1
koder aka kdanilov4500a5f2015-04-17 16:55:17 +0300108 except:
109 msg = "Duringf cleanup - in test {0} for node {1}"
110 logger.exception(msg.format(test, node))
111
koder aka kdanilov4d4771c2015-04-23 01:32:02 +0300112 if exc is not None:
113 res_q.put(exc)
koder aka kdanilov2c473092015-03-29 17:12:13 +0300114
koder aka kdanilov4d4771c2015-04-23 01:32:02 +0300115
koder aka kdanilov2066daf2015-04-23 21:05:41 +0300116def run_tests(cfg, test_block, nodes):
koder aka kdanilov2c473092015-03-29 17:12:13 +0300117 tool_type_mapper = {
118 "io": IOPerfTest,
119 "pgbench": PgBenchTest,
120 }
121
122 test_nodes = [node for node in nodes
123 if 'testnode' in node.roles]
koder aka kdanilov4500a5f2015-04-17 16:55:17 +0300124 test_number_per_type = {}
koder aka kdanilov2c473092015-03-29 17:12:13 +0300125 res_q = Queue.Queue()
126
koder aka kdanilovcee43342015-04-14 22:52:53 +0300127 for name, params in test_block.items():
128 logger.info("Starting {0} tests".format(name))
koder aka kdanilov4500a5f2015-04-17 16:55:17 +0300129 test_num = test_number_per_type.get(name, 0)
130 test_number_per_type[name] = test_num + 1
koder aka kdanilovcee43342015-04-14 22:52:53 +0300131 threads = []
132 barrier = utils.Barrier(len(test_nodes))
koder aka kdanilovec1b9732015-04-23 20:43:29 +0300133 coord_q = Queue.Queue()
134 test_cls = tool_type_mapper[name]
koder aka kdanilov2066daf2015-04-23 21:05:41 +0300135 rem_folder = cfg['default_test_local_folder'].format(name=name)
koder aka kdanilov4500a5f2015-04-17 16:55:17 +0300136
koder aka kdanilovabd6ead2015-04-24 02:03:07 +0300137 for idx, node in enumerate(test_nodes):
koder aka kdanilovcee43342015-04-14 22:52:53 +0300138 msg = "Starting {0} test on {1} node"
139 logger.debug(msg.format(name, node.conn_url))
koder aka kdanilov4500a5f2015-04-17 16:55:17 +0300140
141 dr = os.path.join(
142 cfg_dict['test_log_directory'],
143 "{0}_{1}_{2}".format(name, test_num, node.get_ip())
144 )
145
146 if not os.path.exists(dr):
147 os.makedirs(dr)
148
koder aka kdanilovabd6ead2015-04-24 02:03:07 +0300149 test = test_cls(options=params,
150 is_primary=(idx == 0),
151 on_result_cb=res_q.put,
152 test_uuid=cfg['run_uuid'],
153 node=node,
koder aka kdanilov2066daf2015-04-23 21:05:41 +0300154 remote_dir=rem_folder,
koder aka kdanilovec1b9732015-04-23 20:43:29 +0300155 log_directory=dr,
156 coordination_queue=coord_q)
koder aka kdanilovcee43342015-04-14 22:52:53 +0300157 th = threading.Thread(None, test_thread, None,
158 (test, node, barrier, res_q))
159 threads.append(th)
160 th.daemon = True
161 th.start()
koder aka kdanilov2c473092015-03-29 17:12:13 +0300162
koder aka kdanilovec1b9732015-04-23 20:43:29 +0300163 th = threading.Thread(None, test_cls.coordination_th, None,
164 (coord_q, barrier, len(threads)))
165 threads.append(th)
166 th.daemon = True
167 th.start()
168
koder aka kdanilovcee43342015-04-14 22:52:53 +0300169 def gather_results(res_q, results):
170 while not res_q.empty():
171 val = res_q.get()
koder aka kdanilov66839a92015-04-11 13:22:31 +0300172
koder aka kdanilove2de58c2015-04-24 22:59:36 +0300173 if isinstance(val, utils.StopTestError):
174 raise val
175
koder aka kdanilovcee43342015-04-14 22:52:53 +0300176 if isinstance(val, Exception):
koder aka kdanilovec1b9732015-04-23 20:43:29 +0300177 msg = "Exception during test execution: {0!s}"
178 raise ValueError(msg.format(val))
koder aka kdanilov652cd802015-04-13 12:21:07 +0300179
koder aka kdanilovcee43342015-04-14 22:52:53 +0300180 results.append(val)
koder aka kdanilov652cd802015-04-13 12:21:07 +0300181
koder aka kdanilovcee43342015-04-14 22:52:53 +0300182 results = []
koder aka kdanilov652cd802015-04-13 12:21:07 +0300183
koder aka kdanilove87ae652015-04-20 02:14:35 +0300184 # MAX_WAIT_TIME = 10
185 # end_time = time.time() + MAX_WAIT_TIME
186
187 # while time.time() < end_time:
koder aka kdanilovcee43342015-04-14 22:52:53 +0300188 while True:
189 for th in threads:
190 th.join(1)
191 gather_results(res_q, results)
koder aka kdanilove87ae652015-04-20 02:14:35 +0300192 # if time.time() > end_time:
193 # break
koder aka kdanilov652cd802015-04-13 12:21:07 +0300194
koder aka kdanilovcee43342015-04-14 22:52:53 +0300195 if all(not th.is_alive() for th in threads):
196 break
koder aka kdanilov652cd802015-04-13 12:21:07 +0300197
koder aka kdanilove87ae652015-04-20 02:14:35 +0300198 # if any(th.is_alive() for th in threads):
199 # logger.warning("Some test threads still running")
200
koder aka kdanilovcee43342015-04-14 22:52:53 +0300201 gather_results(res_q, results)
koder aka kdanilov63ad2062015-04-27 13:11:40 +0300202 result = test.merge_results(results)
203 result['__test_meta__'] = {'testnodes_count': len(test_nodes)}
204 yield name, result
koder aka kdanilov2c473092015-03-29 17:12:13 +0300205
206
koder aka kdanilovda45e882015-04-06 02:24:42 +0300207def log_nodes_statistic(_, ctx):
208 nodes = ctx.nodes
koder aka kdanilov2c473092015-03-29 17:12:13 +0300209 logger.info("Found {0} nodes total".format(len(nodes)))
210 per_role = collections.defaultdict(lambda: 0)
211 for node in nodes:
212 for role in node.roles:
213 per_role[role] += 1
214
215 for role, count in sorted(per_role.items()):
216 logger.debug("Found {0} nodes with role {1}".format(count, role))
217
218
koder aka kdanilovda45e882015-04-06 02:24:42 +0300219def connect_stage(cfg, ctx):
220 ctx.clear_calls_stack.append(disconnect_stage)
221 connect_all(ctx.nodes)
222
koder aka kdanilov168f6092015-04-19 02:33:38 +0300223 all_ok = True
koder aka kdanilovda45e882015-04-06 02:24:42 +0300224
koder aka kdanilov168f6092015-04-19 02:33:38 +0300225 for node in ctx.nodes:
226 if node.connection is None:
227 if 'testnode' in node.roles:
228 msg = "Can't connect to testnode {0}"
229 raise RuntimeError(msg.format(node.get_conn_id()))
230 else:
231 msg = "Node {0} would be excluded - can't connect"
232 logger.warning(msg.format(node.get_conn_id()))
233 all_ok = False
234
235 if all_ok:
236 logger.info("All nodes connected successfully")
237
238 ctx.nodes = [node for node in ctx.nodes
239 if node.connection is not None]
koder aka kdanilovcff7b2e2015-04-18 20:48:15 +0300240
241
koder aka kdanilovda45e882015-04-06 02:24:42 +0300242def discover_stage(cfg, ctx):
koder aka kdanilov652cd802015-04-13 12:21:07 +0300243 if cfg.get('discover') is not None:
koder aka kdanilovda45e882015-04-06 02:24:42 +0300244 discover_objs = [i.strip() for i in cfg['discover'].strip().split(",")]
koder aka kdanilovcff7b2e2015-04-18 20:48:15 +0300245
koder aka kdanilove87ae652015-04-20 02:14:35 +0300246 nodes, clean_data = discover(ctx,
247 discover_objs,
248 cfg['clouds'],
249 cfg['var_dir'],
250 not cfg['dont_discover_nodes'])
koder aka kdanilov168f6092015-04-19 02:33:38 +0300251
252 def undiscover_stage(cfg, ctx):
253 undiscover(clean_data)
254
255 ctx.clear_calls_stack.append(undiscover_stage)
koder aka kdanilovcff7b2e2015-04-18 20:48:15 +0300256 ctx.nodes.extend(nodes)
koder aka kdanilovda45e882015-04-06 02:24:42 +0300257
258 for url, roles in cfg.get('explicit_nodes', {}).items():
259 ctx.nodes.append(Node(url, roles.split(",")))
260
261
koder aka kdanilove87ae652015-04-20 02:14:35 +0300262def get_OS_credentials(cfg, ctx, creds_type):
koder aka kdanilovcee43342015-04-14 22:52:53 +0300263 creds = None
koder aka kdanilovda45e882015-04-06 02:24:42 +0300264
koder aka kdanilovcee43342015-04-14 22:52:53 +0300265 if creds_type == 'clouds':
koder aka kdanilov46d4f392015-04-24 11:35:00 +0300266 logger.info("Using OS credentials from 'cloud' section")
koder aka kdanilovcee43342015-04-14 22:52:53 +0300267 if 'openstack' in cfg['clouds']:
koder aka kdanilov1c2b5112015-04-10 16:53:51 +0300268 os_cfg = cfg['clouds']['openstack']
koder aka kdanilovcee43342015-04-14 22:52:53 +0300269
koder aka kdanilov1c2b5112015-04-10 16:53:51 +0300270 tenant = os_cfg['OS_TENANT_NAME'].strip()
271 user = os_cfg['OS_USERNAME'].strip()
272 passwd = os_cfg['OS_PASSWORD'].strip()
273 auth_url = os_cfg['OS_AUTH_URL'].strip()
274
koder aka kdanilovcee43342015-04-14 22:52:53 +0300275 elif 'fuel' in cfg['clouds'] and \
276 'openstack_env' in cfg['clouds']['fuel']:
277 creds = ctx.fuel_openstack_creds
koder aka kdanilov1c2b5112015-04-10 16:53:51 +0300278
koder aka kdanilovcee43342015-04-14 22:52:53 +0300279 elif creds_type == 'ENV':
koder aka kdanilov46d4f392015-04-24 11:35:00 +0300280 logger.info("Using OS credentials from shell environment")
koder aka kdanilovcee43342015-04-14 22:52:53 +0300281 user, passwd, tenant, auth_url = start_vms.ostack_get_creds()
282 elif os.path.isfile(creds_type):
koder aka kdanilov46d4f392015-04-24 11:35:00 +0300283 logger.info("Using OS credentials from " + creds_type)
koder aka kdanilov7306c642015-04-23 15:29:45 +0300284 fc = open(creds_type).read()
285
286 echo = 'echo "$OS_TENANT_NAME:$OS_USERNAME:$OS_PASSWORD@$OS_AUTH_URL"'
287
288 p = subprocess.Popen(['/bin/bash'], shell=False,
289 stdout=subprocess.PIPE,
290 stdin=subprocess.PIPE,
291 stderr=subprocess.STDOUT)
292 p.stdin.write(fc + "\n" + echo)
293 p.stdin.close()
294 code = p.wait()
295 data = p.stdout.read().strip()
296
297 if code != 0:
298 msg = "Failed to get creads from openrc file: " + data
299 logger.error(msg)
300 raise RuntimeError(msg)
301
302 try:
303 user, tenant, passwd_auth_url = data.split(':', 2)
304 passwd, auth_url = passwd_auth_url.rsplit("@", 1)
305 assert (auth_url.startswith("https://") or
306 auth_url.startswith("http://"))
307 except Exception:
308 msg = "Failed to get creads from openrc file: " + data
309 logger.exception(msg)
310 raise
311
koder aka kdanilovcee43342015-04-14 22:52:53 +0300312 else:
313 msg = "Creds {0!r} isn't supported".format(creds_type)
314 raise ValueError(msg)
koder aka kdanilov1c2b5112015-04-10 16:53:51 +0300315
koder aka kdanilovcee43342015-04-14 22:52:53 +0300316 if creds is None:
317 creds = {'name': user,
318 'passwd': passwd,
319 'tenant': tenant,
320 'auth_url': auth_url}
koder aka kdanilov1c2b5112015-04-10 16:53:51 +0300321
koder aka kdanilov46d4f392015-04-24 11:35:00 +0300322 msg = "OS_CREDS: user={name} tenant={tenant} auth_url={auth_url}"
323 logger.debug(msg.format(**creds))
koder aka kdanilovcee43342015-04-14 22:52:53 +0300324 return creds
koder aka kdanilov4e9f3ed2015-04-14 11:26:12 +0300325
koder aka kdanilov1c2b5112015-04-10 16:53:51 +0300326
koder aka kdanilov168f6092015-04-19 02:33:38 +0300327@contextlib.contextmanager
328def create_vms_ctx(ctx, cfg, config):
koder aka kdanilovc368eb62015-04-28 18:22:01 +0300329 params = cfg['vm_configs'][config['cfg_name']].copy()
koder aka kdanilov168f6092015-04-19 02:33:38 +0300330 os_nodes_ids = []
331
332 os_creds_type = config['creds']
koder aka kdanilove87ae652015-04-20 02:14:35 +0300333 os_creds = get_OS_credentials(cfg, ctx, os_creds_type)
koder aka kdanilov168f6092015-04-19 02:33:38 +0300334
335 start_vms.nova_connect(**os_creds)
336
337 logger.info("Preparing openstack")
koder aka kdanilovc368eb62015-04-28 18:22:01 +0300338 params.update(config)
339 params['keypair_file_private'] = params['keypair_name'] + ".pem"
340 params['group_name'] = cfg_dict['run_uuid']
341
342 start_vms.prepare_os_subpr(params=params, **os_creds)
koder aka kdanilov168f6092015-04-19 02:33:38 +0300343
344 new_nodes = []
345 try:
koder aka kdanilov168f6092015-04-19 02:33:38 +0300346 for new_node, node_id in start_vms.launch_vms(params):
347 new_node.roles.append('testnode')
348 ctx.nodes.append(new_node)
349 os_nodes_ids.append(node_id)
350 new_nodes.append(new_node)
351
352 store_nodes_in_log(cfg, os_nodes_ids)
353 ctx.openstack_nodes_ids = os_nodes_ids
354
355 yield new_nodes
356
357 finally:
358 if not cfg['keep_vm']:
359 shut_down_vms_stage(cfg, ctx)
360
361
koder aka kdanilovcee43342015-04-14 22:52:53 +0300362def run_tests_stage(cfg, ctx):
363 ctx.results = []
koder aka kdanilov1c2b5112015-04-10 16:53:51 +0300364
koder aka kdanilovcee43342015-04-14 22:52:53 +0300365 if 'tests' not in cfg:
366 return
gstepanov023c1e42015-04-08 15:50:19 +0300367
koder aka kdanilovcee43342015-04-14 22:52:53 +0300368 for group in cfg['tests']:
369
370 assert len(group.items()) == 1
371 key, config = group.items()[0]
372
373 if 'start_test_nodes' == key:
koder aka kdanilovc368eb62015-04-28 18:22:01 +0300374 if 'openstack' not in config:
375 msg = "No openstack block in config - can't spawn vm's"
376 logger.error(msg)
377 raise utils.StopTestError(msg)
378
379 with create_vms_ctx(ctx, cfg, config['openstack']) as new_nodes:
koder aka kdanilov168f6092015-04-19 02:33:38 +0300380 connect_all(new_nodes, True)
koder aka kdanilovcee43342015-04-14 22:52:53 +0300381
koder aka kdanilov168f6092015-04-19 02:33:38 +0300382 for node in new_nodes:
383 if node.connection is None:
384 msg = "Failed to connect to vm {0}"
385 raise RuntimeError(msg.format(node.get_conn_id()))
koder aka kdanilovcee43342015-04-14 22:52:53 +0300386
koder aka kdanilov168f6092015-04-19 02:33:38 +0300387 deploy_sensors_stage(cfg_dict,
388 ctx,
389 nodes=new_nodes,
390 undeploy=False)
koder aka kdanilov12ae0632015-04-15 01:13:43 +0300391
koder aka kdanilove87ae652015-04-20 02:14:35 +0300392 if not cfg['no_tests']:
393 for test_group in config.get('tests', []):
koder aka kdanilov2066daf2015-04-23 21:05:41 +0300394 test_res = run_tests(cfg, test_group, ctx.nodes)
koder aka kdanilov4d4771c2015-04-23 01:32:02 +0300395 ctx.results.extend(test_res)
koder aka kdanilov4500a5f2015-04-17 16:55:17 +0300396 else:
koder aka kdanilove87ae652015-04-20 02:14:35 +0300397 if not cfg['no_tests']:
koder aka kdanilov2066daf2015-04-23 21:05:41 +0300398 test_res = run_tests(cfg, group, ctx.nodes)
koder aka kdanilov4d4771c2015-04-23 01:32:02 +0300399 ctx.results.extend(test_res)
koder aka kdanilovda45e882015-04-06 02:24:42 +0300400
gstepanov023c1e42015-04-08 15:50:19 +0300401
koder aka kdanilov1c2b5112015-04-10 16:53:51 +0300402def shut_down_vms_stage(cfg, ctx):
koder aka kdanilov66839a92015-04-11 13:22:31 +0300403 vm_ids_fname = cfg_dict['vm_ids_fname']
koder aka kdanilov1c2b5112015-04-10 16:53:51 +0300404 if ctx.openstack_nodes_ids is None:
koder aka kdanilov66839a92015-04-11 13:22:31 +0300405 nodes_ids = open(vm_ids_fname).read().split()
koder aka kdanilov1c2b5112015-04-10 16:53:51 +0300406 else:
407 nodes_ids = ctx.openstack_nodes_ids
408
koder aka kdanilov652cd802015-04-13 12:21:07 +0300409 if len(nodes_ids) != 0:
410 logger.info("Removing nodes")
411 start_vms.clear_nodes(nodes_ids)
412 logger.info("Nodes has been removed")
gstepanov023c1e42015-04-08 15:50:19 +0300413
koder aka kdanilov66839a92015-04-11 13:22:31 +0300414 if os.path.exists(vm_ids_fname):
415 os.remove(vm_ids_fname)
gstepanov023c1e42015-04-08 15:50:19 +0300416
koder aka kdanilov66839a92015-04-11 13:22:31 +0300417
418def store_nodes_in_log(cfg, nodes_ids):
419 with open(cfg['vm_ids_fname'], 'w') as fd:
420 fd.write("\n".join(nodes_ids))
gstepanov023c1e42015-04-08 15:50:19 +0300421
422
423def clear_enviroment(cfg, ctx):
koder aka kdanilov66839a92015-04-11 13:22:31 +0300424 if os.path.exists(cfg_dict['vm_ids_fname']):
koder aka kdanilov1c2b5112015-04-10 16:53:51 +0300425 shut_down_vms_stage(cfg, ctx)
gstepanov023c1e42015-04-08 15:50:19 +0300426
427
koder aka kdanilovda45e882015-04-06 02:24:42 +0300428def disconnect_stage(cfg, ctx):
koder aka kdanilov652cd802015-04-13 12:21:07 +0300429 ssh_utils.close_all_sessions()
430
koder aka kdanilovda45e882015-04-06 02:24:42 +0300431 for node in ctx.nodes:
432 if node.connection is not None:
433 node.connection.close()
434
435
koder aka kdanilov66839a92015-04-11 13:22:31 +0300436def store_raw_results_stage(cfg, ctx):
437
438 raw_results = os.path.join(cfg_dict['var_dir'], 'raw_results.yaml')
439
440 if os.path.exists(raw_results):
441 cont = yaml.load(open(raw_results).read())
442 else:
443 cont = []
444
koder aka kdanilov168f6092015-04-19 02:33:38 +0300445 cont.extend(utils.yamable(ctx.results))
koder aka kdanilov66839a92015-04-11 13:22:31 +0300446 raw_data = pretty_yaml.dumps(cont)
447
448 with open(raw_results, "w") as fd:
449 fd.write(raw_data)
450
451
452def console_report_stage(cfg, ctx):
453 for tp, data in ctx.results:
koder aka kdanilov4500a5f2015-04-17 16:55:17 +0300454 if 'io' == tp and data is not None:
koder aka kdanilove87ae652015-04-20 02:14:35 +0300455 print("\n")
koder aka kdanilovcff7b2e2015-04-18 20:48:15 +0300456 print(IOPerfTest.format_for_console(data))
koder aka kdanilove87ae652015-04-20 02:14:35 +0300457 print("\n")
koder aka kdanilov66839a92015-04-11 13:22:31 +0300458
459
koder aka kdanilove87ae652015-04-20 02:14:35 +0300460def html_report_stage(cfg, ctx):
Yulia Portnova8ca20572015-04-14 14:09:39 +0300461 html_rep_fname = cfg['html_report_file']
koder aka kdanilovcff7b2e2015-04-18 20:48:15 +0300462
463 try:
464 fuel_url = cfg['clouds']['fuel']['url']
465 except KeyError:
466 fuel_url = None
467
468 try:
469 creds = cfg['clouds']['fuel']['creds']
470 except KeyError:
471 creds = None
472
gstepanov69339ac2015-04-16 20:09:33 +0300473 report.make_io_report(ctx.results, html_rep_fname, fuel_url, creds=creds)
koder aka kdanilov652cd802015-04-13 12:21:07 +0300474
koder aka kdanilov652cd802015-04-13 12:21:07 +0300475 text_rep_fname = cfg_dict['text_report_file']
476 with open(text_rep_fname, "w") as fd:
477 for tp, data in ctx.results:
koder aka kdanilov4500a5f2015-04-17 16:55:17 +0300478 if 'io' == tp and data is not None:
koder aka kdanilovcff7b2e2015-04-18 20:48:15 +0300479 fd.write(IOPerfTest.format_for_console(data))
koder aka kdanilov652cd802015-04-13 12:21:07 +0300480 fd.write("\n")
481 fd.flush()
482
483 logger.info("Text report were stored in " + text_rep_fname)
koder aka kdanilovda45e882015-04-06 02:24:42 +0300484
485
486def complete_log_nodes_statistic(cfg, ctx):
487 nodes = ctx.nodes
488 for node in nodes:
489 logger.debug(str(node))
490
491
koder aka kdanilov66839a92015-04-11 13:22:31 +0300492def load_data_from(var_dir):
koder aka kdanilov4e9f3ed2015-04-14 11:26:12 +0300493 def load_data_from_file(cfg, ctx):
koder aka kdanilov66839a92015-04-11 13:22:31 +0300494 raw_results = os.path.join(var_dir, 'raw_results.yaml')
495 ctx.results = yaml.load(open(raw_results).read())
koder aka kdanilov4e9f3ed2015-04-14 11:26:12 +0300496 return load_data_from_file
gstepanovcd256d62015-04-07 17:47:32 +0300497
498
koder aka kdanilov57ce4db2015-04-25 21:25:51 +0300499def start_web_ui(cfg, ctx):
500 if webui is None:
501 logger.error("Can't start webui. Install cherrypy module")
502 ctx.web_thread = None
503 else:
504 th = threading.Thread(None, webui.web_main_thread, "webui", (None,))
505 th.daemon = True
506 th.start()
507 ctx.web_thread = th
508
509
510def stop_web_ui(cfg, ctx):
511 webui.web_main_stop()
512 time.sleep(1)
513
514
koder aka kdanilovcee43342015-04-14 22:52:53 +0300515def parse_args(argv):
koder aka kdanilovcff7b2e2015-04-18 20:48:15 +0300516 descr = "Disk io performance test suite"
517 parser = argparse.ArgumentParser(prog='wally', description=descr)
koder aka kdanilovcee43342015-04-14 22:52:53 +0300518
519 parser.add_argument("-l", dest='extra_logs',
520 action='store_true', default=False,
521 help="print some extra log info")
koder aka kdanilovcee43342015-04-14 22:52:53 +0300522 parser.add_argument("-b", '--build_description',
523 type=str, default="Build info")
524 parser.add_argument("-i", '--build_id', type=str, default="id")
525 parser.add_argument("-t", '--build_type', type=str, default="GA")
526 parser.add_argument("-u", '--username', type=str, default="admin")
koder aka kdanilove87ae652015-04-20 02:14:35 +0300527 parser.add_argument("-n", '--no-tests', action='store_true',
528 help="Don't run tests", default=False)
koder aka kdanilovcff7b2e2015-04-18 20:48:15 +0300529 parser.add_argument("-p", '--post-process-only', metavar="VAR_DIR",
530 help="Only process data from previour run")
531 parser.add_argument("-k", '--keep-vm', action='store_true',
532 help="Don't remove test vm's", default=False)
koder aka kdanilove87ae652015-04-20 02:14:35 +0300533 parser.add_argument("-d", '--dont-discover-nodes', action='store_true',
534 help="Don't connect/discover fuel nodes",
535 default=False)
koder aka kdanilova047e1b2015-04-21 23:16:59 +0300536 parser.add_argument("-r", '--no-html-report', action='store_true',
537 help="Skip html report", default=False)
koder aka kdanilov63ad2062015-04-27 13:11:40 +0300538 parser.add_argument("--params", nargs="*", metavar="testname.paramname",
539 help="Test params", default=[])
koder aka kdanilovcff7b2e2015-04-18 20:48:15 +0300540 parser.add_argument("config_file")
koder aka kdanilovcee43342015-04-14 22:52:53 +0300541
542 return parser.parse_args(argv[1:])
543
544
koder aka kdanilov3f356262015-02-13 08:06:14 -0800545def main(argv):
koder aka kdanilove06762a2015-03-22 23:32:09 +0200546 opts = parse_args(argv)
koder aka kdanilov2c473092015-03-29 17:12:13 +0300547
koder aka kdanilov66839a92015-04-11 13:22:31 +0300548 if opts.post_process_only is not None:
549 stages = [
koder aka kdanilove87ae652015-04-20 02:14:35 +0300550 load_data_from(opts.post_process_only)
koder aka kdanilov66839a92015-04-11 13:22:31 +0300551 ]
552 else:
553 stages = [
554 discover_stage,
555 log_nodes_statistic,
koder aka kdanilovcff7b2e2015-04-18 20:48:15 +0300556 connect_stage,
koder aka kdanilov66839a92015-04-11 13:22:31 +0300557 deploy_sensors_stage,
558 run_tests_stage,
koder aka kdanilove87ae652015-04-20 02:14:35 +0300559 store_raw_results_stage
koder aka kdanilov66839a92015-04-11 13:22:31 +0300560 ]
561
koder aka kdanilove87ae652015-04-20 02:14:35 +0300562 report_stages = [
563 console_report_stage,
koder aka kdanilove87ae652015-04-20 02:14:35 +0300564 ]
565
koder aka kdanilova047e1b2015-04-21 23:16:59 +0300566 if not opts.no_html_report:
567 report_stages.append(html_report_stage)
568
koder aka kdanilovcee43342015-04-14 22:52:53 +0300569 load_config(opts.config_file, opts.post_process_only)
koder aka kdanilovf4b82c22015-04-11 13:35:25 +0300570
koder aka kdanilovcff7b2e2015-04-18 20:48:15 +0300571 if cfg_dict.get('logging', {}).get("extra_logs", False) or opts.extra_logs:
572 level = logging.DEBUG
573 else:
574 level = logging.WARNING
575
576 setup_loggers(level, cfg_dict['log_file'])
koder aka kdanilovf4b82c22015-04-11 13:35:25 +0300577
koder aka kdanilov652cd802015-04-13 12:21:07 +0300578 logger.info("All info would be stored into {0}".format(
579 cfg_dict['var_dir']))
gstepanovcd256d62015-04-07 17:47:32 +0300580
koder aka kdanilovda45e882015-04-06 02:24:42 +0300581 ctx = Context()
gstepanovaffcdb12015-04-07 17:18:29 +0300582 ctx.build_meta['build_id'] = opts.build_id
583 ctx.build_meta['build_descrption'] = opts.build_description
584 ctx.build_meta['build_type'] = opts.build_type
585 ctx.build_meta['username'] = opts.username
koder aka kdanilov57ce4db2015-04-25 21:25:51 +0300586 ctx.sensors_data = SensorDatastore()
koder aka kdanilove87ae652015-04-20 02:14:35 +0300587
koder aka kdanilov168f6092015-04-19 02:33:38 +0300588 cfg_dict['keep_vm'] = opts.keep_vm
koder aka kdanilove87ae652015-04-20 02:14:35 +0300589 cfg_dict['no_tests'] = opts.no_tests
590 cfg_dict['dont_discover_nodes'] = opts.dont_discover_nodes
koder aka kdanilov6c491062015-04-09 22:33:13 +0300591
koder aka kdanilov57ce4db2015-04-25 21:25:51 +0300592 if cfg_dict.get('run_web_ui', False):
593 start_web_ui(cfg_dict, ctx)
594
koder aka kdanilovc368eb62015-04-28 18:22:01 +0300595 msg_templ = "Exception during {0.__name__}: {1!s}"
596 msg_templ_no_exc = "During {0.__name__}"
597
koder aka kdanilovda45e882015-04-06 02:24:42 +0300598 try:
599 for stage in stages:
600 logger.info("Start {0.__name__} stage".format(stage))
601 stage(cfg_dict, ctx)
koder aka kdanilovc368eb62015-04-28 18:22:01 +0300602 except utils.StopTestError as exc:
603 logger.error(msg_templ.format(stage, exc))
604 except Exception:
605 logger.exception(msg_templ_no_exc.format(stage))
koder aka kdanilovda45e882015-04-06 02:24:42 +0300606 finally:
607 exc, cls, tb = sys.exc_info()
608 for stage in ctx.clear_calls_stack[::-1]:
609 try:
610 logger.info("Start {0.__name__} stage".format(stage))
611 stage(cfg_dict, ctx)
koder aka kdanilove2de58c2015-04-24 22:59:36 +0300612 except utils.StopTestError as exc:
koder aka kdanilovc368eb62015-04-28 18:22:01 +0300613 logger.error(msg_templ.format(stage, exc))
614 except Exception:
615 logger.exception(msg_templ_no_exc.format(stage))
koder aka kdanilov2c473092015-03-29 17:12:13 +0300616
koder aka kdanilove2de58c2015-04-24 22:59:36 +0300617 if exc is None:
618 for report_stage in report_stages:
619 report_stage(cfg_dict, ctx)
koder aka kdanilove87ae652015-04-20 02:14:35 +0300620
621 logger.info("All info stored in {0} folder".format(cfg_dict['var_dir']))
koder aka kdanilove2de58c2015-04-24 22:59:36 +0300622
koder aka kdanilov57ce4db2015-04-25 21:25:51 +0300623 if cfg_dict.get('run_web_ui', False):
624 stop_web_ui(cfg_dict, ctx)
625
koder aka kdanilove2de58c2015-04-24 22:59:36 +0300626 if exc is None:
627 logger.info("Tests finished successfully")
628 return 0
629 else:
630 logger.error("Tests are failed. See detailed error above")
631 return 1