blob: 359d917c3b8e7d119604f02e018f9a9f47425b3c [file] [log] [blame]
koder aka kdanilovcff7b2e2015-04-18 20:48:15 +03001from __future__ import print_function
2
gstepanov023c1e42015-04-08 15:50:19 +03003import os
koder aka kdanilov7acd6bd2015-02-12 14:28:30 -08004import sys
koder aka kdanilov57ce4db2015-04-25 21:25:51 +03005import time
koder aka kdanilov2c473092015-03-29 17:12:13 +03006import Queue
koder aka kdanilov7acd6bd2015-02-12 14:28:30 -08007import pprint
koder aka kdanilov416b87a2015-05-12 00:26:04 +03008import signal
koder aka kdanilove21d7472015-02-14 19:02:04 -08009import logging
koder aka kdanilov7acd6bd2015-02-12 14:28:30 -080010import argparse
koder aka kdanilov168f6092015-04-19 02:33:38 +030011import functools
koder aka kdanilov2c473092015-03-29 17:12:13 +030012import threading
koder aka kdanilovd5ed4da2015-05-07 23:33:23 +030013import contextlib
koder aka kdanilov2c473092015-03-29 17:12:13 +030014import collections
koder aka kdanilov7acd6bd2015-02-12 14:28:30 -080015
koder aka kdanilov88407ff2015-05-26 15:35:57 +030016from yaml import load as _yaml_load
17
18try:
19 from yaml import CLoader
20 yaml_load = functools.partial(_yaml_load, Loader=CLoader)
21except ImportError:
22 yaml_load = _yaml_load
23
24
koder aka kdanilovfd2cfa52015-05-20 03:17:42 +030025import texttable
koder aka kdanilovbb5fe072015-05-21 02:50:23 +030026
27try:
28 import faulthandler
29except ImportError:
30 faulthandler = None
31
koder aka kdanilov2c473092015-03-29 17:12:13 +030032from concurrent.futures import ThreadPoolExecutor
koder aka kdanilov6c491062015-04-09 22:33:13 +030033
koder aka kdanilovcff7b2e2015-04-18 20:48:15 +030034from wally import pretty_yaml
koder aka kdanilovf86d7af2015-05-06 04:01:54 +030035from wally.hw_info import get_hw_info
36from wally.discover import discover, Node
koder aka kdanilov63ad2062015-04-27 13:11:40 +030037from wally.timeseries import SensorDatastore
koder aka kdanilovcff7b2e2015-04-18 20:48:15 +030038from wally import utils, report, ssh_utils, start_vms
koder aka kdanilov4af1c1d2015-05-18 15:48:58 +030039from wally.suits import IOPerfTest, PgBenchTest, MysqlTest
koder aka kdanilovfd2cfa52015-05-20 03:17:42 +030040from wally.config import (cfg_dict, load_config, setup_loggers,
koder aka kdanilov88407ff2015-05-26 15:35:57 +030041 get_test_files, save_run_params, load_run_params)
koder aka kdanilov4af1c1d2015-05-18 15:48:58 +030042from wally.sensors_utils import with_sensors_util, sensors_info_util
43
44TOOL_TYPE_MAPPER = {
45 "io": IOPerfTest,
46 "pgbench": PgBenchTest,
47 "mysql": MysqlTest,
48}
koder aka kdanilov63ad2062015-04-27 13:11:40 +030049
koder aka kdanilov57ce4db2015-04-25 21:25:51 +030050
51try:
52 from wally import webui
53except ImportError:
54 webui = None
koder aka kdanilov2c473092015-03-29 17:12:13 +030055
56
koder aka kdanilovcff7b2e2015-04-18 20:48:15 +030057logger = logging.getLogger("wally")
koder aka kdanilovcee43342015-04-14 22:52:53 +030058
koder aka kdanilov7acd6bd2015-02-12 14:28:30 -080059
Yulia Portnova7ddfa732015-02-24 17:32:58 +020060def format_result(res, formatter):
koder aka kdanilove21d7472015-02-14 19:02:04 -080061 data = "\n{0}\n".format("=" * 80)
62 data += pprint.pformat(res) + "\n"
63 data += "{0}\n".format("=" * 80)
koder aka kdanilovfe056622015-02-19 08:46:15 -080064 templ = "{0}\n\n====> {1}\n\n{2}\n\n"
Yulia Portnova7ddfa732015-02-24 17:32:58 +020065 return templ.format(data, formatter(res), "=" * 80)
koder aka kdanilove21d7472015-02-14 19:02:04 -080066
67
koder aka kdanilov1c2b5112015-04-10 16:53:51 +030068class Context(object):
69 def __init__(self):
70 self.build_meta = {}
71 self.nodes = []
72 self.clear_calls_stack = []
73 self.openstack_nodes_ids = []
koder aka kdanilov168f6092015-04-19 02:33:38 +030074 self.sensors_mon_q = None
koder aka kdanilovf86d7af2015-05-06 04:01:54 +030075 self.hw_info = []
koder aka kdanilov1c2b5112015-04-10 16:53:51 +030076
77
koder aka kdanilov168f6092015-04-19 02:33:38 +030078def connect_one(node, vm=False):
koder aka kdanilov0c598a12015-04-21 03:01:40 +030079 if node.conn_url == 'local':
80 node.connection = ssh_utils.connect(node.conn_url)
81 return
82
koder aka kdanilov5d589b42015-03-26 12:25:51 +020083 try:
koder aka kdanilov2c473092015-03-29 17:12:13 +030084 ssh_pref = "ssh://"
85 if node.conn_url.startswith(ssh_pref):
86 url = node.conn_url[len(ssh_pref):]
koder aka kdanilov168f6092015-04-19 02:33:38 +030087
88 if vm:
koder aka kdanilov6b1341a2015-04-21 22:44:21 +030089 conn_timeout = 240
koder aka kdanilov168f6092015-04-19 02:33:38 +030090 else:
koder aka kdanilov6b1341a2015-04-21 22:44:21 +030091 conn_timeout = 30
koder aka kdanilov168f6092015-04-19 02:33:38 +030092
93 node.connection = ssh_utils.connect(url,
koder aka kdanilov6b1341a2015-04-21 22:44:21 +030094 conn_timeout=conn_timeout)
koder aka kdanilov2c473092015-03-29 17:12:13 +030095 else:
96 raise ValueError("Unknown url type {0}".format(node.conn_url))
koder aka kdanilove87ae652015-04-20 02:14:35 +030097 except Exception as exc:
98 # logger.exception("During connect to " + node.get_conn_id())
koder aka kdanilovec1b9732015-04-23 20:43:29 +030099 msg = "During connect to {0}: {1!s}".format(node.get_conn_id(),
100 exc)
koder aka kdanilove87ae652015-04-20 02:14:35 +0300101 logger.error(msg)
koder aka kdanilov168f6092015-04-19 02:33:38 +0300102 node.connection = None
koder aka kdanilov5d589b42015-03-26 12:25:51 +0200103
104
koder aka kdanilov168f6092015-04-19 02:33:38 +0300105def connect_all(nodes, vm=False):
koder aka kdanilov2c473092015-03-29 17:12:13 +0300106 logger.info("Connecting to nodes")
107 with ThreadPoolExecutor(32) as pool:
koder aka kdanilov168f6092015-04-19 02:33:38 +0300108 connect_one_f = functools.partial(connect_one, vm=vm)
109 list(pool.map(connect_one_f, nodes))
koder aka kdanilov2c473092015-03-29 17:12:13 +0300110
111
koder aka kdanilovf86d7af2015-05-06 04:01:54 +0300112def collect_hw_info_stage(cfg, ctx):
113 if os.path.exists(cfg['hwreport_fname']):
114 msg = "{0} already exists. Skip hw info"
115 logger.info(msg.format(cfg['hwreport_fname']))
116 return
117
118 with ThreadPoolExecutor(32) as pool:
119 connections = (node.connection for node in ctx.nodes)
120 ctx.hw_info.extend(pool.map(get_hw_info, connections))
121
122 with open(cfg['hwreport_fname'], 'w') as hwfd:
123 for node, info in zip(ctx.nodes, ctx.hw_info):
124 hwfd.write("-" * 60 + "\n")
125 hwfd.write("Roles : " + ", ".join(node.roles) + "\n")
126 hwfd.write(str(info) + "\n")
127 hwfd.write("-" * 60 + "\n\n")
128
129 if info.hostname is not None:
130 fname = os.path.join(
131 cfg_dict['hwinfo_directory'],
132 info.hostname + "_lshw.xml")
133
134 with open(fname, "w") as fd:
135 fd.write(info.raw)
136 logger.info("Hardware report stored in " + cfg['hwreport_fname'])
137 logger.debug("Raw hardware info in " + cfg['hwinfo_directory'] + " folder")
138
139
koder aka kdanilov652cd802015-04-13 12:21:07 +0300140def test_thread(test, node, barrier, res_q):
koder aka kdanilov4d4771c2015-04-23 01:32:02 +0300141 exc = None
koder aka kdanilov2c473092015-03-29 17:12:13 +0300142 try:
koder aka kdanilova047e1b2015-04-21 23:16:59 +0300143 logger.debug("Run preparation for {0}".format(node.get_conn_id()))
koder aka kdanilov4d4771c2015-04-23 01:32:02 +0300144 test.pre_run()
koder aka kdanilova047e1b2015-04-21 23:16:59 +0300145 logger.debug("Run test for {0}".format(node.get_conn_id()))
koder aka kdanilov4d4771c2015-04-23 01:32:02 +0300146 test.run(barrier)
koder aka kdanilove2de58c2015-04-24 22:59:36 +0300147 except utils.StopTestError as exc:
148 pass
koder aka kdanilov652cd802015-04-13 12:21:07 +0300149 except Exception as exc:
koder aka kdanilove2de58c2015-04-24 22:59:36 +0300150 msg = "In test {0} for node {1}"
151 msg = msg.format(test, node.get_conn_id())
152 logger.exception(msg)
153 exc = utils.StopTestError(msg, exc)
koder aka kdanilov2c473092015-03-29 17:12:13 +0300154
koder aka kdanilov4500a5f2015-04-17 16:55:17 +0300155 try:
koder aka kdanilov4d4771c2015-04-23 01:32:02 +0300156 test.cleanup()
koder aka kdanilove2de58c2015-04-24 22:59:36 +0300157 except utils.StopTestError as exc1:
158 if exc is None:
159 exc = exc1
koder aka kdanilov4500a5f2015-04-17 16:55:17 +0300160 except:
161 msg = "Duringf cleanup - in test {0} for node {1}"
162 logger.exception(msg.format(test, node))
163
koder aka kdanilov4d4771c2015-04-23 01:32:02 +0300164 if exc is not None:
165 res_q.put(exc)
koder aka kdanilov2c473092015-03-29 17:12:13 +0300166
koder aka kdanilov4d4771c2015-04-23 01:32:02 +0300167
koder aka kdanilovfd2cfa52015-05-20 03:17:42 +0300168def run_single_test(test_nodes, name, test_cls, params, log_directory,
169 test_local_folder, run_uuid):
koder aka kdanilov416b87a2015-05-12 00:26:04 +0300170 logger.info("Starting {0} tests".format(name))
171 res_q = Queue.Queue()
172 threads = []
173 coord_q = Queue.Queue()
174 rem_folder = test_local_folder.format(name=name)
175
176 barrier = utils.Barrier(len(test_nodes))
177 for idx, node in enumerate(test_nodes):
178 msg = "Starting {0} test on {1} node"
179 logger.debug(msg.format(name, node.conn_url))
180
koder aka kdanilov416b87a2015-05-12 00:26:04 +0300181 params = params.copy()
182 params['testnodes_count'] = len(test_nodes)
183 test = test_cls(options=params,
184 is_primary=(idx == 0),
185 on_result_cb=res_q.put,
186 test_uuid=run_uuid,
187 node=node,
188 remote_dir=rem_folder,
koder aka kdanilovfd2cfa52015-05-20 03:17:42 +0300189 log_directory=log_directory,
koder aka kdanilov4af1c1d2015-05-18 15:48:58 +0300190 coordination_queue=coord_q,
191 total_nodes_count=len(test_nodes))
koder aka kdanilovfd2cfa52015-05-20 03:17:42 +0300192 th = threading.Thread(None, test_thread,
193 "Test:" + node.get_conn_id(),
koder aka kdanilov416b87a2015-05-12 00:26:04 +0300194 (test, node, barrier, res_q))
195 threads.append(th)
196 th.daemon = True
197 th.start()
198
koder aka kdanilovfd2cfa52015-05-20 03:17:42 +0300199 th = threading.Thread(None, test_cls.coordination_th,
200 "Coordination thread",
koder aka kdanilov416b87a2015-05-12 00:26:04 +0300201 (coord_q, barrier, len(threads)))
202 threads.append(th)
203 th.daemon = True
204 th.start()
205
206 results = []
207 coord_q.put(None)
208
209 while len(threads) != 0:
210 nthreads = []
koder aka kdanilovfd2cfa52015-05-20 03:17:42 +0300211 time.sleep(0.1)
koder aka kdanilov416b87a2015-05-12 00:26:04 +0300212
213 for th in threads:
214 if not th.is_alive():
215 th.join()
216 else:
217 nthreads.append(th)
218
219 threads = nthreads
220
221 while not res_q.empty():
222 val = res_q.get()
223
224 if isinstance(val, utils.StopTestError):
225 raise val
226
227 if isinstance(val, Exception):
228 msg = "Exception during test execution: {0!s}"
229 raise ValueError(msg.format(val))
230
231 results.append(val)
232
koder aka kdanilov416b87a2015-05-12 00:26:04 +0300233 return results
234
235
koder aka kdanilov4af1c1d2015-05-18 15:48:58 +0300236def suspend_vm_nodes(unused_nodes):
237 pausable_nodes_ids = [node.os_vm_id for node in unused_nodes
238 if node.os_vm_id is not None]
239 non_pausable = len(unused_nodes) - len(pausable_nodes_ids)
koder aka kdanilov2c473092015-03-29 17:12:13 +0300240
koder aka kdanilov4af1c1d2015-05-18 15:48:58 +0300241 if 0 != non_pausable:
242 logger.warning("Can't pause {0} nodes".format(
243 non_pausable))
244
245 if len(pausable_nodes_ids) != 0:
246 logger.debug("Try to pause {0} unused nodes".format(
247 len(pausable_nodes_ids)))
248 start_vms.pause(pausable_nodes_ids)
249
250 return pausable_nodes_ids
251
252
253def run_tests(cfg, test_block, nodes):
koder aka kdanilov2c473092015-03-29 17:12:13 +0300254 test_nodes = [node for node in nodes
255 if 'testnode' in node.roles]
koder aka kdanilov4af1c1d2015-05-18 15:48:58 +0300256
257 not_test_nodes = [node for node in nodes
258 if 'testnode' not in node.roles]
259
koder aka kdanilovd5ed4da2015-05-07 23:33:23 +0300260 if len(test_nodes) == 0:
261 logger.error("No test nodes found")
262 return
263
koder aka kdanilovcee43342015-04-14 22:52:53 +0300264 for name, params in test_block.items():
koder aka kdanilovcee43342015-04-14 22:52:53 +0300265 results = []
koder aka kdanilov416b87a2015-05-12 00:26:04 +0300266 limit = params.get('node_limit')
267 if isinstance(limit, (int, long)):
268 vm_limits = [limit]
269 elif limit is None:
270 vm_limits = [len(test_nodes)]
271 else:
272 vm_limits = limit
koder aka kdanilov652cd802015-04-13 12:21:07 +0300273
koder aka kdanilov416b87a2015-05-12 00:26:04 +0300274 for vm_count in vm_limits:
275 if vm_count == 'all':
276 curr_test_nodes = test_nodes
277 unused_nodes = []
278 else:
279 curr_test_nodes = test_nodes[:vm_count]
280 unused_nodes = test_nodes[vm_count:]
koder aka kdanilove87ae652015-04-20 02:14:35 +0300281
koder aka kdanilov416b87a2015-05-12 00:26:04 +0300282 if 0 == len(curr_test_nodes):
283 continue
koder aka kdanilov652cd802015-04-13 12:21:07 +0300284
koder aka kdanilovfd2cfa52015-05-20 03:17:42 +0300285 # make a directory for results
286 all_tests_dirs = os.listdir(cfg_dict['results'])
287
288 if 'name' in params:
289 dir_name = "{0}_{1}".format(name, params['name'])
290 else:
291 for idx in range(len(all_tests_dirs) + 1):
292 dir_name = "{0}_{1}".format(name, idx)
293 if dir_name not in all_tests_dirs:
294 break
295 else:
296 raise utils.StopTestError(
297 "Can't select directory for test results")
298
299 dir_path = os.path.join(cfg_dict['results'], dir_name)
300 if not os.path.exists(dir_path):
301 os.mkdir(dir_path)
302
koder aka kdanilov416b87a2015-05-12 00:26:04 +0300303 if cfg.get('suspend_unused_vms', True):
koder aka kdanilov4af1c1d2015-05-18 15:48:58 +0300304 pausable_nodes_ids = suspend_vm_nodes(unused_nodes)
koder aka kdanilov416b87a2015-05-12 00:26:04 +0300305
306 resumable_nodes_ids = [node.os_vm_id for node in curr_test_nodes
307 if node.os_vm_id is not None]
308
309 if len(resumable_nodes_ids) != 0:
310 logger.debug("Check and unpause {0} nodes".format(
311 len(resumable_nodes_ids)))
312 start_vms.unpause(resumable_nodes_ids)
313
koder aka kdanilov4af1c1d2015-05-18 15:48:58 +0300314 test_cls = TOOL_TYPE_MAPPER[name]
koder aka kdanilov416b87a2015-05-12 00:26:04 +0300315 try:
koder aka kdanilov4af1c1d2015-05-18 15:48:58 +0300316 sens_nodes = curr_test_nodes + not_test_nodes
317 with sensors_info_util(cfg, sens_nodes) as sensor_data:
318 t_start = time.time()
koder aka kdanilovfd2cfa52015-05-20 03:17:42 +0300319 res = run_single_test(curr_test_nodes,
320 name,
321 test_cls,
koder aka kdanilov4af1c1d2015-05-18 15:48:58 +0300322 params,
koder aka kdanilovfd2cfa52015-05-20 03:17:42 +0300323 dir_path,
koder aka kdanilov4af1c1d2015-05-18 15:48:58 +0300324 cfg['default_test_local_folder'],
325 cfg['run_uuid'])
326 t_end = time.time()
koder aka kdanilov416b87a2015-05-12 00:26:04 +0300327 finally:
328 if cfg.get('suspend_unused_vms', True):
329 if len(pausable_nodes_ids) != 0:
330 logger.debug("Unpausing {0} nodes".format(
331 len(pausable_nodes_ids)))
332 start_vms.unpause(pausable_nodes_ids)
333
koder aka kdanilov4af1c1d2015-05-18 15:48:58 +0300334 if sensor_data is not None:
335 fname = "{0}_{1}.csv".format(int(t_start), int(t_end))
336 fpath = os.path.join(cfg['sensor_storage'], fname)
337
338 with open(fpath, "w") as fd:
339 fd.write("\n\n".join(sensor_data))
340
341 results.extend(res)
koder aka kdanilov416b87a2015-05-12 00:26:04 +0300342
343 yield name, results
koder aka kdanilov2c473092015-03-29 17:12:13 +0300344
345
koder aka kdanilovda45e882015-04-06 02:24:42 +0300346def log_nodes_statistic(_, ctx):
347 nodes = ctx.nodes
koder aka kdanilov2c473092015-03-29 17:12:13 +0300348 logger.info("Found {0} nodes total".format(len(nodes)))
349 per_role = collections.defaultdict(lambda: 0)
350 for node in nodes:
351 for role in node.roles:
352 per_role[role] += 1
353
354 for role, count in sorted(per_role.items()):
355 logger.debug("Found {0} nodes with role {1}".format(count, role))
356
357
koder aka kdanilovda45e882015-04-06 02:24:42 +0300358def connect_stage(cfg, ctx):
359 ctx.clear_calls_stack.append(disconnect_stage)
360 connect_all(ctx.nodes)
361
koder aka kdanilov168f6092015-04-19 02:33:38 +0300362 all_ok = True
koder aka kdanilovda45e882015-04-06 02:24:42 +0300363
koder aka kdanilov168f6092015-04-19 02:33:38 +0300364 for node in ctx.nodes:
365 if node.connection is None:
366 if 'testnode' in node.roles:
367 msg = "Can't connect to testnode {0}"
koder aka kdanilovd5ed4da2015-05-07 23:33:23 +0300368 msg = msg.format(node.get_conn_id())
369 logger.error(msg)
370 raise utils.StopTestError(msg)
koder aka kdanilov168f6092015-04-19 02:33:38 +0300371 else:
372 msg = "Node {0} would be excluded - can't connect"
373 logger.warning(msg.format(node.get_conn_id()))
374 all_ok = False
375
376 if all_ok:
377 logger.info("All nodes connected successfully")
378
379 ctx.nodes = [node for node in ctx.nodes
380 if node.connection is not None]
koder aka kdanilovcff7b2e2015-04-18 20:48:15 +0300381
382
koder aka kdanilovda45e882015-04-06 02:24:42 +0300383def discover_stage(cfg, ctx):
koder aka kdanilov652cd802015-04-13 12:21:07 +0300384 if cfg.get('discover') is not None:
koder aka kdanilovda45e882015-04-06 02:24:42 +0300385 discover_objs = [i.strip() for i in cfg['discover'].strip().split(",")]
koder aka kdanilovcff7b2e2015-04-18 20:48:15 +0300386
koder aka kdanilovf86d7af2015-05-06 04:01:54 +0300387 nodes = discover(ctx,
388 discover_objs,
389 cfg['clouds'],
390 cfg['var_dir'],
391 not cfg['dont_discover_nodes'])
koder aka kdanilov168f6092015-04-19 02:33:38 +0300392
koder aka kdanilovcff7b2e2015-04-18 20:48:15 +0300393 ctx.nodes.extend(nodes)
koder aka kdanilovda45e882015-04-06 02:24:42 +0300394
395 for url, roles in cfg.get('explicit_nodes', {}).items():
396 ctx.nodes.append(Node(url, roles.split(",")))
397
398
koder aka kdanilovf86d7af2015-05-06 04:01:54 +0300399def save_nodes_stage(cfg, ctx):
400 cluster = {}
401 for node in ctx.nodes:
402 roles = node.roles[:]
403 if 'testnode' in roles:
404 roles.remove('testnode')
405
406 if len(roles) != 0:
407 cluster[node.conn_url] = roles
408
409 with open(cfg['nodes_report_file'], "w") as fd:
410 fd.write(pretty_yaml.dumps(cluster))
411
412
koder aka kdanilov416b87a2015-05-12 00:26:04 +0300413def reuse_vms_stage(cfg, ctx):
koder aka kdanilovfd2cfa52015-05-20 03:17:42 +0300414 p = cfg.get('clouds', {}).get('openstack', {}).get('vms', [])
koder aka kdanilov416b87a2015-05-12 00:26:04 +0300415
416 for creds in p:
417 vm_name_pattern, conn_pattern = creds.split(",")
koder aka kdanilov4af1c1d2015-05-18 15:48:58 +0300418 msg = "Vm like {0} lookup failed".format(vm_name_pattern)
419 with utils.log_error(msg):
koder aka kdanilovf86d7af2015-05-06 04:01:54 +0300420 msg = "Looking for vm with name like {0}".format(vm_name_pattern)
421 logger.debug(msg)
422
koder aka kdanilov416b87a2015-05-12 00:26:04 +0300423 if not start_vms.is_connected():
424 os_creds = get_OS_credentials(cfg, ctx)
425 else:
426 os_creds = {}
427
koder aka kdanilovf86d7af2015-05-06 04:01:54 +0300428 conn = start_vms.nova_connect(**os_creds)
koder aka kdanilovd5ed4da2015-05-07 23:33:23 +0300429 for ip, vm_id in start_vms.find_vms(conn, vm_name_pattern):
koder aka kdanilovf86d7af2015-05-06 04:01:54 +0300430 node = Node(conn_pattern.format(ip=ip), ['testnode'])
koder aka kdanilovd5ed4da2015-05-07 23:33:23 +0300431 node.os_vm_id = vm_id
koder aka kdanilovf86d7af2015-05-06 04:01:54 +0300432 ctx.nodes.append(node)
koder aka kdanilovf86d7af2015-05-06 04:01:54 +0300433
koder aka kdanilov416b87a2015-05-12 00:26:04 +0300434
435def get_creds_openrc(path):
436 fc = open(path).read()
437
438 echo = 'echo "$OS_TENANT_NAME:$OS_USERNAME:$OS_PASSWORD@$OS_AUTH_URL"'
439
koder aka kdanilov4af1c1d2015-05-18 15:48:58 +0300440 msg = "Failed to get creads from openrc file"
441 with utils.log_error(msg):
koder aka kdanilov416b87a2015-05-12 00:26:04 +0300442 data = utils.run_locally(['/bin/bash'],
443 input_data=fc + "\n" + echo)
koder aka kdanilov416b87a2015-05-12 00:26:04 +0300444
koder aka kdanilov4af1c1d2015-05-18 15:48:58 +0300445 msg = "Failed to get creads from openrc file: " + data
446 with utils.log_error(msg):
koder aka kdanilov416b87a2015-05-12 00:26:04 +0300447 data = data.strip()
448 user, tenant, passwd_auth_url = data.split(':', 2)
449 passwd, auth_url = passwd_auth_url.rsplit("@", 1)
450 assert (auth_url.startswith("https://") or
451 auth_url.startswith("http://"))
koder aka kdanilov4af1c1d2015-05-18 15:48:58 +0300452
koder aka kdanilov416b87a2015-05-12 00:26:04 +0300453 return user, passwd, tenant, auth_url
koder aka kdanilovf86d7af2015-05-06 04:01:54 +0300454
455
koder aka kdanilov416b87a2015-05-12 00:26:04 +0300456def get_OS_credentials(cfg, ctx):
koder aka kdanilovcee43342015-04-14 22:52:53 +0300457 creds = None
koder aka kdanilovfd2cfa52015-05-20 03:17:42 +0300458 tenant = None
459
koder aka kdanilov416b87a2015-05-12 00:26:04 +0300460 if 'openstack' in cfg['clouds']:
461 os_cfg = cfg['clouds']['openstack']
462 if 'OPENRC' in os_cfg:
463 logger.info("Using OS credentials from " + os_cfg['OPENRC'])
464 user, passwd, tenant, auth_url = \
465 get_creds_openrc(os_cfg['OPENRC'])
466 elif 'ENV' in os_cfg:
467 logger.info("Using OS credentials from shell environment")
468 user, passwd, tenant, auth_url = start_vms.ostack_get_creds()
koder aka kdanilovfd2cfa52015-05-20 03:17:42 +0300469 elif 'OS_TENANT_NAME' in os_cfg:
koder aka kdanilov416b87a2015-05-12 00:26:04 +0300470 logger.info("Using predefined credentials")
koder aka kdanilov1c2b5112015-04-10 16:53:51 +0300471 tenant = os_cfg['OS_TENANT_NAME'].strip()
472 user = os_cfg['OS_USERNAME'].strip()
473 passwd = os_cfg['OS_PASSWORD'].strip()
474 auth_url = os_cfg['OS_AUTH_URL'].strip()
475
koder aka kdanilovfd2cfa52015-05-20 03:17:42 +0300476 if tenant is None and 'fuel' in cfg['clouds'] and \
koder aka kdanilov88407ff2015-05-26 15:35:57 +0300477 'openstack_env' in cfg['clouds']['fuel'] and \
478 ctx.fuel_openstack_creds is not None:
koder aka kdanilov416b87a2015-05-12 00:26:04 +0300479 logger.info("Using fuel creds")
480 creds = ctx.fuel_openstack_creds
koder aka kdanilovfd2cfa52015-05-20 03:17:42 +0300481 elif tenant is None:
koder aka kdanilov416b87a2015-05-12 00:26:04 +0300482 logger.error("Can't found OS credentials")
483 raise utils.StopTestError("Can't found OS credentials", None)
koder aka kdanilov1c2b5112015-04-10 16:53:51 +0300484
koder aka kdanilovcee43342015-04-14 22:52:53 +0300485 if creds is None:
486 creds = {'name': user,
487 'passwd': passwd,
488 'tenant': tenant,
489 'auth_url': auth_url}
koder aka kdanilov1c2b5112015-04-10 16:53:51 +0300490
koder aka kdanilov46d4f392015-04-24 11:35:00 +0300491 msg = "OS_CREDS: user={name} tenant={tenant} auth_url={auth_url}"
492 logger.debug(msg.format(**creds))
koder aka kdanilovcee43342015-04-14 22:52:53 +0300493 return creds
koder aka kdanilov4e9f3ed2015-04-14 11:26:12 +0300494
koder aka kdanilov1c2b5112015-04-10 16:53:51 +0300495
koder aka kdanilov168f6092015-04-19 02:33:38 +0300496@contextlib.contextmanager
koder aka kdanilovd5ed4da2015-05-07 23:33:23 +0300497def create_vms_ctx(ctx, cfg, config, already_has_count=0):
koder aka kdanilovc368eb62015-04-28 18:22:01 +0300498 params = cfg['vm_configs'][config['cfg_name']].copy()
koder aka kdanilov168f6092015-04-19 02:33:38 +0300499 os_nodes_ids = []
500
koder aka kdanilov416b87a2015-05-12 00:26:04 +0300501 if not start_vms.is_connected():
502 os_creds = get_OS_credentials(cfg, ctx)
503 else:
504 os_creds = {}
koder aka kdanilov168f6092015-04-19 02:33:38 +0300505 start_vms.nova_connect(**os_creds)
506
koder aka kdanilovc368eb62015-04-28 18:22:01 +0300507 params.update(config)
koder aka kdanilovfd2cfa52015-05-20 03:17:42 +0300508 if 'keypair_file_private' not in params:
509 params['keypair_file_private'] = params['keypair_name'] + ".pem"
510
koder aka kdanilovc368eb62015-04-28 18:22:01 +0300511 params['group_name'] = cfg_dict['run_uuid']
512
koder aka kdanilovd5ed4da2015-05-07 23:33:23 +0300513 if not config.get('skip_preparation', False):
koder aka kdanilov416b87a2015-05-12 00:26:04 +0300514 logger.info("Preparing openstack")
koder aka kdanilovd5ed4da2015-05-07 23:33:23 +0300515 start_vms.prepare_os_subpr(params=params, **os_creds)
koder aka kdanilov168f6092015-04-19 02:33:38 +0300516
517 new_nodes = []
518 try:
koder aka kdanilovd5ed4da2015-05-07 23:33:23 +0300519 for new_node, node_id in start_vms.launch_vms(params,
520 already_has_count):
koder aka kdanilov168f6092015-04-19 02:33:38 +0300521 new_node.roles.append('testnode')
522 ctx.nodes.append(new_node)
523 os_nodes_ids.append(node_id)
524 new_nodes.append(new_node)
525
526 store_nodes_in_log(cfg, os_nodes_ids)
527 ctx.openstack_nodes_ids = os_nodes_ids
528
529 yield new_nodes
530
531 finally:
532 if not cfg['keep_vm']:
533 shut_down_vms_stage(cfg, ctx)
534
535
koder aka kdanilovcee43342015-04-14 22:52:53 +0300536def run_tests_stage(cfg, ctx):
koder aka kdanilovfd2cfa52015-05-20 03:17:42 +0300537 ctx.results = collections.defaultdict(lambda: [])
koder aka kdanilov1c2b5112015-04-10 16:53:51 +0300538
koder aka kdanilovcee43342015-04-14 22:52:53 +0300539 if 'tests' not in cfg:
540 return
gstepanov023c1e42015-04-08 15:50:19 +0300541
koder aka kdanilovcee43342015-04-14 22:52:53 +0300542 for group in cfg['tests']:
543
544 assert len(group.items()) == 1
545 key, config = group.items()[0]
546
547 if 'start_test_nodes' == key:
koder aka kdanilovc368eb62015-04-28 18:22:01 +0300548 if 'openstack' not in config:
549 msg = "No openstack block in config - can't spawn vm's"
550 logger.error(msg)
551 raise utils.StopTestError(msg)
552
koder aka kdanilovd5ed4da2015-05-07 23:33:23 +0300553 num_test_nodes = sum(1 for node in ctx.nodes
554 if 'testnode' in node.roles)
555
556 vm_ctx = create_vms_ctx(ctx, cfg, config['openstack'],
557 num_test_nodes)
558 with vm_ctx as new_nodes:
koder aka kdanilov416b87a2015-05-12 00:26:04 +0300559 if len(new_nodes) != 0:
koder aka kdanilov4af1c1d2015-05-18 15:48:58 +0300560 logger.debug("Connecting to new nodes")
koder aka kdanilov416b87a2015-05-12 00:26:04 +0300561 connect_all(new_nodes, True)
koder aka kdanilovcee43342015-04-14 22:52:53 +0300562
koder aka kdanilov416b87a2015-05-12 00:26:04 +0300563 for node in new_nodes:
564 if node.connection is None:
565 msg = "Failed to connect to vm {0}"
566 raise RuntimeError(msg.format(node.get_conn_id()))
koder aka kdanilovcee43342015-04-14 22:52:53 +0300567
koder aka kdanilov4af1c1d2015-05-18 15:48:58 +0300568 with with_sensors_util(cfg_dict, ctx.nodes):
569 for test_group in config.get('tests', []):
koder aka kdanilovfd2cfa52015-05-20 03:17:42 +0300570 for tp, res in run_tests(cfg, test_group, ctx.nodes):
571 ctx.results[tp].extend(res)
koder aka kdanilov4500a5f2015-04-17 16:55:17 +0300572 else:
koder aka kdanilov4af1c1d2015-05-18 15:48:58 +0300573 with with_sensors_util(cfg_dict, ctx.nodes):
koder aka kdanilovfd2cfa52015-05-20 03:17:42 +0300574 for tp, res in run_tests(cfg, group, ctx.nodes):
575 ctx.results[tp].extend(res)
koder aka kdanilovda45e882015-04-06 02:24:42 +0300576
gstepanov023c1e42015-04-08 15:50:19 +0300577
koder aka kdanilov1c2b5112015-04-10 16:53:51 +0300578def shut_down_vms_stage(cfg, ctx):
koder aka kdanilov66839a92015-04-11 13:22:31 +0300579 vm_ids_fname = cfg_dict['vm_ids_fname']
koder aka kdanilov1c2b5112015-04-10 16:53:51 +0300580 if ctx.openstack_nodes_ids is None:
koder aka kdanilov66839a92015-04-11 13:22:31 +0300581 nodes_ids = open(vm_ids_fname).read().split()
koder aka kdanilov1c2b5112015-04-10 16:53:51 +0300582 else:
583 nodes_ids = ctx.openstack_nodes_ids
584
koder aka kdanilov652cd802015-04-13 12:21:07 +0300585 if len(nodes_ids) != 0:
586 logger.info("Removing nodes")
587 start_vms.clear_nodes(nodes_ids)
588 logger.info("Nodes has been removed")
gstepanov023c1e42015-04-08 15:50:19 +0300589
koder aka kdanilov66839a92015-04-11 13:22:31 +0300590 if os.path.exists(vm_ids_fname):
591 os.remove(vm_ids_fname)
gstepanov023c1e42015-04-08 15:50:19 +0300592
koder aka kdanilov66839a92015-04-11 13:22:31 +0300593
594def store_nodes_in_log(cfg, nodes_ids):
595 with open(cfg['vm_ids_fname'], 'w') as fd:
596 fd.write("\n".join(nodes_ids))
gstepanov023c1e42015-04-08 15:50:19 +0300597
598
599def clear_enviroment(cfg, ctx):
koder aka kdanilov66839a92015-04-11 13:22:31 +0300600 if os.path.exists(cfg_dict['vm_ids_fname']):
koder aka kdanilov1c2b5112015-04-10 16:53:51 +0300601 shut_down_vms_stage(cfg, ctx)
gstepanov023c1e42015-04-08 15:50:19 +0300602
603
koder aka kdanilovda45e882015-04-06 02:24:42 +0300604def disconnect_stage(cfg, ctx):
koder aka kdanilov652cd802015-04-13 12:21:07 +0300605 ssh_utils.close_all_sessions()
606
koder aka kdanilovda45e882015-04-06 02:24:42 +0300607 for node in ctx.nodes:
608 if node.connection is not None:
609 node.connection.close()
610
611
koder aka kdanilov66839a92015-04-11 13:22:31 +0300612def store_raw_results_stage(cfg, ctx):
613
koder aka kdanilovfd2cfa52015-05-20 03:17:42 +0300614 raw_results = cfg_dict['raw_results']
koder aka kdanilov66839a92015-04-11 13:22:31 +0300615
616 if os.path.exists(raw_results):
koder aka kdanilov88407ff2015-05-26 15:35:57 +0300617 cont = yaml_load(open(raw_results).read())
koder aka kdanilov66839a92015-04-11 13:22:31 +0300618 else:
619 cont = []
620
koder aka kdanilovfd2cfa52015-05-20 03:17:42 +0300621 cont.extend(utils.yamable(ctx.results).items())
koder aka kdanilov66839a92015-04-11 13:22:31 +0300622 raw_data = pretty_yaml.dumps(cont)
623
624 with open(raw_results, "w") as fd:
625 fd.write(raw_data)
626
627
628def console_report_stage(cfg, ctx):
koder aka kdanilovfd2cfa52015-05-20 03:17:42 +0300629 first_report = True
630 text_rep_fname = cfg['text_report_file']
631 with open(text_rep_fname, "w") as fd:
632 for tp, data in ctx.results.items():
633 if 'io' == tp and data is not None:
634 dinfo = report.process_disk_info(data)
635 rep = IOPerfTest.format_for_console(data, dinfo)
636 elif tp in ['mysql', 'pgbench'] and data is not None:
637 rep = MysqlTest.format_for_console(data)
638 else:
639 logger.warning("Can't generate text report for " + tp)
640 continue
koder aka kdanilov416b87a2015-05-12 00:26:04 +0300641
koder aka kdanilovfd2cfa52015-05-20 03:17:42 +0300642 fd.write(rep)
643 fd.write("\n")
koder aka kdanilov416b87a2015-05-12 00:26:04 +0300644
koder aka kdanilovfd2cfa52015-05-20 03:17:42 +0300645 if first_report:
646 logger.info("Text report were stored in " + text_rep_fname)
647 first_report = False
648
koder aka kdanilov4af1c1d2015-05-18 15:48:58 +0300649 print("\n" + rep + "\n")
koder aka kdanilov416b87a2015-05-12 00:26:04 +0300650
koder aka kdanilov66839a92015-04-11 13:22:31 +0300651
koder aka kdanilov88407ff2015-05-26 15:35:57 +0300652def test_load_report_stage(cfg, ctx):
653 load_rep_fname = cfg['load_report_file']
654 found = False
655 for idx, (tp, data) in enumerate(ctx.results.items()):
656 if 'io' == tp and data is not None:
657 if found:
658 logger.error("Making reports for more than one " +
659 "io block isn't supported! All " +
660 "report, except first are skipped")
661 continue
662 found = True
663 report.make_load_report(idx, cfg['results'], load_rep_fname)
664
665
koder aka kdanilove87ae652015-04-20 02:14:35 +0300666def html_report_stage(cfg, ctx):
Yulia Portnova8ca20572015-04-14 14:09:39 +0300667 html_rep_fname = cfg['html_report_file']
koder aka kdanilovf86d7af2015-05-06 04:01:54 +0300668 found = False
koder aka kdanilovfd2cfa52015-05-20 03:17:42 +0300669 for tp, data in ctx.results.items():
koder aka kdanilovf86d7af2015-05-06 04:01:54 +0300670 if 'io' == tp and data is not None:
671 if found:
672 logger.error("Making reports for more than one " +
673 "io block isn't supported! All " +
674 "report, except first are skipped")
675 continue
676 found = True
677 dinfo = report.process_disk_info(data)
koder aka kdanilov88407ff2015-05-26 15:35:57 +0300678 report.make_io_report(dinfo,
679 cfg.get('comment', ''),
680 html_rep_fname,
koder aka kdanilovf86d7af2015-05-06 04:01:54 +0300681 lab_info=ctx.hw_info)
koder aka kdanilovcff7b2e2015-04-18 20:48:15 +0300682
koder aka kdanilovda45e882015-04-06 02:24:42 +0300683
684def complete_log_nodes_statistic(cfg, ctx):
685 nodes = ctx.nodes
686 for node in nodes:
687 logger.debug(str(node))
688
689
koder aka kdanilov88407ff2015-05-26 15:35:57 +0300690def load_data_from_file(var_dir, _, ctx):
691 raw_results = os.path.join(var_dir, 'raw_results.yaml')
692 ctx.results = {}
693 for tp, results in yaml_load(open(raw_results).read()):
694 cls = TOOL_TYPE_MAPPER[tp]
695 ctx.results[tp] = map(cls.load, results)
koder aka kdanilov4af1c1d2015-05-18 15:48:58 +0300696
koder aka kdanilov88407ff2015-05-26 15:35:57 +0300697
698def load_data_from(var_dir):
699 return functools.partial(load_data_from_file, var_dir)
gstepanovcd256d62015-04-07 17:47:32 +0300700
701
koder aka kdanilov57ce4db2015-04-25 21:25:51 +0300702def start_web_ui(cfg, ctx):
703 if webui is None:
704 logger.error("Can't start webui. Install cherrypy module")
705 ctx.web_thread = None
706 else:
707 th = threading.Thread(None, webui.web_main_thread, "webui", (None,))
708 th.daemon = True
709 th.start()
710 ctx.web_thread = th
711
712
713def stop_web_ui(cfg, ctx):
714 webui.web_main_stop()
715 time.sleep(1)
716
717
koder aka kdanilovcee43342015-04-14 22:52:53 +0300718def parse_args(argv):
koder aka kdanilovcff7b2e2015-04-18 20:48:15 +0300719 descr = "Disk io performance test suite"
720 parser = argparse.ArgumentParser(prog='wally', description=descr)
koder aka kdanilovcee43342015-04-14 22:52:53 +0300721
722 parser.add_argument("-l", dest='extra_logs',
723 action='store_true', default=False,
724 help="print some extra log info")
koder aka kdanilovcee43342015-04-14 22:52:53 +0300725 parser.add_argument("-b", '--build_description',
726 type=str, default="Build info")
727 parser.add_argument("-i", '--build_id', type=str, default="id")
728 parser.add_argument("-t", '--build_type', type=str, default="GA")
729 parser.add_argument("-u", '--username', type=str, default="admin")
koder aka kdanilove87ae652015-04-20 02:14:35 +0300730 parser.add_argument("-n", '--no-tests', action='store_true',
731 help="Don't run tests", default=False)
koder aka kdanilovcff7b2e2015-04-18 20:48:15 +0300732 parser.add_argument("-p", '--post-process-only', metavar="VAR_DIR",
733 help="Only process data from previour run")
koder aka kdanilov88407ff2015-05-26 15:35:57 +0300734 parser.add_argument("-x", '--xxx', action='store_true')
koder aka kdanilovcff7b2e2015-04-18 20:48:15 +0300735 parser.add_argument("-k", '--keep-vm', action='store_true',
736 help="Don't remove test vm's", default=False)
koder aka kdanilove87ae652015-04-20 02:14:35 +0300737 parser.add_argument("-d", '--dont-discover-nodes', action='store_true',
738 help="Don't connect/discover fuel nodes",
739 default=False)
koder aka kdanilova047e1b2015-04-21 23:16:59 +0300740 parser.add_argument("-r", '--no-html-report', action='store_true',
741 help="Skip html report", default=False)
koder aka kdanilovf86d7af2015-05-06 04:01:54 +0300742 parser.add_argument("--params", metavar="testname.paramname",
koder aka kdanilov63ad2062015-04-27 13:11:40 +0300743 help="Test params", default=[])
koder aka kdanilovfd2cfa52015-05-20 03:17:42 +0300744 parser.add_argument("--ls", action='store_true', default=False)
koder aka kdanilov88407ff2015-05-26 15:35:57 +0300745 parser.add_argument("-c", "--comment", default="")
koder aka kdanilovcff7b2e2015-04-18 20:48:15 +0300746 parser.add_argument("config_file")
koder aka kdanilovcee43342015-04-14 22:52:53 +0300747
748 return parser.parse_args(argv[1:])
749
750
koder aka kdanilov4af1c1d2015-05-18 15:48:58 +0300751def get_stage_name(func):
koder aka kdanilov7248c7b2015-05-31 22:53:03 +0300752 nm = get_func_name(func)
753 if nm.endswith("stage"):
754 return nm
koder aka kdanilov4af1c1d2015-05-18 15:48:58 +0300755 else:
koder aka kdanilov7248c7b2015-05-31 22:53:03 +0300756 return nm + " stage"
koder aka kdanilov416b87a2015-05-12 00:26:04 +0300757
758
koder aka kdanilov88407ff2015-05-26 15:35:57 +0300759def get_test_names(raw_res):
760 res = set()
761 for tp, data in raw_res:
762 for block in data:
763 res.add("{0}({1})".format(tp, block.get('test_name', '-')))
764 return res
koder aka kdanilovbb5fe072015-05-21 02:50:23 +0300765
766
koder aka kdanilovfd2cfa52015-05-20 03:17:42 +0300767def list_results(path):
768 results = []
769
770 for dname in os.listdir(path):
771
koder aka kdanilov88407ff2015-05-26 15:35:57 +0300772 files_cfg = get_test_files(os.path.join(path, dname))
koder aka kdanilovfd2cfa52015-05-20 03:17:42 +0300773
koder aka kdanilov88407ff2015-05-26 15:35:57 +0300774 if not os.path.isfile(files_cfg['raw_results']):
koder aka kdanilovfd2cfa52015-05-20 03:17:42 +0300775 continue
776
koder aka kdanilov88407ff2015-05-26 15:35:57 +0300777 mt = os.path.getmtime(files_cfg['raw_results'])
778 res_mtime = time.ctime(mt)
koder aka kdanilovfd2cfa52015-05-20 03:17:42 +0300779
koder aka kdanilov88407ff2015-05-26 15:35:57 +0300780 raw_res = yaml_load(open(files_cfg['raw_results']).read())
781 test_names = ",".join(sorted(get_test_names(raw_res)))
koder aka kdanilovfd2cfa52015-05-20 03:17:42 +0300782
koder aka kdanilov88407ff2015-05-26 15:35:57 +0300783 params = load_run_params(files_cfg['run_params_file'])
koder aka kdanilovbb5fe072015-05-21 02:50:23 +0300784
koder aka kdanilov88407ff2015-05-26 15:35:57 +0300785 comm = params.get('comment')
786 results.append((mt, dname, test_names, res_mtime,
787 '-' if comm is None else comm))
koder aka kdanilovfd2cfa52015-05-20 03:17:42 +0300788
koder aka kdanilov88407ff2015-05-26 15:35:57 +0300789 tab = texttable.Texttable(max_width=200)
koder aka kdanilovfd2cfa52015-05-20 03:17:42 +0300790 tab.set_deco(tab.HEADER | tab.VLINES | tab.BORDER)
koder aka kdanilov88407ff2015-05-26 15:35:57 +0300791 tab.set_cols_align(["l", "l", "l", "l"])
792 results.sort()
koder aka kdanilovfd2cfa52015-05-20 03:17:42 +0300793
koder aka kdanilov88407ff2015-05-26 15:35:57 +0300794 for data in results[::-1]:
795 tab.add_row(data[1:])
koder aka kdanilovfd2cfa52015-05-20 03:17:42 +0300796
koder aka kdanilov88407ff2015-05-26 15:35:57 +0300797 tab.header(["Name", "Tests", "etime", "Comment"])
koder aka kdanilovfd2cfa52015-05-20 03:17:42 +0300798
799 print(tab.draw())
800
801
koder aka kdanilov7248c7b2015-05-31 22:53:03 +0300802def get_func_name(obj):
803 if hasattr(obj, '__name__'):
804 return obj.__name__
805 if hasattr(obj, 'func_name'):
806 return obj.func_name
807 return obj.func.func_name
808
809
810@contextlib.contextmanager
811def log_stage(func):
812 msg_templ = "Exception during {0}: {1!s}"
813 msg_templ_no_exc = "During {0}"
814
815 logger.info("Start " + get_stage_name(func))
816
817 try:
818 yield
819 except utils.StopTestError as exc:
820 logger.error(msg_templ.format(
821 get_func_name(func), exc))
822 except Exception:
823 logger.exception(msg_templ_no_exc.format(
824 get_func_name(func)))
825
826
koder aka kdanilov3f356262015-02-13 08:06:14 -0800827def main(argv):
koder aka kdanilovbb5fe072015-05-21 02:50:23 +0300828 if faulthandler is not None:
829 faulthandler.register(signal.SIGUSR1, all_threads=True)
830
koder aka kdanilove06762a2015-03-22 23:32:09 +0200831 opts = parse_args(argv)
koder aka kdanilov88407ff2015-05-26 15:35:57 +0300832
833 if opts.ls:
834 list_results(opts.config_file)
835 exit(0)
836
837 data_dir = load_config(opts.config_file, opts.post_process_only)
838
839 if opts.post_process_only is None:
840 cfg_dict['comment'] = opts.comment
841 save_run_params()
koder aka kdanilov2c473092015-03-29 17:12:13 +0300842
koder aka kdanilov4af1c1d2015-05-18 15:48:58 +0300843 if cfg_dict.get('logging', {}).get("extra_logs", False) or opts.extra_logs:
844 level = logging.DEBUG
845 else:
846 level = logging.WARNING
847
848 setup_loggers(level, cfg_dict['log_file'])
849
koder aka kdanilovfd2cfa52015-05-20 03:17:42 +0300850 if not os.path.exists(cfg_dict['saved_config_file']):
851 with open(cfg_dict['saved_config_file'], 'w') as fd:
852 fd.write(open(opts.config_file).read())
853
koder aka kdanilov66839a92015-04-11 13:22:31 +0300854 if opts.post_process_only is not None:
855 stages = [
koder aka kdanilov88407ff2015-05-26 15:35:57 +0300856 load_data_from(data_dir)
koder aka kdanilov66839a92015-04-11 13:22:31 +0300857 ]
858 else:
859 stages = [
koder aka kdanilovf86d7af2015-05-06 04:01:54 +0300860 discover_stage
861 ]
862
koder aka kdanilovf86d7af2015-05-06 04:01:54 +0300863 stages.extend([
koder aka kdanilov416b87a2015-05-12 00:26:04 +0300864 reuse_vms_stage,
koder aka kdanilov66839a92015-04-11 13:22:31 +0300865 log_nodes_statistic,
koder aka kdanilovf86d7af2015-05-06 04:01:54 +0300866 save_nodes_stage,
867 connect_stage])
868
869 if cfg_dict.get('collect_info', True):
870 stages.append(collect_hw_info_stage)
871
872 stages.extend([
koder aka kdanilov4af1c1d2015-05-18 15:48:58 +0300873 # deploy_sensors_stage,
koder aka kdanilov66839a92015-04-11 13:22:31 +0300874 run_tests_stage,
koder aka kdanilov416b87a2015-05-12 00:26:04 +0300875 store_raw_results_stage,
koder aka kdanilov4af1c1d2015-05-18 15:48:58 +0300876 # gather_sensors_stage
koder aka kdanilovf86d7af2015-05-06 04:01:54 +0300877 ])
koder aka kdanilov66839a92015-04-11 13:22:31 +0300878
koder aka kdanilove87ae652015-04-20 02:14:35 +0300879 report_stages = [
880 console_report_stage,
koder aka kdanilove87ae652015-04-20 02:14:35 +0300881 ]
882
koder aka kdanilov88407ff2015-05-26 15:35:57 +0300883 if opts.xxx:
884 report_stages.append(test_load_report_stage)
885 elif not opts.no_html_report:
koder aka kdanilova047e1b2015-04-21 23:16:59 +0300886 report_stages.append(html_report_stage)
887
koder aka kdanilov652cd802015-04-13 12:21:07 +0300888 logger.info("All info would be stored into {0}".format(
889 cfg_dict['var_dir']))
gstepanovcd256d62015-04-07 17:47:32 +0300890
koder aka kdanilovda45e882015-04-06 02:24:42 +0300891 ctx = Context()
gstepanovaffcdb12015-04-07 17:18:29 +0300892 ctx.build_meta['build_id'] = opts.build_id
893 ctx.build_meta['build_descrption'] = opts.build_description
894 ctx.build_meta['build_type'] = opts.build_type
895 ctx.build_meta['username'] = opts.username
koder aka kdanilov57ce4db2015-04-25 21:25:51 +0300896 ctx.sensors_data = SensorDatastore()
koder aka kdanilove87ae652015-04-20 02:14:35 +0300897
koder aka kdanilov168f6092015-04-19 02:33:38 +0300898 cfg_dict['keep_vm'] = opts.keep_vm
koder aka kdanilove87ae652015-04-20 02:14:35 +0300899 cfg_dict['no_tests'] = opts.no_tests
900 cfg_dict['dont_discover_nodes'] = opts.dont_discover_nodes
koder aka kdanilov6c491062015-04-09 22:33:13 +0300901
koder aka kdanilov57ce4db2015-04-25 21:25:51 +0300902 if cfg_dict.get('run_web_ui', False):
903 start_web_ui(cfg_dict, ctx)
904
koder aka kdanilov7248c7b2015-05-31 22:53:03 +0300905 for stage in stages:
906 ok = False
907 with log_stage(stage):
koder aka kdanilov4af1c1d2015-05-18 15:48:58 +0300908 logger.info("Start " + get_stage_name(stage))
koder aka kdanilovda45e882015-04-06 02:24:42 +0300909 stage(cfg_dict, ctx)
koder aka kdanilov7248c7b2015-05-31 22:53:03 +0300910 ok = True
911 if not ok:
912 break
koder aka kdanilovf86d7af2015-05-06 04:01:54 +0300913
koder aka kdanilov7248c7b2015-05-31 22:53:03 +0300914 exc, cls, tb = sys.exc_info()
915 for stage in ctx.clear_calls_stack[::-1]:
916 with log_stage(stage):
917 stage(cfg_dict, ctx)
918
919 logger.debug("Start utils.cleanup")
920 for clean_func, args, kwargs in utils.iter_clean_func():
921 with log_stage(clean_func):
922 clean_func(*args, **kwargs)
koder aka kdanilov2c473092015-03-29 17:12:13 +0300923
koder aka kdanilove2de58c2015-04-24 22:59:36 +0300924 if exc is None:
925 for report_stage in report_stages:
koder aka kdanilov7248c7b2015-05-31 22:53:03 +0300926 with log_stage(report_stage):
927 report_stage(cfg_dict, ctx)
koder aka kdanilove87ae652015-04-20 02:14:35 +0300928
929 logger.info("All info stored in {0} folder".format(cfg_dict['var_dir']))
koder aka kdanilove2de58c2015-04-24 22:59:36 +0300930
koder aka kdanilov57ce4db2015-04-25 21:25:51 +0300931 if cfg_dict.get('run_web_ui', False):
932 stop_web_ui(cfg_dict, ctx)
933
koder aka kdanilove2de58c2015-04-24 22:59:36 +0300934 if exc is None:
935 logger.info("Tests finished successfully")
936 return 0
937 else:
938 logger.error("Tests are failed. See detailed error above")
939 return 1