koder aka kdanilov | cff7b2e | 2015-04-18 20:48:15 +0300 | [diff] [blame] | 1 | from __future__ import print_function |
| 2 | |
gstepanov | 023c1e4 | 2015-04-08 15:50:19 +0300 | [diff] [blame] | 3 | import os |
koder aka kdanilov | 7acd6bd | 2015-02-12 14:28:30 -0800 | [diff] [blame] | 4 | import sys |
koder aka kdanilov | 57ce4db | 2015-04-25 21:25:51 +0300 | [diff] [blame] | 5 | import time |
koder aka kdanilov | 2c47309 | 2015-03-29 17:12:13 +0300 | [diff] [blame] | 6 | import Queue |
koder aka kdanilov | 7acd6bd | 2015-02-12 14:28:30 -0800 | [diff] [blame] | 7 | import pprint |
koder aka kdanilov | 416b87a | 2015-05-12 00:26:04 +0300 | [diff] [blame] | 8 | import signal |
koder aka kdanilov | e21d747 | 2015-02-14 19:02:04 -0800 | [diff] [blame] | 9 | import logging |
koder aka kdanilov | 7acd6bd | 2015-02-12 14:28:30 -0800 | [diff] [blame] | 10 | import argparse |
koder aka kdanilov | 168f609 | 2015-04-19 02:33:38 +0300 | [diff] [blame] | 11 | import functools |
koder aka kdanilov | 2c47309 | 2015-03-29 17:12:13 +0300 | [diff] [blame] | 12 | import threading |
koder aka kdanilov | d5ed4da | 2015-05-07 23:33:23 +0300 | [diff] [blame] | 13 | import contextlib |
koder aka kdanilov | 2c47309 | 2015-03-29 17:12:13 +0300 | [diff] [blame] | 14 | import collections |
koder aka kdanilov | 7acd6bd | 2015-02-12 14:28:30 -0800 | [diff] [blame] | 15 | |
koder aka kdanilov | 88407ff | 2015-05-26 15:35:57 +0300 | [diff] [blame] | 16 | from yaml import load as _yaml_load |
| 17 | |
| 18 | try: |
| 19 | from yaml import CLoader |
| 20 | yaml_load = functools.partial(_yaml_load, Loader=CLoader) |
| 21 | except ImportError: |
| 22 | yaml_load = _yaml_load |
| 23 | |
| 24 | |
koder aka kdanilov | fd2cfa5 | 2015-05-20 03:17:42 +0300 | [diff] [blame] | 25 | import texttable |
koder aka kdanilov | bb5fe07 | 2015-05-21 02:50:23 +0300 | [diff] [blame] | 26 | |
| 27 | try: |
| 28 | import faulthandler |
| 29 | except ImportError: |
| 30 | faulthandler = None |
| 31 | |
koder aka kdanilov | 2c47309 | 2015-03-29 17:12:13 +0300 | [diff] [blame] | 32 | from concurrent.futures import ThreadPoolExecutor |
koder aka kdanilov | 6c49106 | 2015-04-09 22:33:13 +0300 | [diff] [blame] | 33 | |
koder aka kdanilov | cff7b2e | 2015-04-18 20:48:15 +0300 | [diff] [blame] | 34 | from wally import pretty_yaml |
koder aka kdanilov | f86d7af | 2015-05-06 04:01:54 +0300 | [diff] [blame] | 35 | from wally.hw_info import get_hw_info |
| 36 | from wally.discover import discover, Node |
koder aka kdanilov | 63ad206 | 2015-04-27 13:11:40 +0300 | [diff] [blame] | 37 | from wally.timeseries import SensorDatastore |
koder aka kdanilov | cff7b2e | 2015-04-18 20:48:15 +0300 | [diff] [blame] | 38 | from wally import utils, report, ssh_utils, start_vms |
koder aka kdanilov | 4af1c1d | 2015-05-18 15:48:58 +0300 | [diff] [blame] | 39 | from wally.suits import IOPerfTest, PgBenchTest, MysqlTest |
koder aka kdanilov | fd2cfa5 | 2015-05-20 03:17:42 +0300 | [diff] [blame] | 40 | from wally.config import (cfg_dict, load_config, setup_loggers, |
koder aka kdanilov | 88407ff | 2015-05-26 15:35:57 +0300 | [diff] [blame] | 41 | get_test_files, save_run_params, load_run_params) |
koder aka kdanilov | 4af1c1d | 2015-05-18 15:48:58 +0300 | [diff] [blame] | 42 | from wally.sensors_utils import with_sensors_util, sensors_info_util |
| 43 | |
| 44 | TOOL_TYPE_MAPPER = { |
| 45 | "io": IOPerfTest, |
| 46 | "pgbench": PgBenchTest, |
| 47 | "mysql": MysqlTest, |
| 48 | } |
koder aka kdanilov | 63ad206 | 2015-04-27 13:11:40 +0300 | [diff] [blame] | 49 | |
koder aka kdanilov | 57ce4db | 2015-04-25 21:25:51 +0300 | [diff] [blame] | 50 | |
| 51 | try: |
| 52 | from wally import webui |
| 53 | except ImportError: |
| 54 | webui = None |
koder aka kdanilov | 2c47309 | 2015-03-29 17:12:13 +0300 | [diff] [blame] | 55 | |
| 56 | |
koder aka kdanilov | cff7b2e | 2015-04-18 20:48:15 +0300 | [diff] [blame] | 57 | logger = logging.getLogger("wally") |
koder aka kdanilov | cee4334 | 2015-04-14 22:52:53 +0300 | [diff] [blame] | 58 | |
koder aka kdanilov | 7acd6bd | 2015-02-12 14:28:30 -0800 | [diff] [blame] | 59 | |
Yulia Portnova | 7ddfa73 | 2015-02-24 17:32:58 +0200 | [diff] [blame] | 60 | def format_result(res, formatter): |
koder aka kdanilov | e21d747 | 2015-02-14 19:02:04 -0800 | [diff] [blame] | 61 | data = "\n{0}\n".format("=" * 80) |
| 62 | data += pprint.pformat(res) + "\n" |
| 63 | data += "{0}\n".format("=" * 80) |
koder aka kdanilov | fe05662 | 2015-02-19 08:46:15 -0800 | [diff] [blame] | 64 | templ = "{0}\n\n====> {1}\n\n{2}\n\n" |
Yulia Portnova | 7ddfa73 | 2015-02-24 17:32:58 +0200 | [diff] [blame] | 65 | return templ.format(data, formatter(res), "=" * 80) |
koder aka kdanilov | e21d747 | 2015-02-14 19:02:04 -0800 | [diff] [blame] | 66 | |
| 67 | |
koder aka kdanilov | 1c2b511 | 2015-04-10 16:53:51 +0300 | [diff] [blame] | 68 | class Context(object): |
| 69 | def __init__(self): |
| 70 | self.build_meta = {} |
| 71 | self.nodes = [] |
| 72 | self.clear_calls_stack = [] |
| 73 | self.openstack_nodes_ids = [] |
koder aka kdanilov | 168f609 | 2015-04-19 02:33:38 +0300 | [diff] [blame] | 74 | self.sensors_mon_q = None |
koder aka kdanilov | f86d7af | 2015-05-06 04:01:54 +0300 | [diff] [blame] | 75 | self.hw_info = [] |
koder aka kdanilov | 1c2b511 | 2015-04-10 16:53:51 +0300 | [diff] [blame] | 76 | |
| 77 | |
koder aka kdanilov | 168f609 | 2015-04-19 02:33:38 +0300 | [diff] [blame] | 78 | def connect_one(node, vm=False): |
koder aka kdanilov | 0c598a1 | 2015-04-21 03:01:40 +0300 | [diff] [blame] | 79 | if node.conn_url == 'local': |
| 80 | node.connection = ssh_utils.connect(node.conn_url) |
| 81 | return |
| 82 | |
koder aka kdanilov | 5d589b4 | 2015-03-26 12:25:51 +0200 | [diff] [blame] | 83 | try: |
koder aka kdanilov | 2c47309 | 2015-03-29 17:12:13 +0300 | [diff] [blame] | 84 | ssh_pref = "ssh://" |
| 85 | if node.conn_url.startswith(ssh_pref): |
| 86 | url = node.conn_url[len(ssh_pref):] |
koder aka kdanilov | 168f609 | 2015-04-19 02:33:38 +0300 | [diff] [blame] | 87 | |
| 88 | if vm: |
koder aka kdanilov | 6b1341a | 2015-04-21 22:44:21 +0300 | [diff] [blame] | 89 | conn_timeout = 240 |
koder aka kdanilov | 168f609 | 2015-04-19 02:33:38 +0300 | [diff] [blame] | 90 | else: |
koder aka kdanilov | 6b1341a | 2015-04-21 22:44:21 +0300 | [diff] [blame] | 91 | conn_timeout = 30 |
koder aka kdanilov | 168f609 | 2015-04-19 02:33:38 +0300 | [diff] [blame] | 92 | |
| 93 | node.connection = ssh_utils.connect(url, |
koder aka kdanilov | 6b1341a | 2015-04-21 22:44:21 +0300 | [diff] [blame] | 94 | conn_timeout=conn_timeout) |
koder aka kdanilov | 2c47309 | 2015-03-29 17:12:13 +0300 | [diff] [blame] | 95 | else: |
| 96 | raise ValueError("Unknown url type {0}".format(node.conn_url)) |
koder aka kdanilov | e87ae65 | 2015-04-20 02:14:35 +0300 | [diff] [blame] | 97 | except Exception as exc: |
| 98 | # logger.exception("During connect to " + node.get_conn_id()) |
koder aka kdanilov | ec1b973 | 2015-04-23 20:43:29 +0300 | [diff] [blame] | 99 | msg = "During connect to {0}: {1!s}".format(node.get_conn_id(), |
| 100 | exc) |
koder aka kdanilov | e87ae65 | 2015-04-20 02:14:35 +0300 | [diff] [blame] | 101 | logger.error(msg) |
koder aka kdanilov | 168f609 | 2015-04-19 02:33:38 +0300 | [diff] [blame] | 102 | node.connection = None |
koder aka kdanilov | 5d589b4 | 2015-03-26 12:25:51 +0200 | [diff] [blame] | 103 | |
| 104 | |
koder aka kdanilov | 168f609 | 2015-04-19 02:33:38 +0300 | [diff] [blame] | 105 | def connect_all(nodes, vm=False): |
koder aka kdanilov | 2c47309 | 2015-03-29 17:12:13 +0300 | [diff] [blame] | 106 | logger.info("Connecting to nodes") |
| 107 | with ThreadPoolExecutor(32) as pool: |
koder aka kdanilov | 168f609 | 2015-04-19 02:33:38 +0300 | [diff] [blame] | 108 | connect_one_f = functools.partial(connect_one, vm=vm) |
| 109 | list(pool.map(connect_one_f, nodes)) |
koder aka kdanilov | 2c47309 | 2015-03-29 17:12:13 +0300 | [diff] [blame] | 110 | |
| 111 | |
koder aka kdanilov | f86d7af | 2015-05-06 04:01:54 +0300 | [diff] [blame] | 112 | def collect_hw_info_stage(cfg, ctx): |
| 113 | if os.path.exists(cfg['hwreport_fname']): |
| 114 | msg = "{0} already exists. Skip hw info" |
| 115 | logger.info(msg.format(cfg['hwreport_fname'])) |
| 116 | return |
| 117 | |
| 118 | with ThreadPoolExecutor(32) as pool: |
| 119 | connections = (node.connection for node in ctx.nodes) |
| 120 | ctx.hw_info.extend(pool.map(get_hw_info, connections)) |
| 121 | |
| 122 | with open(cfg['hwreport_fname'], 'w') as hwfd: |
| 123 | for node, info in zip(ctx.nodes, ctx.hw_info): |
| 124 | hwfd.write("-" * 60 + "\n") |
| 125 | hwfd.write("Roles : " + ", ".join(node.roles) + "\n") |
| 126 | hwfd.write(str(info) + "\n") |
| 127 | hwfd.write("-" * 60 + "\n\n") |
| 128 | |
| 129 | if info.hostname is not None: |
| 130 | fname = os.path.join( |
| 131 | cfg_dict['hwinfo_directory'], |
| 132 | info.hostname + "_lshw.xml") |
| 133 | |
| 134 | with open(fname, "w") as fd: |
| 135 | fd.write(info.raw) |
| 136 | logger.info("Hardware report stored in " + cfg['hwreport_fname']) |
| 137 | logger.debug("Raw hardware info in " + cfg['hwinfo_directory'] + " folder") |
| 138 | |
| 139 | |
koder aka kdanilov | 652cd80 | 2015-04-13 12:21:07 +0300 | [diff] [blame] | 140 | def test_thread(test, node, barrier, res_q): |
koder aka kdanilov | 4d4771c | 2015-04-23 01:32:02 +0300 | [diff] [blame] | 141 | exc = None |
koder aka kdanilov | 2c47309 | 2015-03-29 17:12:13 +0300 | [diff] [blame] | 142 | try: |
koder aka kdanilov | a047e1b | 2015-04-21 23:16:59 +0300 | [diff] [blame] | 143 | logger.debug("Run preparation for {0}".format(node.get_conn_id())) |
koder aka kdanilov | 4d4771c | 2015-04-23 01:32:02 +0300 | [diff] [blame] | 144 | test.pre_run() |
koder aka kdanilov | a047e1b | 2015-04-21 23:16:59 +0300 | [diff] [blame] | 145 | logger.debug("Run test for {0}".format(node.get_conn_id())) |
koder aka kdanilov | 4d4771c | 2015-04-23 01:32:02 +0300 | [diff] [blame] | 146 | test.run(barrier) |
koder aka kdanilov | e2de58c | 2015-04-24 22:59:36 +0300 | [diff] [blame] | 147 | except utils.StopTestError as exc: |
| 148 | pass |
koder aka kdanilov | 652cd80 | 2015-04-13 12:21:07 +0300 | [diff] [blame] | 149 | except Exception as exc: |
koder aka kdanilov | e2de58c | 2015-04-24 22:59:36 +0300 | [diff] [blame] | 150 | msg = "In test {0} for node {1}" |
| 151 | msg = msg.format(test, node.get_conn_id()) |
| 152 | logger.exception(msg) |
| 153 | exc = utils.StopTestError(msg, exc) |
koder aka kdanilov | 2c47309 | 2015-03-29 17:12:13 +0300 | [diff] [blame] | 154 | |
koder aka kdanilov | 4500a5f | 2015-04-17 16:55:17 +0300 | [diff] [blame] | 155 | try: |
koder aka kdanilov | 4d4771c | 2015-04-23 01:32:02 +0300 | [diff] [blame] | 156 | test.cleanup() |
koder aka kdanilov | e2de58c | 2015-04-24 22:59:36 +0300 | [diff] [blame] | 157 | except utils.StopTestError as exc1: |
| 158 | if exc is None: |
| 159 | exc = exc1 |
koder aka kdanilov | 4500a5f | 2015-04-17 16:55:17 +0300 | [diff] [blame] | 160 | except: |
| 161 | msg = "Duringf cleanup - in test {0} for node {1}" |
| 162 | logger.exception(msg.format(test, node)) |
| 163 | |
koder aka kdanilov | 4d4771c | 2015-04-23 01:32:02 +0300 | [diff] [blame] | 164 | if exc is not None: |
| 165 | res_q.put(exc) |
koder aka kdanilov | 2c47309 | 2015-03-29 17:12:13 +0300 | [diff] [blame] | 166 | |
koder aka kdanilov | 4d4771c | 2015-04-23 01:32:02 +0300 | [diff] [blame] | 167 | |
koder aka kdanilov | fd2cfa5 | 2015-05-20 03:17:42 +0300 | [diff] [blame] | 168 | def run_single_test(test_nodes, name, test_cls, params, log_directory, |
| 169 | test_local_folder, run_uuid): |
koder aka kdanilov | 416b87a | 2015-05-12 00:26:04 +0300 | [diff] [blame] | 170 | logger.info("Starting {0} tests".format(name)) |
| 171 | res_q = Queue.Queue() |
| 172 | threads = [] |
| 173 | coord_q = Queue.Queue() |
| 174 | rem_folder = test_local_folder.format(name=name) |
| 175 | |
| 176 | barrier = utils.Barrier(len(test_nodes)) |
| 177 | for idx, node in enumerate(test_nodes): |
| 178 | msg = "Starting {0} test on {1} node" |
| 179 | logger.debug(msg.format(name, node.conn_url)) |
| 180 | |
koder aka kdanilov | 416b87a | 2015-05-12 00:26:04 +0300 | [diff] [blame] | 181 | params = params.copy() |
| 182 | params['testnodes_count'] = len(test_nodes) |
| 183 | test = test_cls(options=params, |
| 184 | is_primary=(idx == 0), |
| 185 | on_result_cb=res_q.put, |
| 186 | test_uuid=run_uuid, |
| 187 | node=node, |
| 188 | remote_dir=rem_folder, |
koder aka kdanilov | fd2cfa5 | 2015-05-20 03:17:42 +0300 | [diff] [blame] | 189 | log_directory=log_directory, |
koder aka kdanilov | 4af1c1d | 2015-05-18 15:48:58 +0300 | [diff] [blame] | 190 | coordination_queue=coord_q, |
| 191 | total_nodes_count=len(test_nodes)) |
koder aka kdanilov | fd2cfa5 | 2015-05-20 03:17:42 +0300 | [diff] [blame] | 192 | th = threading.Thread(None, test_thread, |
| 193 | "Test:" + node.get_conn_id(), |
koder aka kdanilov | 416b87a | 2015-05-12 00:26:04 +0300 | [diff] [blame] | 194 | (test, node, barrier, res_q)) |
| 195 | threads.append(th) |
| 196 | th.daemon = True |
| 197 | th.start() |
| 198 | |
koder aka kdanilov | fd2cfa5 | 2015-05-20 03:17:42 +0300 | [diff] [blame] | 199 | th = threading.Thread(None, test_cls.coordination_th, |
| 200 | "Coordination thread", |
koder aka kdanilov | 416b87a | 2015-05-12 00:26:04 +0300 | [diff] [blame] | 201 | (coord_q, barrier, len(threads))) |
| 202 | threads.append(th) |
| 203 | th.daemon = True |
| 204 | th.start() |
| 205 | |
| 206 | results = [] |
| 207 | coord_q.put(None) |
| 208 | |
| 209 | while len(threads) != 0: |
| 210 | nthreads = [] |
koder aka kdanilov | fd2cfa5 | 2015-05-20 03:17:42 +0300 | [diff] [blame] | 211 | time.sleep(0.1) |
koder aka kdanilov | 416b87a | 2015-05-12 00:26:04 +0300 | [diff] [blame] | 212 | |
| 213 | for th in threads: |
| 214 | if not th.is_alive(): |
| 215 | th.join() |
| 216 | else: |
| 217 | nthreads.append(th) |
| 218 | |
| 219 | threads = nthreads |
| 220 | |
| 221 | while not res_q.empty(): |
| 222 | val = res_q.get() |
| 223 | |
| 224 | if isinstance(val, utils.StopTestError): |
| 225 | raise val |
| 226 | |
| 227 | if isinstance(val, Exception): |
| 228 | msg = "Exception during test execution: {0!s}" |
| 229 | raise ValueError(msg.format(val)) |
| 230 | |
| 231 | results.append(val) |
| 232 | |
koder aka kdanilov | 416b87a | 2015-05-12 00:26:04 +0300 | [diff] [blame] | 233 | return results |
| 234 | |
| 235 | |
koder aka kdanilov | 4af1c1d | 2015-05-18 15:48:58 +0300 | [diff] [blame] | 236 | def suspend_vm_nodes(unused_nodes): |
| 237 | pausable_nodes_ids = [node.os_vm_id for node in unused_nodes |
| 238 | if node.os_vm_id is not None] |
| 239 | non_pausable = len(unused_nodes) - len(pausable_nodes_ids) |
koder aka kdanilov | 2c47309 | 2015-03-29 17:12:13 +0300 | [diff] [blame] | 240 | |
koder aka kdanilov | 4af1c1d | 2015-05-18 15:48:58 +0300 | [diff] [blame] | 241 | if 0 != non_pausable: |
| 242 | logger.warning("Can't pause {0} nodes".format( |
| 243 | non_pausable)) |
| 244 | |
| 245 | if len(pausable_nodes_ids) != 0: |
| 246 | logger.debug("Try to pause {0} unused nodes".format( |
| 247 | len(pausable_nodes_ids))) |
| 248 | start_vms.pause(pausable_nodes_ids) |
| 249 | |
| 250 | return pausable_nodes_ids |
| 251 | |
| 252 | |
| 253 | def run_tests(cfg, test_block, nodes): |
koder aka kdanilov | 2c47309 | 2015-03-29 17:12:13 +0300 | [diff] [blame] | 254 | test_nodes = [node for node in nodes |
| 255 | if 'testnode' in node.roles] |
koder aka kdanilov | 4af1c1d | 2015-05-18 15:48:58 +0300 | [diff] [blame] | 256 | |
| 257 | not_test_nodes = [node for node in nodes |
| 258 | if 'testnode' not in node.roles] |
| 259 | |
koder aka kdanilov | d5ed4da | 2015-05-07 23:33:23 +0300 | [diff] [blame] | 260 | if len(test_nodes) == 0: |
| 261 | logger.error("No test nodes found") |
| 262 | return |
| 263 | |
koder aka kdanilov | cee4334 | 2015-04-14 22:52:53 +0300 | [diff] [blame] | 264 | for name, params in test_block.items(): |
koder aka kdanilov | cee4334 | 2015-04-14 22:52:53 +0300 | [diff] [blame] | 265 | results = [] |
koder aka kdanilov | 416b87a | 2015-05-12 00:26:04 +0300 | [diff] [blame] | 266 | limit = params.get('node_limit') |
| 267 | if isinstance(limit, (int, long)): |
| 268 | vm_limits = [limit] |
| 269 | elif limit is None: |
| 270 | vm_limits = [len(test_nodes)] |
| 271 | else: |
| 272 | vm_limits = limit |
koder aka kdanilov | 652cd80 | 2015-04-13 12:21:07 +0300 | [diff] [blame] | 273 | |
koder aka kdanilov | 416b87a | 2015-05-12 00:26:04 +0300 | [diff] [blame] | 274 | for vm_count in vm_limits: |
| 275 | if vm_count == 'all': |
| 276 | curr_test_nodes = test_nodes |
| 277 | unused_nodes = [] |
| 278 | else: |
| 279 | curr_test_nodes = test_nodes[:vm_count] |
| 280 | unused_nodes = test_nodes[vm_count:] |
koder aka kdanilov | e87ae65 | 2015-04-20 02:14:35 +0300 | [diff] [blame] | 281 | |
koder aka kdanilov | 416b87a | 2015-05-12 00:26:04 +0300 | [diff] [blame] | 282 | if 0 == len(curr_test_nodes): |
| 283 | continue |
koder aka kdanilov | 652cd80 | 2015-04-13 12:21:07 +0300 | [diff] [blame] | 284 | |
koder aka kdanilov | fd2cfa5 | 2015-05-20 03:17:42 +0300 | [diff] [blame] | 285 | # make a directory for results |
| 286 | all_tests_dirs = os.listdir(cfg_dict['results']) |
| 287 | |
| 288 | if 'name' in params: |
| 289 | dir_name = "{0}_{1}".format(name, params['name']) |
| 290 | else: |
| 291 | for idx in range(len(all_tests_dirs) + 1): |
| 292 | dir_name = "{0}_{1}".format(name, idx) |
| 293 | if dir_name not in all_tests_dirs: |
| 294 | break |
| 295 | else: |
| 296 | raise utils.StopTestError( |
| 297 | "Can't select directory for test results") |
| 298 | |
| 299 | dir_path = os.path.join(cfg_dict['results'], dir_name) |
| 300 | if not os.path.exists(dir_path): |
| 301 | os.mkdir(dir_path) |
| 302 | |
koder aka kdanilov | 416b87a | 2015-05-12 00:26:04 +0300 | [diff] [blame] | 303 | if cfg.get('suspend_unused_vms', True): |
koder aka kdanilov | 4af1c1d | 2015-05-18 15:48:58 +0300 | [diff] [blame] | 304 | pausable_nodes_ids = suspend_vm_nodes(unused_nodes) |
koder aka kdanilov | 416b87a | 2015-05-12 00:26:04 +0300 | [diff] [blame] | 305 | |
| 306 | resumable_nodes_ids = [node.os_vm_id for node in curr_test_nodes |
| 307 | if node.os_vm_id is not None] |
| 308 | |
| 309 | if len(resumable_nodes_ids) != 0: |
| 310 | logger.debug("Check and unpause {0} nodes".format( |
| 311 | len(resumable_nodes_ids))) |
| 312 | start_vms.unpause(resumable_nodes_ids) |
| 313 | |
koder aka kdanilov | 4af1c1d | 2015-05-18 15:48:58 +0300 | [diff] [blame] | 314 | test_cls = TOOL_TYPE_MAPPER[name] |
koder aka kdanilov | 416b87a | 2015-05-12 00:26:04 +0300 | [diff] [blame] | 315 | try: |
koder aka kdanilov | 4af1c1d | 2015-05-18 15:48:58 +0300 | [diff] [blame] | 316 | sens_nodes = curr_test_nodes + not_test_nodes |
| 317 | with sensors_info_util(cfg, sens_nodes) as sensor_data: |
| 318 | t_start = time.time() |
koder aka kdanilov | fd2cfa5 | 2015-05-20 03:17:42 +0300 | [diff] [blame] | 319 | res = run_single_test(curr_test_nodes, |
| 320 | name, |
| 321 | test_cls, |
koder aka kdanilov | 4af1c1d | 2015-05-18 15:48:58 +0300 | [diff] [blame] | 322 | params, |
koder aka kdanilov | fd2cfa5 | 2015-05-20 03:17:42 +0300 | [diff] [blame] | 323 | dir_path, |
koder aka kdanilov | 4af1c1d | 2015-05-18 15:48:58 +0300 | [diff] [blame] | 324 | cfg['default_test_local_folder'], |
| 325 | cfg['run_uuid']) |
| 326 | t_end = time.time() |
koder aka kdanilov | 416b87a | 2015-05-12 00:26:04 +0300 | [diff] [blame] | 327 | finally: |
| 328 | if cfg.get('suspend_unused_vms', True): |
| 329 | if len(pausable_nodes_ids) != 0: |
| 330 | logger.debug("Unpausing {0} nodes".format( |
| 331 | len(pausable_nodes_ids))) |
| 332 | start_vms.unpause(pausable_nodes_ids) |
| 333 | |
koder aka kdanilov | 4af1c1d | 2015-05-18 15:48:58 +0300 | [diff] [blame] | 334 | if sensor_data is not None: |
| 335 | fname = "{0}_{1}.csv".format(int(t_start), int(t_end)) |
| 336 | fpath = os.path.join(cfg['sensor_storage'], fname) |
| 337 | |
| 338 | with open(fpath, "w") as fd: |
| 339 | fd.write("\n\n".join(sensor_data)) |
| 340 | |
| 341 | results.extend(res) |
koder aka kdanilov | 416b87a | 2015-05-12 00:26:04 +0300 | [diff] [blame] | 342 | |
| 343 | yield name, results |
koder aka kdanilov | 2c47309 | 2015-03-29 17:12:13 +0300 | [diff] [blame] | 344 | |
| 345 | |
koder aka kdanilov | da45e88 | 2015-04-06 02:24:42 +0300 | [diff] [blame] | 346 | def log_nodes_statistic(_, ctx): |
| 347 | nodes = ctx.nodes |
koder aka kdanilov | 2c47309 | 2015-03-29 17:12:13 +0300 | [diff] [blame] | 348 | logger.info("Found {0} nodes total".format(len(nodes))) |
| 349 | per_role = collections.defaultdict(lambda: 0) |
| 350 | for node in nodes: |
| 351 | for role in node.roles: |
| 352 | per_role[role] += 1 |
| 353 | |
| 354 | for role, count in sorted(per_role.items()): |
| 355 | logger.debug("Found {0} nodes with role {1}".format(count, role)) |
| 356 | |
| 357 | |
koder aka kdanilov | da45e88 | 2015-04-06 02:24:42 +0300 | [diff] [blame] | 358 | def connect_stage(cfg, ctx): |
| 359 | ctx.clear_calls_stack.append(disconnect_stage) |
| 360 | connect_all(ctx.nodes) |
| 361 | |
koder aka kdanilov | 168f609 | 2015-04-19 02:33:38 +0300 | [diff] [blame] | 362 | all_ok = True |
koder aka kdanilov | da45e88 | 2015-04-06 02:24:42 +0300 | [diff] [blame] | 363 | |
koder aka kdanilov | 168f609 | 2015-04-19 02:33:38 +0300 | [diff] [blame] | 364 | for node in ctx.nodes: |
| 365 | if node.connection is None: |
| 366 | if 'testnode' in node.roles: |
| 367 | msg = "Can't connect to testnode {0}" |
koder aka kdanilov | d5ed4da | 2015-05-07 23:33:23 +0300 | [diff] [blame] | 368 | msg = msg.format(node.get_conn_id()) |
| 369 | logger.error(msg) |
| 370 | raise utils.StopTestError(msg) |
koder aka kdanilov | 168f609 | 2015-04-19 02:33:38 +0300 | [diff] [blame] | 371 | else: |
| 372 | msg = "Node {0} would be excluded - can't connect" |
| 373 | logger.warning(msg.format(node.get_conn_id())) |
| 374 | all_ok = False |
| 375 | |
| 376 | if all_ok: |
| 377 | logger.info("All nodes connected successfully") |
| 378 | |
| 379 | ctx.nodes = [node for node in ctx.nodes |
| 380 | if node.connection is not None] |
koder aka kdanilov | cff7b2e | 2015-04-18 20:48:15 +0300 | [diff] [blame] | 381 | |
| 382 | |
koder aka kdanilov | da45e88 | 2015-04-06 02:24:42 +0300 | [diff] [blame] | 383 | def discover_stage(cfg, ctx): |
koder aka kdanilov | 652cd80 | 2015-04-13 12:21:07 +0300 | [diff] [blame] | 384 | if cfg.get('discover') is not None: |
koder aka kdanilov | da45e88 | 2015-04-06 02:24:42 +0300 | [diff] [blame] | 385 | discover_objs = [i.strip() for i in cfg['discover'].strip().split(",")] |
koder aka kdanilov | cff7b2e | 2015-04-18 20:48:15 +0300 | [diff] [blame] | 386 | |
koder aka kdanilov | f86d7af | 2015-05-06 04:01:54 +0300 | [diff] [blame] | 387 | nodes = discover(ctx, |
| 388 | discover_objs, |
| 389 | cfg['clouds'], |
| 390 | cfg['var_dir'], |
| 391 | not cfg['dont_discover_nodes']) |
koder aka kdanilov | 168f609 | 2015-04-19 02:33:38 +0300 | [diff] [blame] | 392 | |
koder aka kdanilov | cff7b2e | 2015-04-18 20:48:15 +0300 | [diff] [blame] | 393 | ctx.nodes.extend(nodes) |
koder aka kdanilov | da45e88 | 2015-04-06 02:24:42 +0300 | [diff] [blame] | 394 | |
| 395 | for url, roles in cfg.get('explicit_nodes', {}).items(): |
| 396 | ctx.nodes.append(Node(url, roles.split(","))) |
| 397 | |
| 398 | |
koder aka kdanilov | f86d7af | 2015-05-06 04:01:54 +0300 | [diff] [blame] | 399 | def save_nodes_stage(cfg, ctx): |
| 400 | cluster = {} |
| 401 | for node in ctx.nodes: |
| 402 | roles = node.roles[:] |
| 403 | if 'testnode' in roles: |
| 404 | roles.remove('testnode') |
| 405 | |
| 406 | if len(roles) != 0: |
| 407 | cluster[node.conn_url] = roles |
| 408 | |
| 409 | with open(cfg['nodes_report_file'], "w") as fd: |
| 410 | fd.write(pretty_yaml.dumps(cluster)) |
| 411 | |
| 412 | |
koder aka kdanilov | 416b87a | 2015-05-12 00:26:04 +0300 | [diff] [blame] | 413 | def reuse_vms_stage(cfg, ctx): |
koder aka kdanilov | fd2cfa5 | 2015-05-20 03:17:42 +0300 | [diff] [blame] | 414 | p = cfg.get('clouds', {}).get('openstack', {}).get('vms', []) |
koder aka kdanilov | 416b87a | 2015-05-12 00:26:04 +0300 | [diff] [blame] | 415 | |
| 416 | for creds in p: |
| 417 | vm_name_pattern, conn_pattern = creds.split(",") |
koder aka kdanilov | 4af1c1d | 2015-05-18 15:48:58 +0300 | [diff] [blame] | 418 | msg = "Vm like {0} lookup failed".format(vm_name_pattern) |
| 419 | with utils.log_error(msg): |
koder aka kdanilov | f86d7af | 2015-05-06 04:01:54 +0300 | [diff] [blame] | 420 | msg = "Looking for vm with name like {0}".format(vm_name_pattern) |
| 421 | logger.debug(msg) |
| 422 | |
koder aka kdanilov | 416b87a | 2015-05-12 00:26:04 +0300 | [diff] [blame] | 423 | if not start_vms.is_connected(): |
| 424 | os_creds = get_OS_credentials(cfg, ctx) |
| 425 | else: |
| 426 | os_creds = {} |
| 427 | |
koder aka kdanilov | f86d7af | 2015-05-06 04:01:54 +0300 | [diff] [blame] | 428 | conn = start_vms.nova_connect(**os_creds) |
koder aka kdanilov | d5ed4da | 2015-05-07 23:33:23 +0300 | [diff] [blame] | 429 | for ip, vm_id in start_vms.find_vms(conn, vm_name_pattern): |
koder aka kdanilov | f86d7af | 2015-05-06 04:01:54 +0300 | [diff] [blame] | 430 | node = Node(conn_pattern.format(ip=ip), ['testnode']) |
koder aka kdanilov | d5ed4da | 2015-05-07 23:33:23 +0300 | [diff] [blame] | 431 | node.os_vm_id = vm_id |
koder aka kdanilov | f86d7af | 2015-05-06 04:01:54 +0300 | [diff] [blame] | 432 | ctx.nodes.append(node) |
koder aka kdanilov | f86d7af | 2015-05-06 04:01:54 +0300 | [diff] [blame] | 433 | |
koder aka kdanilov | 416b87a | 2015-05-12 00:26:04 +0300 | [diff] [blame] | 434 | |
| 435 | def get_creds_openrc(path): |
| 436 | fc = open(path).read() |
| 437 | |
| 438 | echo = 'echo "$OS_TENANT_NAME:$OS_USERNAME:$OS_PASSWORD@$OS_AUTH_URL"' |
| 439 | |
koder aka kdanilov | 4af1c1d | 2015-05-18 15:48:58 +0300 | [diff] [blame] | 440 | msg = "Failed to get creads from openrc file" |
| 441 | with utils.log_error(msg): |
koder aka kdanilov | 416b87a | 2015-05-12 00:26:04 +0300 | [diff] [blame] | 442 | data = utils.run_locally(['/bin/bash'], |
| 443 | input_data=fc + "\n" + echo) |
koder aka kdanilov | 416b87a | 2015-05-12 00:26:04 +0300 | [diff] [blame] | 444 | |
koder aka kdanilov | 4af1c1d | 2015-05-18 15:48:58 +0300 | [diff] [blame] | 445 | msg = "Failed to get creads from openrc file: " + data |
| 446 | with utils.log_error(msg): |
koder aka kdanilov | 416b87a | 2015-05-12 00:26:04 +0300 | [diff] [blame] | 447 | data = data.strip() |
| 448 | user, tenant, passwd_auth_url = data.split(':', 2) |
| 449 | passwd, auth_url = passwd_auth_url.rsplit("@", 1) |
| 450 | assert (auth_url.startswith("https://") or |
| 451 | auth_url.startswith("http://")) |
koder aka kdanilov | 4af1c1d | 2015-05-18 15:48:58 +0300 | [diff] [blame] | 452 | |
koder aka kdanilov | 416b87a | 2015-05-12 00:26:04 +0300 | [diff] [blame] | 453 | return user, passwd, tenant, auth_url |
koder aka kdanilov | f86d7af | 2015-05-06 04:01:54 +0300 | [diff] [blame] | 454 | |
| 455 | |
koder aka kdanilov | 416b87a | 2015-05-12 00:26:04 +0300 | [diff] [blame] | 456 | def get_OS_credentials(cfg, ctx): |
koder aka kdanilov | cee4334 | 2015-04-14 22:52:53 +0300 | [diff] [blame] | 457 | creds = None |
koder aka kdanilov | fd2cfa5 | 2015-05-20 03:17:42 +0300 | [diff] [blame] | 458 | tenant = None |
| 459 | |
koder aka kdanilov | 416b87a | 2015-05-12 00:26:04 +0300 | [diff] [blame] | 460 | if 'openstack' in cfg['clouds']: |
| 461 | os_cfg = cfg['clouds']['openstack'] |
| 462 | if 'OPENRC' in os_cfg: |
| 463 | logger.info("Using OS credentials from " + os_cfg['OPENRC']) |
| 464 | user, passwd, tenant, auth_url = \ |
| 465 | get_creds_openrc(os_cfg['OPENRC']) |
| 466 | elif 'ENV' in os_cfg: |
| 467 | logger.info("Using OS credentials from shell environment") |
| 468 | user, passwd, tenant, auth_url = start_vms.ostack_get_creds() |
koder aka kdanilov | fd2cfa5 | 2015-05-20 03:17:42 +0300 | [diff] [blame] | 469 | elif 'OS_TENANT_NAME' in os_cfg: |
koder aka kdanilov | 416b87a | 2015-05-12 00:26:04 +0300 | [diff] [blame] | 470 | logger.info("Using predefined credentials") |
koder aka kdanilov | 1c2b511 | 2015-04-10 16:53:51 +0300 | [diff] [blame] | 471 | tenant = os_cfg['OS_TENANT_NAME'].strip() |
| 472 | user = os_cfg['OS_USERNAME'].strip() |
| 473 | passwd = os_cfg['OS_PASSWORD'].strip() |
| 474 | auth_url = os_cfg['OS_AUTH_URL'].strip() |
| 475 | |
koder aka kdanilov | fd2cfa5 | 2015-05-20 03:17:42 +0300 | [diff] [blame] | 476 | if tenant is None and 'fuel' in cfg['clouds'] and \ |
koder aka kdanilov | 88407ff | 2015-05-26 15:35:57 +0300 | [diff] [blame] | 477 | 'openstack_env' in cfg['clouds']['fuel'] and \ |
| 478 | ctx.fuel_openstack_creds is not None: |
koder aka kdanilov | 416b87a | 2015-05-12 00:26:04 +0300 | [diff] [blame] | 479 | logger.info("Using fuel creds") |
| 480 | creds = ctx.fuel_openstack_creds |
koder aka kdanilov | fd2cfa5 | 2015-05-20 03:17:42 +0300 | [diff] [blame] | 481 | elif tenant is None: |
koder aka kdanilov | 416b87a | 2015-05-12 00:26:04 +0300 | [diff] [blame] | 482 | logger.error("Can't found OS credentials") |
| 483 | raise utils.StopTestError("Can't found OS credentials", None) |
koder aka kdanilov | 1c2b511 | 2015-04-10 16:53:51 +0300 | [diff] [blame] | 484 | |
koder aka kdanilov | cee4334 | 2015-04-14 22:52:53 +0300 | [diff] [blame] | 485 | if creds is None: |
| 486 | creds = {'name': user, |
| 487 | 'passwd': passwd, |
| 488 | 'tenant': tenant, |
| 489 | 'auth_url': auth_url} |
koder aka kdanilov | 1c2b511 | 2015-04-10 16:53:51 +0300 | [diff] [blame] | 490 | |
koder aka kdanilov | 46d4f39 | 2015-04-24 11:35:00 +0300 | [diff] [blame] | 491 | msg = "OS_CREDS: user={name} tenant={tenant} auth_url={auth_url}" |
| 492 | logger.debug(msg.format(**creds)) |
koder aka kdanilov | cee4334 | 2015-04-14 22:52:53 +0300 | [diff] [blame] | 493 | return creds |
koder aka kdanilov | 4e9f3ed | 2015-04-14 11:26:12 +0300 | [diff] [blame] | 494 | |
koder aka kdanilov | 1c2b511 | 2015-04-10 16:53:51 +0300 | [diff] [blame] | 495 | |
koder aka kdanilov | 168f609 | 2015-04-19 02:33:38 +0300 | [diff] [blame] | 496 | @contextlib.contextmanager |
koder aka kdanilov | d5ed4da | 2015-05-07 23:33:23 +0300 | [diff] [blame] | 497 | def create_vms_ctx(ctx, cfg, config, already_has_count=0): |
koder aka kdanilov | c368eb6 | 2015-04-28 18:22:01 +0300 | [diff] [blame] | 498 | params = cfg['vm_configs'][config['cfg_name']].copy() |
koder aka kdanilov | 168f609 | 2015-04-19 02:33:38 +0300 | [diff] [blame] | 499 | os_nodes_ids = [] |
| 500 | |
koder aka kdanilov | 416b87a | 2015-05-12 00:26:04 +0300 | [diff] [blame] | 501 | if not start_vms.is_connected(): |
| 502 | os_creds = get_OS_credentials(cfg, ctx) |
| 503 | else: |
| 504 | os_creds = {} |
koder aka kdanilov | 168f609 | 2015-04-19 02:33:38 +0300 | [diff] [blame] | 505 | start_vms.nova_connect(**os_creds) |
| 506 | |
koder aka kdanilov | c368eb6 | 2015-04-28 18:22:01 +0300 | [diff] [blame] | 507 | params.update(config) |
koder aka kdanilov | fd2cfa5 | 2015-05-20 03:17:42 +0300 | [diff] [blame] | 508 | if 'keypair_file_private' not in params: |
| 509 | params['keypair_file_private'] = params['keypair_name'] + ".pem" |
| 510 | |
koder aka kdanilov | c368eb6 | 2015-04-28 18:22:01 +0300 | [diff] [blame] | 511 | params['group_name'] = cfg_dict['run_uuid'] |
| 512 | |
koder aka kdanilov | d5ed4da | 2015-05-07 23:33:23 +0300 | [diff] [blame] | 513 | if not config.get('skip_preparation', False): |
koder aka kdanilov | 416b87a | 2015-05-12 00:26:04 +0300 | [diff] [blame] | 514 | logger.info("Preparing openstack") |
koder aka kdanilov | d5ed4da | 2015-05-07 23:33:23 +0300 | [diff] [blame] | 515 | start_vms.prepare_os_subpr(params=params, **os_creds) |
koder aka kdanilov | 168f609 | 2015-04-19 02:33:38 +0300 | [diff] [blame] | 516 | |
| 517 | new_nodes = [] |
| 518 | try: |
koder aka kdanilov | d5ed4da | 2015-05-07 23:33:23 +0300 | [diff] [blame] | 519 | for new_node, node_id in start_vms.launch_vms(params, |
| 520 | already_has_count): |
koder aka kdanilov | 168f609 | 2015-04-19 02:33:38 +0300 | [diff] [blame] | 521 | new_node.roles.append('testnode') |
| 522 | ctx.nodes.append(new_node) |
| 523 | os_nodes_ids.append(node_id) |
| 524 | new_nodes.append(new_node) |
| 525 | |
| 526 | store_nodes_in_log(cfg, os_nodes_ids) |
| 527 | ctx.openstack_nodes_ids = os_nodes_ids |
| 528 | |
| 529 | yield new_nodes |
| 530 | |
| 531 | finally: |
| 532 | if not cfg['keep_vm']: |
| 533 | shut_down_vms_stage(cfg, ctx) |
| 534 | |
| 535 | |
koder aka kdanilov | cee4334 | 2015-04-14 22:52:53 +0300 | [diff] [blame] | 536 | def run_tests_stage(cfg, ctx): |
koder aka kdanilov | fd2cfa5 | 2015-05-20 03:17:42 +0300 | [diff] [blame] | 537 | ctx.results = collections.defaultdict(lambda: []) |
koder aka kdanilov | 1c2b511 | 2015-04-10 16:53:51 +0300 | [diff] [blame] | 538 | |
koder aka kdanilov | cee4334 | 2015-04-14 22:52:53 +0300 | [diff] [blame] | 539 | if 'tests' not in cfg: |
| 540 | return |
gstepanov | 023c1e4 | 2015-04-08 15:50:19 +0300 | [diff] [blame] | 541 | |
koder aka kdanilov | cee4334 | 2015-04-14 22:52:53 +0300 | [diff] [blame] | 542 | for group in cfg['tests']: |
| 543 | |
| 544 | assert len(group.items()) == 1 |
| 545 | key, config = group.items()[0] |
| 546 | |
| 547 | if 'start_test_nodes' == key: |
koder aka kdanilov | c368eb6 | 2015-04-28 18:22:01 +0300 | [diff] [blame] | 548 | if 'openstack' not in config: |
| 549 | msg = "No openstack block in config - can't spawn vm's" |
| 550 | logger.error(msg) |
| 551 | raise utils.StopTestError(msg) |
| 552 | |
koder aka kdanilov | d5ed4da | 2015-05-07 23:33:23 +0300 | [diff] [blame] | 553 | num_test_nodes = sum(1 for node in ctx.nodes |
| 554 | if 'testnode' in node.roles) |
| 555 | |
| 556 | vm_ctx = create_vms_ctx(ctx, cfg, config['openstack'], |
| 557 | num_test_nodes) |
| 558 | with vm_ctx as new_nodes: |
koder aka kdanilov | 416b87a | 2015-05-12 00:26:04 +0300 | [diff] [blame] | 559 | if len(new_nodes) != 0: |
koder aka kdanilov | 4af1c1d | 2015-05-18 15:48:58 +0300 | [diff] [blame] | 560 | logger.debug("Connecting to new nodes") |
koder aka kdanilov | 416b87a | 2015-05-12 00:26:04 +0300 | [diff] [blame] | 561 | connect_all(new_nodes, True) |
koder aka kdanilov | cee4334 | 2015-04-14 22:52:53 +0300 | [diff] [blame] | 562 | |
koder aka kdanilov | 416b87a | 2015-05-12 00:26:04 +0300 | [diff] [blame] | 563 | for node in new_nodes: |
| 564 | if node.connection is None: |
| 565 | msg = "Failed to connect to vm {0}" |
| 566 | raise RuntimeError(msg.format(node.get_conn_id())) |
koder aka kdanilov | cee4334 | 2015-04-14 22:52:53 +0300 | [diff] [blame] | 567 | |
koder aka kdanilov | 4af1c1d | 2015-05-18 15:48:58 +0300 | [diff] [blame] | 568 | with with_sensors_util(cfg_dict, ctx.nodes): |
| 569 | for test_group in config.get('tests', []): |
koder aka kdanilov | fd2cfa5 | 2015-05-20 03:17:42 +0300 | [diff] [blame] | 570 | for tp, res in run_tests(cfg, test_group, ctx.nodes): |
| 571 | ctx.results[tp].extend(res) |
koder aka kdanilov | 4500a5f | 2015-04-17 16:55:17 +0300 | [diff] [blame] | 572 | else: |
koder aka kdanilov | 4af1c1d | 2015-05-18 15:48:58 +0300 | [diff] [blame] | 573 | with with_sensors_util(cfg_dict, ctx.nodes): |
koder aka kdanilov | fd2cfa5 | 2015-05-20 03:17:42 +0300 | [diff] [blame] | 574 | for tp, res in run_tests(cfg, group, ctx.nodes): |
| 575 | ctx.results[tp].extend(res) |
koder aka kdanilov | da45e88 | 2015-04-06 02:24:42 +0300 | [diff] [blame] | 576 | |
gstepanov | 023c1e4 | 2015-04-08 15:50:19 +0300 | [diff] [blame] | 577 | |
koder aka kdanilov | 1c2b511 | 2015-04-10 16:53:51 +0300 | [diff] [blame] | 578 | def shut_down_vms_stage(cfg, ctx): |
koder aka kdanilov | 66839a9 | 2015-04-11 13:22:31 +0300 | [diff] [blame] | 579 | vm_ids_fname = cfg_dict['vm_ids_fname'] |
koder aka kdanilov | 1c2b511 | 2015-04-10 16:53:51 +0300 | [diff] [blame] | 580 | if ctx.openstack_nodes_ids is None: |
koder aka kdanilov | 66839a9 | 2015-04-11 13:22:31 +0300 | [diff] [blame] | 581 | nodes_ids = open(vm_ids_fname).read().split() |
koder aka kdanilov | 1c2b511 | 2015-04-10 16:53:51 +0300 | [diff] [blame] | 582 | else: |
| 583 | nodes_ids = ctx.openstack_nodes_ids |
| 584 | |
koder aka kdanilov | 652cd80 | 2015-04-13 12:21:07 +0300 | [diff] [blame] | 585 | if len(nodes_ids) != 0: |
| 586 | logger.info("Removing nodes") |
| 587 | start_vms.clear_nodes(nodes_ids) |
| 588 | logger.info("Nodes has been removed") |
gstepanov | 023c1e4 | 2015-04-08 15:50:19 +0300 | [diff] [blame] | 589 | |
koder aka kdanilov | 66839a9 | 2015-04-11 13:22:31 +0300 | [diff] [blame] | 590 | if os.path.exists(vm_ids_fname): |
| 591 | os.remove(vm_ids_fname) |
gstepanov | 023c1e4 | 2015-04-08 15:50:19 +0300 | [diff] [blame] | 592 | |
koder aka kdanilov | 66839a9 | 2015-04-11 13:22:31 +0300 | [diff] [blame] | 593 | |
| 594 | def store_nodes_in_log(cfg, nodes_ids): |
| 595 | with open(cfg['vm_ids_fname'], 'w') as fd: |
| 596 | fd.write("\n".join(nodes_ids)) |
gstepanov | 023c1e4 | 2015-04-08 15:50:19 +0300 | [diff] [blame] | 597 | |
| 598 | |
| 599 | def clear_enviroment(cfg, ctx): |
koder aka kdanilov | 66839a9 | 2015-04-11 13:22:31 +0300 | [diff] [blame] | 600 | if os.path.exists(cfg_dict['vm_ids_fname']): |
koder aka kdanilov | 1c2b511 | 2015-04-10 16:53:51 +0300 | [diff] [blame] | 601 | shut_down_vms_stage(cfg, ctx) |
gstepanov | 023c1e4 | 2015-04-08 15:50:19 +0300 | [diff] [blame] | 602 | |
| 603 | |
koder aka kdanilov | da45e88 | 2015-04-06 02:24:42 +0300 | [diff] [blame] | 604 | def disconnect_stage(cfg, ctx): |
koder aka kdanilov | 652cd80 | 2015-04-13 12:21:07 +0300 | [diff] [blame] | 605 | ssh_utils.close_all_sessions() |
| 606 | |
koder aka kdanilov | da45e88 | 2015-04-06 02:24:42 +0300 | [diff] [blame] | 607 | for node in ctx.nodes: |
| 608 | if node.connection is not None: |
| 609 | node.connection.close() |
| 610 | |
| 611 | |
koder aka kdanilov | 66839a9 | 2015-04-11 13:22:31 +0300 | [diff] [blame] | 612 | def store_raw_results_stage(cfg, ctx): |
| 613 | |
koder aka kdanilov | fd2cfa5 | 2015-05-20 03:17:42 +0300 | [diff] [blame] | 614 | raw_results = cfg_dict['raw_results'] |
koder aka kdanilov | 66839a9 | 2015-04-11 13:22:31 +0300 | [diff] [blame] | 615 | |
| 616 | if os.path.exists(raw_results): |
koder aka kdanilov | 88407ff | 2015-05-26 15:35:57 +0300 | [diff] [blame] | 617 | cont = yaml_load(open(raw_results).read()) |
koder aka kdanilov | 66839a9 | 2015-04-11 13:22:31 +0300 | [diff] [blame] | 618 | else: |
| 619 | cont = [] |
| 620 | |
koder aka kdanilov | fd2cfa5 | 2015-05-20 03:17:42 +0300 | [diff] [blame] | 621 | cont.extend(utils.yamable(ctx.results).items()) |
koder aka kdanilov | 66839a9 | 2015-04-11 13:22:31 +0300 | [diff] [blame] | 622 | raw_data = pretty_yaml.dumps(cont) |
| 623 | |
| 624 | with open(raw_results, "w") as fd: |
| 625 | fd.write(raw_data) |
| 626 | |
| 627 | |
| 628 | def console_report_stage(cfg, ctx): |
koder aka kdanilov | fd2cfa5 | 2015-05-20 03:17:42 +0300 | [diff] [blame] | 629 | first_report = True |
| 630 | text_rep_fname = cfg['text_report_file'] |
| 631 | with open(text_rep_fname, "w") as fd: |
| 632 | for tp, data in ctx.results.items(): |
| 633 | if 'io' == tp and data is not None: |
| 634 | dinfo = report.process_disk_info(data) |
| 635 | rep = IOPerfTest.format_for_console(data, dinfo) |
| 636 | elif tp in ['mysql', 'pgbench'] and data is not None: |
| 637 | rep = MysqlTest.format_for_console(data) |
| 638 | else: |
| 639 | logger.warning("Can't generate text report for " + tp) |
| 640 | continue |
koder aka kdanilov | 416b87a | 2015-05-12 00:26:04 +0300 | [diff] [blame] | 641 | |
koder aka kdanilov | fd2cfa5 | 2015-05-20 03:17:42 +0300 | [diff] [blame] | 642 | fd.write(rep) |
| 643 | fd.write("\n") |
koder aka kdanilov | 416b87a | 2015-05-12 00:26:04 +0300 | [diff] [blame] | 644 | |
koder aka kdanilov | fd2cfa5 | 2015-05-20 03:17:42 +0300 | [diff] [blame] | 645 | if first_report: |
| 646 | logger.info("Text report were stored in " + text_rep_fname) |
| 647 | first_report = False |
| 648 | |
koder aka kdanilov | 4af1c1d | 2015-05-18 15:48:58 +0300 | [diff] [blame] | 649 | print("\n" + rep + "\n") |
koder aka kdanilov | 416b87a | 2015-05-12 00:26:04 +0300 | [diff] [blame] | 650 | |
koder aka kdanilov | 66839a9 | 2015-04-11 13:22:31 +0300 | [diff] [blame] | 651 | |
koder aka kdanilov | 88407ff | 2015-05-26 15:35:57 +0300 | [diff] [blame] | 652 | def test_load_report_stage(cfg, ctx): |
| 653 | load_rep_fname = cfg['load_report_file'] |
| 654 | found = False |
| 655 | for idx, (tp, data) in enumerate(ctx.results.items()): |
| 656 | if 'io' == tp and data is not None: |
| 657 | if found: |
| 658 | logger.error("Making reports for more than one " + |
| 659 | "io block isn't supported! All " + |
| 660 | "report, except first are skipped") |
| 661 | continue |
| 662 | found = True |
| 663 | report.make_load_report(idx, cfg['results'], load_rep_fname) |
| 664 | |
| 665 | |
koder aka kdanilov | e87ae65 | 2015-04-20 02:14:35 +0300 | [diff] [blame] | 666 | def html_report_stage(cfg, ctx): |
Yulia Portnova | 8ca2057 | 2015-04-14 14:09:39 +0300 | [diff] [blame] | 667 | html_rep_fname = cfg['html_report_file'] |
koder aka kdanilov | f86d7af | 2015-05-06 04:01:54 +0300 | [diff] [blame] | 668 | found = False |
koder aka kdanilov | fd2cfa5 | 2015-05-20 03:17:42 +0300 | [diff] [blame] | 669 | for tp, data in ctx.results.items(): |
koder aka kdanilov | f86d7af | 2015-05-06 04:01:54 +0300 | [diff] [blame] | 670 | if 'io' == tp and data is not None: |
| 671 | if found: |
| 672 | logger.error("Making reports for more than one " + |
| 673 | "io block isn't supported! All " + |
| 674 | "report, except first are skipped") |
| 675 | continue |
| 676 | found = True |
| 677 | dinfo = report.process_disk_info(data) |
koder aka kdanilov | 88407ff | 2015-05-26 15:35:57 +0300 | [diff] [blame] | 678 | report.make_io_report(dinfo, |
| 679 | cfg.get('comment', ''), |
| 680 | html_rep_fname, |
koder aka kdanilov | f86d7af | 2015-05-06 04:01:54 +0300 | [diff] [blame] | 681 | lab_info=ctx.hw_info) |
koder aka kdanilov | cff7b2e | 2015-04-18 20:48:15 +0300 | [diff] [blame] | 682 | |
koder aka kdanilov | da45e88 | 2015-04-06 02:24:42 +0300 | [diff] [blame] | 683 | |
| 684 | def complete_log_nodes_statistic(cfg, ctx): |
| 685 | nodes = ctx.nodes |
| 686 | for node in nodes: |
| 687 | logger.debug(str(node)) |
| 688 | |
| 689 | |
koder aka kdanilov | 88407ff | 2015-05-26 15:35:57 +0300 | [diff] [blame] | 690 | def load_data_from_file(var_dir, _, ctx): |
| 691 | raw_results = os.path.join(var_dir, 'raw_results.yaml') |
| 692 | ctx.results = {} |
| 693 | for tp, results in yaml_load(open(raw_results).read()): |
| 694 | cls = TOOL_TYPE_MAPPER[tp] |
| 695 | ctx.results[tp] = map(cls.load, results) |
koder aka kdanilov | 4af1c1d | 2015-05-18 15:48:58 +0300 | [diff] [blame] | 696 | |
koder aka kdanilov | 88407ff | 2015-05-26 15:35:57 +0300 | [diff] [blame] | 697 | |
| 698 | def load_data_from(var_dir): |
| 699 | return functools.partial(load_data_from_file, var_dir) |
gstepanov | cd256d6 | 2015-04-07 17:47:32 +0300 | [diff] [blame] | 700 | |
| 701 | |
koder aka kdanilov | 57ce4db | 2015-04-25 21:25:51 +0300 | [diff] [blame] | 702 | def start_web_ui(cfg, ctx): |
| 703 | if webui is None: |
| 704 | logger.error("Can't start webui. Install cherrypy module") |
| 705 | ctx.web_thread = None |
| 706 | else: |
| 707 | th = threading.Thread(None, webui.web_main_thread, "webui", (None,)) |
| 708 | th.daemon = True |
| 709 | th.start() |
| 710 | ctx.web_thread = th |
| 711 | |
| 712 | |
| 713 | def stop_web_ui(cfg, ctx): |
| 714 | webui.web_main_stop() |
| 715 | time.sleep(1) |
| 716 | |
| 717 | |
koder aka kdanilov | cee4334 | 2015-04-14 22:52:53 +0300 | [diff] [blame] | 718 | def parse_args(argv): |
koder aka kdanilov | cff7b2e | 2015-04-18 20:48:15 +0300 | [diff] [blame] | 719 | descr = "Disk io performance test suite" |
| 720 | parser = argparse.ArgumentParser(prog='wally', description=descr) |
koder aka kdanilov | cee4334 | 2015-04-14 22:52:53 +0300 | [diff] [blame] | 721 | |
| 722 | parser.add_argument("-l", dest='extra_logs', |
| 723 | action='store_true', default=False, |
| 724 | help="print some extra log info") |
koder aka kdanilov | cee4334 | 2015-04-14 22:52:53 +0300 | [diff] [blame] | 725 | parser.add_argument("-b", '--build_description', |
| 726 | type=str, default="Build info") |
| 727 | parser.add_argument("-i", '--build_id', type=str, default="id") |
| 728 | parser.add_argument("-t", '--build_type', type=str, default="GA") |
| 729 | parser.add_argument("-u", '--username', type=str, default="admin") |
koder aka kdanilov | e87ae65 | 2015-04-20 02:14:35 +0300 | [diff] [blame] | 730 | parser.add_argument("-n", '--no-tests', action='store_true', |
| 731 | help="Don't run tests", default=False) |
koder aka kdanilov | cff7b2e | 2015-04-18 20:48:15 +0300 | [diff] [blame] | 732 | parser.add_argument("-p", '--post-process-only', metavar="VAR_DIR", |
| 733 | help="Only process data from previour run") |
koder aka kdanilov | 88407ff | 2015-05-26 15:35:57 +0300 | [diff] [blame] | 734 | parser.add_argument("-x", '--xxx', action='store_true') |
koder aka kdanilov | cff7b2e | 2015-04-18 20:48:15 +0300 | [diff] [blame] | 735 | parser.add_argument("-k", '--keep-vm', action='store_true', |
| 736 | help="Don't remove test vm's", default=False) |
koder aka kdanilov | e87ae65 | 2015-04-20 02:14:35 +0300 | [diff] [blame] | 737 | parser.add_argument("-d", '--dont-discover-nodes', action='store_true', |
| 738 | help="Don't connect/discover fuel nodes", |
| 739 | default=False) |
koder aka kdanilov | a047e1b | 2015-04-21 23:16:59 +0300 | [diff] [blame] | 740 | parser.add_argument("-r", '--no-html-report', action='store_true', |
| 741 | help="Skip html report", default=False) |
koder aka kdanilov | f86d7af | 2015-05-06 04:01:54 +0300 | [diff] [blame] | 742 | parser.add_argument("--params", metavar="testname.paramname", |
koder aka kdanilov | 63ad206 | 2015-04-27 13:11:40 +0300 | [diff] [blame] | 743 | help="Test params", default=[]) |
koder aka kdanilov | fd2cfa5 | 2015-05-20 03:17:42 +0300 | [diff] [blame] | 744 | parser.add_argument("--ls", action='store_true', default=False) |
koder aka kdanilov | 88407ff | 2015-05-26 15:35:57 +0300 | [diff] [blame] | 745 | parser.add_argument("-c", "--comment", default="") |
koder aka kdanilov | cff7b2e | 2015-04-18 20:48:15 +0300 | [diff] [blame] | 746 | parser.add_argument("config_file") |
koder aka kdanilov | cee4334 | 2015-04-14 22:52:53 +0300 | [diff] [blame] | 747 | |
| 748 | return parser.parse_args(argv[1:]) |
| 749 | |
| 750 | |
koder aka kdanilov | 4af1c1d | 2015-05-18 15:48:58 +0300 | [diff] [blame] | 751 | def get_stage_name(func): |
koder aka kdanilov | 7248c7b | 2015-05-31 22:53:03 +0300 | [diff] [blame^] | 752 | nm = get_func_name(func) |
| 753 | if nm.endswith("stage"): |
| 754 | return nm |
koder aka kdanilov | 4af1c1d | 2015-05-18 15:48:58 +0300 | [diff] [blame] | 755 | else: |
koder aka kdanilov | 7248c7b | 2015-05-31 22:53:03 +0300 | [diff] [blame^] | 756 | return nm + " stage" |
koder aka kdanilov | 416b87a | 2015-05-12 00:26:04 +0300 | [diff] [blame] | 757 | |
| 758 | |
koder aka kdanilov | 88407ff | 2015-05-26 15:35:57 +0300 | [diff] [blame] | 759 | def get_test_names(raw_res): |
| 760 | res = set() |
| 761 | for tp, data in raw_res: |
| 762 | for block in data: |
| 763 | res.add("{0}({1})".format(tp, block.get('test_name', '-'))) |
| 764 | return res |
koder aka kdanilov | bb5fe07 | 2015-05-21 02:50:23 +0300 | [diff] [blame] | 765 | |
| 766 | |
koder aka kdanilov | fd2cfa5 | 2015-05-20 03:17:42 +0300 | [diff] [blame] | 767 | def list_results(path): |
| 768 | results = [] |
| 769 | |
| 770 | for dname in os.listdir(path): |
| 771 | |
koder aka kdanilov | 88407ff | 2015-05-26 15:35:57 +0300 | [diff] [blame] | 772 | files_cfg = get_test_files(os.path.join(path, dname)) |
koder aka kdanilov | fd2cfa5 | 2015-05-20 03:17:42 +0300 | [diff] [blame] | 773 | |
koder aka kdanilov | 88407ff | 2015-05-26 15:35:57 +0300 | [diff] [blame] | 774 | if not os.path.isfile(files_cfg['raw_results']): |
koder aka kdanilov | fd2cfa5 | 2015-05-20 03:17:42 +0300 | [diff] [blame] | 775 | continue |
| 776 | |
koder aka kdanilov | 88407ff | 2015-05-26 15:35:57 +0300 | [diff] [blame] | 777 | mt = os.path.getmtime(files_cfg['raw_results']) |
| 778 | res_mtime = time.ctime(mt) |
koder aka kdanilov | fd2cfa5 | 2015-05-20 03:17:42 +0300 | [diff] [blame] | 779 | |
koder aka kdanilov | 88407ff | 2015-05-26 15:35:57 +0300 | [diff] [blame] | 780 | raw_res = yaml_load(open(files_cfg['raw_results']).read()) |
| 781 | test_names = ",".join(sorted(get_test_names(raw_res))) |
koder aka kdanilov | fd2cfa5 | 2015-05-20 03:17:42 +0300 | [diff] [blame] | 782 | |
koder aka kdanilov | 88407ff | 2015-05-26 15:35:57 +0300 | [diff] [blame] | 783 | params = load_run_params(files_cfg['run_params_file']) |
koder aka kdanilov | bb5fe07 | 2015-05-21 02:50:23 +0300 | [diff] [blame] | 784 | |
koder aka kdanilov | 88407ff | 2015-05-26 15:35:57 +0300 | [diff] [blame] | 785 | comm = params.get('comment') |
| 786 | results.append((mt, dname, test_names, res_mtime, |
| 787 | '-' if comm is None else comm)) |
koder aka kdanilov | fd2cfa5 | 2015-05-20 03:17:42 +0300 | [diff] [blame] | 788 | |
koder aka kdanilov | 88407ff | 2015-05-26 15:35:57 +0300 | [diff] [blame] | 789 | tab = texttable.Texttable(max_width=200) |
koder aka kdanilov | fd2cfa5 | 2015-05-20 03:17:42 +0300 | [diff] [blame] | 790 | tab.set_deco(tab.HEADER | tab.VLINES | tab.BORDER) |
koder aka kdanilov | 88407ff | 2015-05-26 15:35:57 +0300 | [diff] [blame] | 791 | tab.set_cols_align(["l", "l", "l", "l"]) |
| 792 | results.sort() |
koder aka kdanilov | fd2cfa5 | 2015-05-20 03:17:42 +0300 | [diff] [blame] | 793 | |
koder aka kdanilov | 88407ff | 2015-05-26 15:35:57 +0300 | [diff] [blame] | 794 | for data in results[::-1]: |
| 795 | tab.add_row(data[1:]) |
koder aka kdanilov | fd2cfa5 | 2015-05-20 03:17:42 +0300 | [diff] [blame] | 796 | |
koder aka kdanilov | 88407ff | 2015-05-26 15:35:57 +0300 | [diff] [blame] | 797 | tab.header(["Name", "Tests", "etime", "Comment"]) |
koder aka kdanilov | fd2cfa5 | 2015-05-20 03:17:42 +0300 | [diff] [blame] | 798 | |
| 799 | print(tab.draw()) |
| 800 | |
| 801 | |
koder aka kdanilov | 7248c7b | 2015-05-31 22:53:03 +0300 | [diff] [blame^] | 802 | def get_func_name(obj): |
| 803 | if hasattr(obj, '__name__'): |
| 804 | return obj.__name__ |
| 805 | if hasattr(obj, 'func_name'): |
| 806 | return obj.func_name |
| 807 | return obj.func.func_name |
| 808 | |
| 809 | |
| 810 | @contextlib.contextmanager |
| 811 | def log_stage(func): |
| 812 | msg_templ = "Exception during {0}: {1!s}" |
| 813 | msg_templ_no_exc = "During {0}" |
| 814 | |
| 815 | logger.info("Start " + get_stage_name(func)) |
| 816 | |
| 817 | try: |
| 818 | yield |
| 819 | except utils.StopTestError as exc: |
| 820 | logger.error(msg_templ.format( |
| 821 | get_func_name(func), exc)) |
| 822 | except Exception: |
| 823 | logger.exception(msg_templ_no_exc.format( |
| 824 | get_func_name(func))) |
| 825 | |
| 826 | |
koder aka kdanilov | 3f35626 | 2015-02-13 08:06:14 -0800 | [diff] [blame] | 827 | def main(argv): |
koder aka kdanilov | bb5fe07 | 2015-05-21 02:50:23 +0300 | [diff] [blame] | 828 | if faulthandler is not None: |
| 829 | faulthandler.register(signal.SIGUSR1, all_threads=True) |
| 830 | |
koder aka kdanilov | e06762a | 2015-03-22 23:32:09 +0200 | [diff] [blame] | 831 | opts = parse_args(argv) |
koder aka kdanilov | 88407ff | 2015-05-26 15:35:57 +0300 | [diff] [blame] | 832 | |
| 833 | if opts.ls: |
| 834 | list_results(opts.config_file) |
| 835 | exit(0) |
| 836 | |
| 837 | data_dir = load_config(opts.config_file, opts.post_process_only) |
| 838 | |
| 839 | if opts.post_process_only is None: |
| 840 | cfg_dict['comment'] = opts.comment |
| 841 | save_run_params() |
koder aka kdanilov | 2c47309 | 2015-03-29 17:12:13 +0300 | [diff] [blame] | 842 | |
koder aka kdanilov | 4af1c1d | 2015-05-18 15:48:58 +0300 | [diff] [blame] | 843 | if cfg_dict.get('logging', {}).get("extra_logs", False) or opts.extra_logs: |
| 844 | level = logging.DEBUG |
| 845 | else: |
| 846 | level = logging.WARNING |
| 847 | |
| 848 | setup_loggers(level, cfg_dict['log_file']) |
| 849 | |
koder aka kdanilov | fd2cfa5 | 2015-05-20 03:17:42 +0300 | [diff] [blame] | 850 | if not os.path.exists(cfg_dict['saved_config_file']): |
| 851 | with open(cfg_dict['saved_config_file'], 'w') as fd: |
| 852 | fd.write(open(opts.config_file).read()) |
| 853 | |
koder aka kdanilov | 66839a9 | 2015-04-11 13:22:31 +0300 | [diff] [blame] | 854 | if opts.post_process_only is not None: |
| 855 | stages = [ |
koder aka kdanilov | 88407ff | 2015-05-26 15:35:57 +0300 | [diff] [blame] | 856 | load_data_from(data_dir) |
koder aka kdanilov | 66839a9 | 2015-04-11 13:22:31 +0300 | [diff] [blame] | 857 | ] |
| 858 | else: |
| 859 | stages = [ |
koder aka kdanilov | f86d7af | 2015-05-06 04:01:54 +0300 | [diff] [blame] | 860 | discover_stage |
| 861 | ] |
| 862 | |
koder aka kdanilov | f86d7af | 2015-05-06 04:01:54 +0300 | [diff] [blame] | 863 | stages.extend([ |
koder aka kdanilov | 416b87a | 2015-05-12 00:26:04 +0300 | [diff] [blame] | 864 | reuse_vms_stage, |
koder aka kdanilov | 66839a9 | 2015-04-11 13:22:31 +0300 | [diff] [blame] | 865 | log_nodes_statistic, |
koder aka kdanilov | f86d7af | 2015-05-06 04:01:54 +0300 | [diff] [blame] | 866 | save_nodes_stage, |
| 867 | connect_stage]) |
| 868 | |
| 869 | if cfg_dict.get('collect_info', True): |
| 870 | stages.append(collect_hw_info_stage) |
| 871 | |
| 872 | stages.extend([ |
koder aka kdanilov | 4af1c1d | 2015-05-18 15:48:58 +0300 | [diff] [blame] | 873 | # deploy_sensors_stage, |
koder aka kdanilov | 66839a9 | 2015-04-11 13:22:31 +0300 | [diff] [blame] | 874 | run_tests_stage, |
koder aka kdanilov | 416b87a | 2015-05-12 00:26:04 +0300 | [diff] [blame] | 875 | store_raw_results_stage, |
koder aka kdanilov | 4af1c1d | 2015-05-18 15:48:58 +0300 | [diff] [blame] | 876 | # gather_sensors_stage |
koder aka kdanilov | f86d7af | 2015-05-06 04:01:54 +0300 | [diff] [blame] | 877 | ]) |
koder aka kdanilov | 66839a9 | 2015-04-11 13:22:31 +0300 | [diff] [blame] | 878 | |
koder aka kdanilov | e87ae65 | 2015-04-20 02:14:35 +0300 | [diff] [blame] | 879 | report_stages = [ |
| 880 | console_report_stage, |
koder aka kdanilov | e87ae65 | 2015-04-20 02:14:35 +0300 | [diff] [blame] | 881 | ] |
| 882 | |
koder aka kdanilov | 88407ff | 2015-05-26 15:35:57 +0300 | [diff] [blame] | 883 | if opts.xxx: |
| 884 | report_stages.append(test_load_report_stage) |
| 885 | elif not opts.no_html_report: |
koder aka kdanilov | a047e1b | 2015-04-21 23:16:59 +0300 | [diff] [blame] | 886 | report_stages.append(html_report_stage) |
| 887 | |
koder aka kdanilov | 652cd80 | 2015-04-13 12:21:07 +0300 | [diff] [blame] | 888 | logger.info("All info would be stored into {0}".format( |
| 889 | cfg_dict['var_dir'])) |
gstepanov | cd256d6 | 2015-04-07 17:47:32 +0300 | [diff] [blame] | 890 | |
koder aka kdanilov | da45e88 | 2015-04-06 02:24:42 +0300 | [diff] [blame] | 891 | ctx = Context() |
gstepanov | affcdb1 | 2015-04-07 17:18:29 +0300 | [diff] [blame] | 892 | ctx.build_meta['build_id'] = opts.build_id |
| 893 | ctx.build_meta['build_descrption'] = opts.build_description |
| 894 | ctx.build_meta['build_type'] = opts.build_type |
| 895 | ctx.build_meta['username'] = opts.username |
koder aka kdanilov | 57ce4db | 2015-04-25 21:25:51 +0300 | [diff] [blame] | 896 | ctx.sensors_data = SensorDatastore() |
koder aka kdanilov | e87ae65 | 2015-04-20 02:14:35 +0300 | [diff] [blame] | 897 | |
koder aka kdanilov | 168f609 | 2015-04-19 02:33:38 +0300 | [diff] [blame] | 898 | cfg_dict['keep_vm'] = opts.keep_vm |
koder aka kdanilov | e87ae65 | 2015-04-20 02:14:35 +0300 | [diff] [blame] | 899 | cfg_dict['no_tests'] = opts.no_tests |
| 900 | cfg_dict['dont_discover_nodes'] = opts.dont_discover_nodes |
koder aka kdanilov | 6c49106 | 2015-04-09 22:33:13 +0300 | [diff] [blame] | 901 | |
koder aka kdanilov | 57ce4db | 2015-04-25 21:25:51 +0300 | [diff] [blame] | 902 | if cfg_dict.get('run_web_ui', False): |
| 903 | start_web_ui(cfg_dict, ctx) |
| 904 | |
koder aka kdanilov | 7248c7b | 2015-05-31 22:53:03 +0300 | [diff] [blame^] | 905 | for stage in stages: |
| 906 | ok = False |
| 907 | with log_stage(stage): |
koder aka kdanilov | 4af1c1d | 2015-05-18 15:48:58 +0300 | [diff] [blame] | 908 | logger.info("Start " + get_stage_name(stage)) |
koder aka kdanilov | da45e88 | 2015-04-06 02:24:42 +0300 | [diff] [blame] | 909 | stage(cfg_dict, ctx) |
koder aka kdanilov | 7248c7b | 2015-05-31 22:53:03 +0300 | [diff] [blame^] | 910 | ok = True |
| 911 | if not ok: |
| 912 | break |
koder aka kdanilov | f86d7af | 2015-05-06 04:01:54 +0300 | [diff] [blame] | 913 | |
koder aka kdanilov | 7248c7b | 2015-05-31 22:53:03 +0300 | [diff] [blame^] | 914 | exc, cls, tb = sys.exc_info() |
| 915 | for stage in ctx.clear_calls_stack[::-1]: |
| 916 | with log_stage(stage): |
| 917 | stage(cfg_dict, ctx) |
| 918 | |
| 919 | logger.debug("Start utils.cleanup") |
| 920 | for clean_func, args, kwargs in utils.iter_clean_func(): |
| 921 | with log_stage(clean_func): |
| 922 | clean_func(*args, **kwargs) |
koder aka kdanilov | 2c47309 | 2015-03-29 17:12:13 +0300 | [diff] [blame] | 923 | |
koder aka kdanilov | e2de58c | 2015-04-24 22:59:36 +0300 | [diff] [blame] | 924 | if exc is None: |
| 925 | for report_stage in report_stages: |
koder aka kdanilov | 7248c7b | 2015-05-31 22:53:03 +0300 | [diff] [blame^] | 926 | with log_stage(report_stage): |
| 927 | report_stage(cfg_dict, ctx) |
koder aka kdanilov | e87ae65 | 2015-04-20 02:14:35 +0300 | [diff] [blame] | 928 | |
| 929 | logger.info("All info stored in {0} folder".format(cfg_dict['var_dir'])) |
koder aka kdanilov | e2de58c | 2015-04-24 22:59:36 +0300 | [diff] [blame] | 930 | |
koder aka kdanilov | 57ce4db | 2015-04-25 21:25:51 +0300 | [diff] [blame] | 931 | if cfg_dict.get('run_web_ui', False): |
| 932 | stop_web_ui(cfg_dict, ctx) |
| 933 | |
koder aka kdanilov | e2de58c | 2015-04-24 22:59:36 +0300 | [diff] [blame] | 934 | if exc is None: |
| 935 | logger.info("Tests finished successfully") |
| 936 | return 0 |
| 937 | else: |
| 938 | logger.error("Tests are failed. See detailed error above") |
| 939 | return 1 |