blob: fa1a801f66c3745a708b5b577bbb5a5ed255bfb0 [file] [log] [blame]
koder aka kdanilov0fdaaee2015-06-30 11:10:48 +03001import os
koder aka kdanilov0fdaaee2015-06-30 11:10:48 +03002import time
3import signal
koder aka kdanilov7f59d562016-12-26 01:34:23 +02004import pprint
koder aka kdanilovf2865172016-12-30 03:35:11 +02005import getpass
koder aka kdanilov0fdaaee2015-06-30 11:10:48 +03006import logging
koder aka kdanilov108ac362017-01-19 20:17:16 +02007import tempfile
koder aka kdanilov0fdaaee2015-06-30 11:10:48 +03008import argparse
9import functools
koder aka kdanilov108ac362017-01-19 20:17:16 +020010import subprocess
koder aka kdanilov39e449e2016-12-17 15:15:26 +020011import contextlib
12from typing import List, Tuple, Any, Callable, IO, cast, Optional, Iterator
koder aka kdanilov0fdaaee2015-06-30 11:10:48 +030013from yaml import load as _yaml_load
14
koder aka kdanilov22d134e2016-11-08 11:33:19 +020015
16YLoader = Callable[[IO], Any]
17yaml_load = None # type: YLoader
18
19
koder aka kdanilov0fdaaee2015-06-30 11:10:48 +030020try:
21 from yaml import CLoader
koder aka kdanilov22d134e2016-11-08 11:33:19 +020022 yaml_load = cast(YLoader, functools.partial(_yaml_load, Loader=CLoader))
koder aka kdanilov0fdaaee2015-06-30 11:10:48 +030023except ImportError:
koder aka kdanilov22d134e2016-11-08 11:33:19 +020024 yaml_load = cast(YLoader, _yaml_load)
koder aka kdanilov0fdaaee2015-06-30 11:10:48 +030025
26
27import texttable
28
29try:
30 import faulthandler
31except ImportError:
32 faulthandler = None
33
koder aka kdanilove7e1a4d2016-12-17 20:29:52 +020034from . import utils, node
koder aka kdanilov108ac362017-01-19 20:17:16 +020035from .node_utils import log_nodes_statistic
koder aka kdanilov3d2bc4f2016-11-12 18:31:18 +020036from .storage import make_storage, Storage
koder aka kdanilov22d134e2016-11-08 11:33:19 +020037from .config import Config
koder aka kdanilov3b4da8b2016-10-17 00:17:53 +030038from .logger import setup_loggers
koder aka kdanilov39e449e2016-12-17 15:15:26 +020039from .stage import Stage
koder aka kdanilov22d134e2016-11-08 11:33:19 +020040from .test_run_class import TestRun
koder aka kdanilov962ee5f2016-12-19 02:40:08 +020041from .ssh import set_ssh_key_passwd
koder aka kdanilov0fdaaee2015-06-30 11:10:48 +030042
43
koder aka kdanilov39e449e2016-12-17 15:15:26 +020044# stages
45from .ceph import DiscoverCephStage
46from .openstack import DiscoverOSStage
47from .fuel import DiscoverFuelStage
koder aka kdanilov23e6bdf2016-12-24 02:18:54 +020048from .run_test import (CollectInfoStage, ExplicitNodesStage, SaveNodesStage,
koder aka kdanilov7f59d562016-12-26 01:34:23 +020049 RunTestsStage, ConnectStage, SleepStage, PrepareNodes,
50 LoadStoredNodesStage)
koder aka kdanilova732a602017-02-01 20:29:56 +020051# from .process_results import CalcStatisticStage
koder aka kdanilov39e449e2016-12-17 15:15:26 +020052from .report import ConsoleReportStage, HtmlReportStage
53from .sensors import StartSensorsStage, CollectSensorsStage
54
55
koder aka kdanilov0fdaaee2015-06-30 11:10:48 +030056logger = logging.getLogger("wally")
57
58
koder aka kdanilov39e449e2016-12-17 15:15:26 +020059@contextlib.contextmanager
koder aka kdanilov962ee5f2016-12-19 02:40:08 +020060def log_stage(stage: Stage, cleanup: bool = False) -> Iterator[None]:
61 logger.info("Start " + stage.name() + ("::cleanup" if cleanup else ""))
koder aka kdanilov39e449e2016-12-17 15:15:26 +020062 try:
63 yield
64 except utils.StopTestError as exc:
koder aka kdanilov962ee5f2016-12-19 02:40:08 +020065 raise
koder aka kdanilov39e449e2016-12-17 15:15:26 +020066 except Exception:
koder aka kdanilov962ee5f2016-12-19 02:40:08 +020067 logger.exception("During %s", stage.name() + ("::cleanup" if cleanup else ""))
68 raise
koder aka kdanilov39e449e2016-12-17 15:15:26 +020069
70
koder aka kdanilov22d134e2016-11-08 11:33:19 +020071def list_results(path: str) -> List[Tuple[str, str, str, str]]:
koder aka kdanilov73084622016-11-16 21:51:08 +020072 results = [] # type: List[Tuple[float, str, str, str, str]]
koder aka kdanilov0fdaaee2015-06-30 11:10:48 +030073
koder aka kdanilov22d134e2016-11-08 11:33:19 +020074 for dir_name in os.listdir(path):
75 full_path = os.path.join(path, dir_name)
76
koder aka kdanilov0fdaaee2015-06-30 11:10:48 +030077 try:
koder aka kdanilov22d134e2016-11-08 11:33:19 +020078 stor = make_storage(full_path, existing=True)
79 except Exception as exc:
80 logger.warning("Can't load folder {}. Error {}".format(full_path, exc))
koder aka kdanilov0fdaaee2015-06-30 11:10:48 +030081
koder aka kdanilov7f59d562016-12-26 01:34:23 +020082 comment = cast(str, stor.get('info/comment'))
83 run_uuid = cast(str, stor.get('info/run_uuid'))
84 run_time = cast(float, stor.get('info/run_time'))
koder aka kdanilov22d134e2016-11-08 11:33:19 +020085 test_types = ""
koder aka kdanilov73084622016-11-16 21:51:08 +020086 results.append((run_time,
koder aka kdanilov22d134e2016-11-08 11:33:19 +020087 run_uuid,
88 test_types,
koder aka kdanilov73084622016-11-16 21:51:08 +020089 time.ctime(run_time),
koder aka kdanilov22d134e2016-11-08 11:33:19 +020090 '-' if comment is None else comment))
koder aka kdanilov0fdaaee2015-06-30 11:10:48 +030091
koder aka kdanilov0fdaaee2015-06-30 11:10:48 +030092 results.sort()
koder aka kdanilov22d134e2016-11-08 11:33:19 +020093 return [i[1:] for i in results]
koder aka kdanilov0fdaaee2015-06-30 11:10:48 +030094
95
koder aka kdanilov22d134e2016-11-08 11:33:19 +020096def log_nodes_statistic_stage(ctx: TestRun) -> None:
koder aka kdanilov108ac362017-01-19 20:17:16 +020097 log_nodes_statistic(ctx.nodes)
koder aka kdanilov0fdaaee2015-06-30 11:10:48 +030098
99
100def parse_args(argv):
101 descr = "Disk io performance test suite"
102 parser = argparse.ArgumentParser(prog='wally', description=descr)
koder aka kdanilov22d134e2016-11-08 11:33:19 +0200103 parser.add_argument("-l", '--log-level', help="print some extra log info")
koder aka kdanilov962ee5f2016-12-19 02:40:08 +0200104 parser.add_argument("--ssh-key-passwd", default=None, help="Pass ssh key password")
koder aka kdanilovf2865172016-12-30 03:35:11 +0200105 parser.add_argument("--ssh-key-passwd-kbd", action="store_true", help="Enter ssh key password interactivelly")
koder aka kdanilov3d2bc4f2016-11-12 18:31:18 +0200106 parser.add_argument("-s", '--settings-dir', default=None,
107 help="Folder to store key/settings/history files")
koder aka kdanilov0fdaaee2015-06-30 11:10:48 +0300108
109 subparsers = parser.add_subparsers(dest='subparser_name')
110
111 # ---------------------------------------------------------------------
koder aka kdanilov22d134e2016-11-08 11:33:19 +0200112 report_parser = subparsers.add_parser('ls', help='list all results')
koder aka kdanilov0fdaaee2015-06-30 11:10:48 +0300113 report_parser.add_argument("result_storage", help="Folder with test results")
114
115 # ---------------------------------------------------------------------
116 compare_help = 'compare two results'
117 report_parser = subparsers.add_parser('compare', help=compare_help)
118 report_parser.add_argument("data_path1", help="First folder with test results")
119 report_parser.add_argument("data_path2", help="Second folder with test results")
120
121 # ---------------------------------------------------------------------
122 report_help = 'run report on previously obtained results'
123 report_parser = subparsers.add_parser('report', help=report_help)
koder aka kdanilov0fdaaee2015-06-30 11:10:48 +0300124 report_parser.add_argument("data_dir", help="folder with rest results")
125
126 # ---------------------------------------------------------------------
koder aka kdanilov108ac362017-01-19 20:17:16 +0200127 ipython_help = 'run ipython in prepared environment'
128 ipython_parser = subparsers.add_parser('ipython', help=ipython_help)
129 ipython_parser.add_argument("storage_dir", help="Storage path")
130 # ---------------------------------------------------------------------
131 jupyter_help = 'run ipython in prepared environment'
132 jupyter_parser = subparsers.add_parser('jupyter', help=jupyter_help)
133 jupyter_parser.add_argument("storage_dir", help="Storage path")
134
135 # ---------------------------------------------------------------------
koder aka kdanilov0fdaaee2015-06-30 11:10:48 +0300136 test_parser = subparsers.add_parser('test', help='run tests')
koder aka kdanilov22d134e2016-11-08 11:33:19 +0200137 test_parser.add_argument('--build-description', type=str, default="Build info")
koder aka kdanilov0fdaaee2015-06-30 11:10:48 +0300138 test_parser.add_argument('--build-id', type=str, default="id")
139 test_parser.add_argument('--build-type', type=str, default="GA")
koder aka kdanilov39e449e2016-12-17 15:15:26 +0200140 test_parser.add_argument('--dont-collect', action='store_true', help="Don't collect cluster info")
koder aka kdanilov22d134e2016-11-08 11:33:19 +0200141 test_parser.add_argument('-n', '--no-tests', action='store_true', help="Don't run tests")
142 test_parser.add_argument('--load-report', action='store_true')
143 test_parser.add_argument("-k", '--keep-vm', action='store_true', help="Don't remove test vm's")
koder aka kdanilov0fdaaee2015-06-30 11:10:48 +0300144 test_parser.add_argument("-d", '--dont-discover-nodes', action='store_true',
koder aka kdanilov22d134e2016-11-08 11:33:19 +0200145 help="Don't connect/discover fuel nodes")
146 test_parser.add_argument('--no-report', action='store_true', help="Skip report stages")
koder aka kdanilov962ee5f2016-12-19 02:40:08 +0200147 test_parser.add_argument('--result-dir', default=None, help="Save results to DIR", metavar="DIR")
koder aka kdanilov0fdaaee2015-06-30 11:10:48 +0300148 test_parser.add_argument("comment", help="Test information")
koder aka kdanilov962ee5f2016-12-19 02:40:08 +0200149 test_parser.add_argument("config_file", help="Yaml config file")
koder aka kdanilov0fdaaee2015-06-30 11:10:48 +0300150
151 # ---------------------------------------------------------------------
koder aka kdanilove7e1a4d2016-12-17 20:29:52 +0200152 test_parser = subparsers.add_parser('resume', help='resume tests')
153 test_parser.add_argument("storage_dir", help="Path to test directory")
koder aka kdanilov0fdaaee2015-06-30 11:10:48 +0300154
koder aka kdanilov7f59d562016-12-26 01:34:23 +0200155 # ---------------------------------------------------------------------
koder aka kdanilov108ac362017-01-19 20:17:16 +0200156 test_parser = subparsers.add_parser('db', help='Exec command on DB')
koder aka kdanilov7f59d562016-12-26 01:34:23 +0200157 test_parser.add_argument("cmd", choices=("show",), help="Command to execute")
158 test_parser.add_argument("params", nargs='*', help="Command params")
159 test_parser.add_argument("storage_dir", help="Storage path")
160
koder aka kdanilov0fdaaee2015-06-30 11:10:48 +0300161 return parser.parse_args(argv[1:])
162
163
koder aka kdanilov3d2bc4f2016-11-12 18:31:18 +0200164def get_config_path(config: Config, opts_value: Optional[str]) -> str:
165 if opts_value is None and 'settings_dir' not in config:
166 val = "~/.wally"
167 elif opts_value is not None:
168 val = opts_value
169 else:
170 val = config.settings_dir
171
172 return os.path.abspath(os.path.expanduser(val))
173
174
koder aka kdanilov962ee5f2016-12-19 02:40:08 +0200175def find_cfg_file(name: str, included_from: str = None) -> str:
176 paths = [".", os.path.expanduser('~/.wally')]
177 if included_from is not None:
178 paths.append(os.path.dirname(included_from))
179
180 search_paths = set(os.path.abspath(path) for path in paths if os.path.isdir(path))
181
182 for folder in search_paths:
183 path = os.path.join(folder, name)
184 if os.path.exists(path):
185 return path
186
187 raise FileNotFoundError(name)
188
189
190def load_config(path: str) -> Config:
191 path = os.path.abspath(path)
192 cfg_dict = yaml_load(open(path).read())
193
194 while 'include' in cfg_dict:
195 inc = cfg_dict.pop('include')
196 if isinstance(inc, str):
197 inc = [inc]
198
199 for fname in inc:
200 inc_path = find_cfg_file(fname, path)
201 inc_dict = yaml_load(open(inc_path).read())
202 inc_dict.update(cfg_dict)
203 cfg_dict = inc_dict
204
205 return Config(cfg_dict)
206
207
koder aka kdanilov7f59d562016-12-26 01:34:23 +0200208def get_run_stages() -> List[Stage]:
209 return [DiscoverCephStage(),
210 DiscoverOSStage(),
211 DiscoverFuelStage(),
212 ExplicitNodesStage(),
213 StartSensorsStage(),
214 RunTestsStage(),
215 CollectSensorsStage(),
216 ConnectStage(),
217 SleepStage(),
218 PrepareNodes()]
219
220
koder aka kdanilov108ac362017-01-19 20:17:16 +0200221notebook_kern = """
222{
223 "cells": [
224 {
225 "cell_type": "code",
226 "execution_count": 1,
227 "metadata": {
228 "collapsed": true
229 },
230 "outputs": [],
231 "source": [
232 "from wally.storage import make_storage\n",
233 "from wally.hlstorage import ResultStorage\n"
234 "storage = make_storage(\"$STORAGE\", existing=True)\n",
235 "rstorage = ResultStorage(storage=storage)\n"
236 ]
237 }
238 ],
239 "metadata": {
240 "kernelspec": {
241 "display_name": "Python 3",
242 "language": "python",
243 "name": "python3"
244 },
245 "language_info": {
246 "codemirror_mode": {
247 "name": "ipython",
248 "version": 3
249 },
250 "file_extension": ".py",
251 "mimetype": "text/x-python",
252 "name": "python",
253 "nbconvert_exporter": "python",
254 "pygments_lexer": "ipython3",
255 "version": "3.5.2"
256 }
257 },
258 "nbformat": 4,
259 "nbformat_minor": 2
260}"""
261
262
koder aka kdanilov22d134e2016-11-08 11:33:19 +0200263def main(argv: List[str]) -> int:
koder aka kdanilov0fdaaee2015-06-30 11:10:48 +0300264 if faulthandler is not None:
265 faulthandler.register(signal.SIGUSR1, all_threads=True)
266
267 opts = parse_args(argv)
koder aka kdanilov39e449e2016-12-17 15:15:26 +0200268 stages = [] # type: List[Stage]
koder aka kdanilov22d134e2016-11-08 11:33:19 +0200269
270 # stop mypy from telling that config & storage might be undeclared
271 config = None # type: Config
koder aka kdanilov3d2bc4f2016-11-12 18:31:18 +0200272 storage = None # type: Storage
koder aka kdanilov0fdaaee2015-06-30 11:10:48 +0300273
274 if opts.subparser_name == 'test':
koder aka kdanilov962ee5f2016-12-19 02:40:08 +0200275 config = load_config(opts.config_file)
koder aka kdanilove7e1a4d2016-12-17 20:29:52 +0200276 config.storage_url, config.run_uuid = utils.get_uniq_path_uuid(config.results_dir)
277 config.comment = opts.comment
278 config.keep_vm = opts.keep_vm
279 config.no_tests = opts.no_tests
280 config.dont_discover_nodes = opts.dont_discover_nodes
281 config.build_id = opts.build_id
282 config.build_description = opts.build_description
283 config.build_type = opts.build_type
284 config.settings_dir = get_config_path(config, opts.settings_dir)
koder aka kdanilov0fdaaee2015-06-30 11:10:48 +0300285
koder aka kdanilove7e1a4d2016-12-17 20:29:52 +0200286 storage = make_storage(config.storage_url)
koder aka kdanilov7f59d562016-12-26 01:34:23 +0200287 storage.put(config, 'config')
koder aka kdanilovffaf48d2016-12-27 02:25:29 +0200288
koder aka kdanilov7f59d562016-12-26 01:34:23 +0200289 stages.extend(get_run_stages())
koder aka kdanilovffaf48d2016-12-27 02:25:29 +0200290 stages.append(SaveNodesStage())
koder aka kdanilov0fdaaee2015-06-30 11:10:48 +0300291
koder aka kdanilov39e449e2016-12-17 15:15:26 +0200292 if not opts.dont_collect:
koder aka kdanilov962ee5f2016-12-19 02:40:08 +0200293 stages.append(CollectInfoStage())
koder aka kdanilov0fdaaee2015-06-30 11:10:48 +0300294
koder aka kdanilov962ee5f2016-12-19 02:40:08 +0200295 argv2 = argv[:]
296 if '--ssh-key-passwd' in argv2:
297 # don't save ssh key password to storage
298 argv2[argv2.index("--ssh-key-passwd") + 1] = "<removed from output>"
koder aka kdanilov7f59d562016-12-26 01:34:23 +0200299 storage.put(argv2, 'cli')
koder aka kdanilove7e1a4d2016-12-17 20:29:52 +0200300
301 elif opts.subparser_name == 'resume':
koder aka kdanilov7f59d562016-12-26 01:34:23 +0200302 opts.resumed = True
koder aka kdanilove7e1a4d2016-12-17 20:29:52 +0200303 storage = make_storage(opts.storage_dir, existing=True)
304 config = storage.load(Config, 'config')
koder aka kdanilov7f59d562016-12-26 01:34:23 +0200305 stages.extend(get_run_stages())
306 stages.append(LoadStoredNodesStage())
koder aka kdanilov108ac362017-01-19 20:17:16 +0200307 prev_opts = storage.get('cli') # type: List[str]
308
koder aka kdanilov7f59d562016-12-26 01:34:23 +0200309 if '--ssh-key-passwd' in prev_opts and opts.ssh_key_passwd:
310 prev_opts[prev_opts.index("--ssh-key-passwd") + 1] = opts.ssh_key_passwd
311
312 restored_opts = parse_args(prev_opts)
313 opts.__dict__.update(restored_opts.__dict__)
314 opts.subparser_name = 'resume'
koder aka kdanilove7e1a4d2016-12-17 20:29:52 +0200315
koder aka kdanilov0fdaaee2015-06-30 11:10:48 +0300316 elif opts.subparser_name == 'ls':
koder aka kdanilov22d134e2016-11-08 11:33:19 +0200317 tab = texttable.Texttable(max_width=200)
318 tab.set_deco(tab.HEADER | tab.VLINES | tab.BORDER)
319 tab.set_cols_align(["l", "l", "l", "l"])
320 tab.header(["Name", "Tests", "Run at", "Comment"])
321 tab.add_rows(list_results(opts.result_storage))
322 print(tab.draw())
koder aka kdanilov0fdaaee2015-06-30 11:10:48 +0300323 return 0
324
325 elif opts.subparser_name == 'report':
koder aka kdanilovffaf48d2016-12-27 02:25:29 +0200326 if getattr(opts, "no_report", False):
327 print(" --no-report option can't be used with 'report' cmd")
328 return 1
koder aka kdanilov22d134e2016-11-08 11:33:19 +0200329 storage = make_storage(opts.data_dir, existing=True)
koder aka kdanilovffaf48d2016-12-27 02:25:29 +0200330 config = storage.load(Config, 'config')
331 stages.append(LoadStoredNodesStage())
koder aka kdanilov0fdaaee2015-06-30 11:10:48 +0300332
333 elif opts.subparser_name == 'compare':
koder aka kdanilov73084622016-11-16 21:51:08 +0200334 # x = run_test.load_data_from_path(opts.data_path1)
335 # y = run_test.load_data_from_path(opts.data_path2)
336 # print(run_test.IOPerfTest.format_diff_for_console(
337 # [x['io'][0], y['io'][0]]))
koder aka kdanilov0fdaaee2015-06-30 11:10:48 +0300338 return 0
339
koder aka kdanilov7f59d562016-12-26 01:34:23 +0200340 elif opts.subparser_name == 'db':
341 storage = make_storage(opts.storage_dir, existing=True)
342 if opts.cmd == 'show':
343 if len(opts.params) != 1:
344 print("'show' command requires parameter - key to show")
345 return 1
346 pprint.pprint(storage.get(opts.params[0]))
347 else:
348 print("Unknown/not_implemented command {!r}".format(opts.cmd))
349 return 1
350 return 0
koder aka kdanilov108ac362017-01-19 20:17:16 +0200351 elif opts.subparser_name == 'ipython':
352 storage = make_storage(opts.storage_dir, existing=True)
353 from .hlstorage import ResultStorage
354 rstorage = ResultStorage(storage=storage)
355
356 import IPython
357 IPython.embed()
358
359 return 0
koder aka kdanilova732a602017-02-01 20:29:56 +0200360 # elif opts.subparser_name == 'jupyter':
361 # with tempfile.NamedTemporaryFile() as fd:
362 # fd.write(notebook_kern.replace("$STORAGE", opts.storage_dir))
363 # subprocess.call("jupyter notebook ", shell=True)
364 # return 0
365 else:
366 print("Subparser {!r} is not supported".format(opts.subparser_name))
367 return 1
koder aka kdanilov7f59d562016-12-26 01:34:23 +0200368
koder aka kdanilov39e449e2016-12-17 15:15:26 +0200369 report_stages = [] # type: List[Stage]
koder aka kdanilov22d134e2016-11-08 11:33:19 +0200370 if not getattr(opts, "no_report", False):
koder aka kdanilov108ac362017-01-19 20:17:16 +0200371 # report_stages.append(CalcStatisticStage())
372 # report_stages.append(ConsoleReportStage())
koder aka kdanilov962ee5f2016-12-19 02:40:08 +0200373 report_stages.append(HtmlReportStage())
koder aka kdanilov0fdaaee2015-06-30 11:10:48 +0300374
koder aka kdanilov22d134e2016-11-08 11:33:19 +0200375 # log level is not a part of config
koder aka kdanilov0fdaaee2015-06-30 11:10:48 +0300376 if opts.log_level is not None:
377 str_level = opts.log_level
378 else:
koder aka kdanilov22d134e2016-11-08 11:33:19 +0200379 str_level = config.get('logging/log_level', 'INFO')
koder aka kdanilov0fdaaee2015-06-30 11:10:48 +0300380
koder aka kdanilov962ee5f2016-12-19 02:40:08 +0200381 log_config_file = config.get('logging/config', None)
382
383 if log_config_file is not None:
384 log_config_file = find_cfg_file(log_config_file, opts.config_file)
385
386 setup_loggers(getattr(logging, str_level),
koder aka kdanilov7f59d562016-12-26 01:34:23 +0200387 log_fd=storage.get_fd('log', "w"),
koder aka kdanilov962ee5f2016-12-19 02:40:08 +0200388 config_file=log_config_file)
389
koder aka kdanilov22d134e2016-11-08 11:33:19 +0200390 logger.info("All info would be stored into %r", config.storage_url)
391
392 ctx = TestRun(config, storage)
koder aka kdanilove7e1a4d2016-12-17 20:29:52 +0200393 ctx.rpc_code, ctx.default_rpc_plugins = node.get_rpc_server_code()
koder aka kdanilov0fdaaee2015-06-30 11:10:48 +0300394
koder aka kdanilov962ee5f2016-12-19 02:40:08 +0200395 if opts.ssh_key_passwd is not None:
396 set_ssh_key_passwd(opts.ssh_key_passwd)
koder aka kdanilovf2865172016-12-30 03:35:11 +0200397 elif opts.ssh_key_passwd_kbd:
398 set_ssh_key_passwd(getpass.getpass("Ssh key password: ").strip())
koder aka kdanilov962ee5f2016-12-19 02:40:08 +0200399
koder aka kdanilov39e449e2016-12-17 15:15:26 +0200400 stages.sort(key=lambda x: x.priority)
401
koder aka kdanilov962ee5f2016-12-19 02:40:08 +0200402 # TODO: run only stages, which have config
koder aka kdanilov39e449e2016-12-17 15:15:26 +0200403 failed = False
404 cleanup_stages = []
koder aka kdanilov7f59d562016-12-26 01:34:23 +0200405
koder aka kdanilov0fdaaee2015-06-30 11:10:48 +0300406 for stage in stages:
koder aka kdanilove7e1a4d2016-12-17 20:29:52 +0200407 if stage.config_block is not None:
408 if stage.config_block not in ctx.config:
409 continue
410
411 cleanup_stages.append(stage)
koder aka kdanilov39e449e2016-12-17 15:15:26 +0200412 try:
koder aka kdanilov39e449e2016-12-17 15:15:26 +0200413 with log_stage(stage):
414 stage.run(ctx)
koder aka kdanilov962ee5f2016-12-19 02:40:08 +0200415 except (Exception, KeyboardInterrupt):
koder aka kdanilov39e449e2016-12-17 15:15:26 +0200416 failed = True
koder aka kdanilov0fdaaee2015-06-30 11:10:48 +0300417 break
koder aka kdanilov7f59d562016-12-26 01:34:23 +0200418 ctx.storage.sync()
419 ctx.storage.sync()
koder aka kdanilov0fdaaee2015-06-30 11:10:48 +0300420
koder aka kdanilov39e449e2016-12-17 15:15:26 +0200421 logger.debug("Start cleanup")
422 cleanup_failed = False
423 for stage in cleanup_stages[::-1]:
424 try:
koder aka kdanilov962ee5f2016-12-19 02:40:08 +0200425 with log_stage(stage, cleanup=True):
koder aka kdanilov39e449e2016-12-17 15:15:26 +0200426 stage.cleanup(ctx)
427 except:
428 cleanup_failed = True
koder aka kdanilov7f59d562016-12-26 01:34:23 +0200429 ctx.storage.sync()
koder aka kdanilov0fdaaee2015-06-30 11:10:48 +0300430
koder aka kdanilov39e449e2016-12-17 15:15:26 +0200431 if not failed:
koder aka kdanilov0fdaaee2015-06-30 11:10:48 +0300432 for report_stage in report_stages:
433 with log_stage(report_stage):
koder aka kdanilovffaf48d2016-12-27 02:25:29 +0200434 try:
435 report_stage.run(ctx)
436 except utils.StopTestError:
437 logger.error("Report stage %s requested stop execution", report_stage.name())
438 failed = True
439 break
koder aka kdanilov0fdaaee2015-06-30 11:10:48 +0300440
koder aka kdanilov7f59d562016-12-26 01:34:23 +0200441 ctx.storage.sync()
442
koder aka kdanilov22d134e2016-11-08 11:33:19 +0200443 logger.info("All info is stored into %r", config.storage_url)
koder aka kdanilov0fdaaee2015-06-30 11:10:48 +0300444
koder aka kdanilov39e449e2016-12-17 15:15:26 +0200445 if failed or cleanup_failed:
koder aka kdanilov22d134e2016-11-08 11:33:19 +0200446 logger.error("Tests are failed. See error details in log above")
koder aka kdanilov0fdaaee2015-06-30 11:10:48 +0300447 return 1
koder aka kdanilov39e449e2016-12-17 15:15:26 +0200448 else:
449 logger.info("Tests finished successfully")
450 return 0