koder aka kdanilov | 0fdaaee | 2015-06-30 11:10:48 +0300 | [diff] [blame] | 1 | import os |
koder aka kdanilov | 0fdaaee | 2015-06-30 11:10:48 +0300 | [diff] [blame] | 2 | import time |
| 3 | import signal |
koder aka kdanilov | 7f59d56 | 2016-12-26 01:34:23 +0200 | [diff] [blame] | 4 | import pprint |
koder aka kdanilov | f286517 | 2016-12-30 03:35:11 +0200 | [diff] [blame] | 5 | import getpass |
koder aka kdanilov | 0fdaaee | 2015-06-30 11:10:48 +0300 | [diff] [blame] | 6 | import logging |
koder aka kdanilov | 108ac36 | 2017-01-19 20:17:16 +0200 | [diff] [blame] | 7 | import tempfile |
koder aka kdanilov | 0fdaaee | 2015-06-30 11:10:48 +0300 | [diff] [blame] | 8 | import argparse |
| 9 | import functools |
koder aka kdanilov | 108ac36 | 2017-01-19 20:17:16 +0200 | [diff] [blame] | 10 | import subprocess |
koder aka kdanilov | 39e449e | 2016-12-17 15:15:26 +0200 | [diff] [blame] | 11 | import contextlib |
| 12 | from typing import List, Tuple, Any, Callable, IO, cast, Optional, Iterator |
koder aka kdanilov | 0fdaaee | 2015-06-30 11:10:48 +0300 | [diff] [blame] | 13 | from yaml import load as _yaml_load |
| 14 | |
koder aka kdanilov | 22d134e | 2016-11-08 11:33:19 +0200 | [diff] [blame] | 15 | |
| 16 | YLoader = Callable[[IO], Any] |
| 17 | yaml_load = None # type: YLoader |
| 18 | |
| 19 | |
koder aka kdanilov | 0fdaaee | 2015-06-30 11:10:48 +0300 | [diff] [blame] | 20 | try: |
| 21 | from yaml import CLoader |
koder aka kdanilov | 22d134e | 2016-11-08 11:33:19 +0200 | [diff] [blame] | 22 | yaml_load = cast(YLoader, functools.partial(_yaml_load, Loader=CLoader)) |
koder aka kdanilov | 0fdaaee | 2015-06-30 11:10:48 +0300 | [diff] [blame] | 23 | except ImportError: |
koder aka kdanilov | 22d134e | 2016-11-08 11:33:19 +0200 | [diff] [blame] | 24 | yaml_load = cast(YLoader, _yaml_load) |
koder aka kdanilov | 0fdaaee | 2015-06-30 11:10:48 +0300 | [diff] [blame] | 25 | |
| 26 | |
| 27 | import texttable |
| 28 | |
| 29 | try: |
| 30 | import faulthandler |
| 31 | except ImportError: |
| 32 | faulthandler = None |
| 33 | |
koder aka kdanilov | e7e1a4d | 2016-12-17 20:29:52 +0200 | [diff] [blame] | 34 | from . import utils, node |
koder aka kdanilov | 108ac36 | 2017-01-19 20:17:16 +0200 | [diff] [blame] | 35 | from .node_utils import log_nodes_statistic |
koder aka kdanilov | 3d2bc4f | 2016-11-12 18:31:18 +0200 | [diff] [blame] | 36 | from .storage import make_storage, Storage |
koder aka kdanilov | 22d134e | 2016-11-08 11:33:19 +0200 | [diff] [blame] | 37 | from .config import Config |
koder aka kdanilov | 3b4da8b | 2016-10-17 00:17:53 +0300 | [diff] [blame] | 38 | from .logger import setup_loggers |
koder aka kdanilov | 39e449e | 2016-12-17 15:15:26 +0200 | [diff] [blame] | 39 | from .stage import Stage |
koder aka kdanilov | 22d134e | 2016-11-08 11:33:19 +0200 | [diff] [blame] | 40 | from .test_run_class import TestRun |
koder aka kdanilov | 962ee5f | 2016-12-19 02:40:08 +0200 | [diff] [blame] | 41 | from .ssh import set_ssh_key_passwd |
koder aka kdanilov | 0fdaaee | 2015-06-30 11:10:48 +0300 | [diff] [blame] | 42 | |
| 43 | |
koder aka kdanilov | 39e449e | 2016-12-17 15:15:26 +0200 | [diff] [blame] | 44 | # stages |
| 45 | from .ceph import DiscoverCephStage |
| 46 | from .openstack import DiscoverOSStage |
| 47 | from .fuel import DiscoverFuelStage |
koder aka kdanilov | 23e6bdf | 2016-12-24 02:18:54 +0200 | [diff] [blame] | 48 | from .run_test import (CollectInfoStage, ExplicitNodesStage, SaveNodesStage, |
koder aka kdanilov | 7f59d56 | 2016-12-26 01:34:23 +0200 | [diff] [blame] | 49 | RunTestsStage, ConnectStage, SleepStage, PrepareNodes, |
| 50 | LoadStoredNodesStage) |
koder aka kdanilov | a732a60 | 2017-02-01 20:29:56 +0200 | [diff] [blame] | 51 | # from .process_results import CalcStatisticStage |
koder aka kdanilov | 39e449e | 2016-12-17 15:15:26 +0200 | [diff] [blame] | 52 | from .report import ConsoleReportStage, HtmlReportStage |
| 53 | from .sensors import StartSensorsStage, CollectSensorsStage |
| 54 | |
| 55 | |
koder aka kdanilov | 0fdaaee | 2015-06-30 11:10:48 +0300 | [diff] [blame] | 56 | logger = logging.getLogger("wally") |
| 57 | |
| 58 | |
koder aka kdanilov | 39e449e | 2016-12-17 15:15:26 +0200 | [diff] [blame] | 59 | @contextlib.contextmanager |
koder aka kdanilov | 962ee5f | 2016-12-19 02:40:08 +0200 | [diff] [blame] | 60 | def log_stage(stage: Stage, cleanup: bool = False) -> Iterator[None]: |
| 61 | logger.info("Start " + stage.name() + ("::cleanup" if cleanup else "")) |
koder aka kdanilov | 39e449e | 2016-12-17 15:15:26 +0200 | [diff] [blame] | 62 | try: |
| 63 | yield |
| 64 | except utils.StopTestError as exc: |
koder aka kdanilov | 962ee5f | 2016-12-19 02:40:08 +0200 | [diff] [blame] | 65 | raise |
koder aka kdanilov | 39e449e | 2016-12-17 15:15:26 +0200 | [diff] [blame] | 66 | except Exception: |
koder aka kdanilov | 962ee5f | 2016-12-19 02:40:08 +0200 | [diff] [blame] | 67 | logger.exception("During %s", stage.name() + ("::cleanup" if cleanup else "")) |
| 68 | raise |
koder aka kdanilov | 39e449e | 2016-12-17 15:15:26 +0200 | [diff] [blame] | 69 | |
| 70 | |
koder aka kdanilov | 22d134e | 2016-11-08 11:33:19 +0200 | [diff] [blame] | 71 | def list_results(path: str) -> List[Tuple[str, str, str, str]]: |
koder aka kdanilov | 7308462 | 2016-11-16 21:51:08 +0200 | [diff] [blame] | 72 | results = [] # type: List[Tuple[float, str, str, str, str]] |
koder aka kdanilov | 0fdaaee | 2015-06-30 11:10:48 +0300 | [diff] [blame] | 73 | |
koder aka kdanilov | 22d134e | 2016-11-08 11:33:19 +0200 | [diff] [blame] | 74 | for dir_name in os.listdir(path): |
| 75 | full_path = os.path.join(path, dir_name) |
| 76 | |
koder aka kdanilov | 0fdaaee | 2015-06-30 11:10:48 +0300 | [diff] [blame] | 77 | try: |
koder aka kdanilov | 22d134e | 2016-11-08 11:33:19 +0200 | [diff] [blame] | 78 | stor = make_storage(full_path, existing=True) |
| 79 | except Exception as exc: |
| 80 | logger.warning("Can't load folder {}. Error {}".format(full_path, exc)) |
koder aka kdanilov | 0fdaaee | 2015-06-30 11:10:48 +0300 | [diff] [blame] | 81 | |
koder aka kdanilov | 7f59d56 | 2016-12-26 01:34:23 +0200 | [diff] [blame] | 82 | comment = cast(str, stor.get('info/comment')) |
| 83 | run_uuid = cast(str, stor.get('info/run_uuid')) |
| 84 | run_time = cast(float, stor.get('info/run_time')) |
koder aka kdanilov | 22d134e | 2016-11-08 11:33:19 +0200 | [diff] [blame] | 85 | test_types = "" |
koder aka kdanilov | 7308462 | 2016-11-16 21:51:08 +0200 | [diff] [blame] | 86 | results.append((run_time, |
koder aka kdanilov | 22d134e | 2016-11-08 11:33:19 +0200 | [diff] [blame] | 87 | run_uuid, |
| 88 | test_types, |
koder aka kdanilov | 7308462 | 2016-11-16 21:51:08 +0200 | [diff] [blame] | 89 | time.ctime(run_time), |
koder aka kdanilov | 22d134e | 2016-11-08 11:33:19 +0200 | [diff] [blame] | 90 | '-' if comment is None else comment)) |
koder aka kdanilov | 0fdaaee | 2015-06-30 11:10:48 +0300 | [diff] [blame] | 91 | |
koder aka kdanilov | 0fdaaee | 2015-06-30 11:10:48 +0300 | [diff] [blame] | 92 | results.sort() |
koder aka kdanilov | 22d134e | 2016-11-08 11:33:19 +0200 | [diff] [blame] | 93 | return [i[1:] for i in results] |
koder aka kdanilov | 0fdaaee | 2015-06-30 11:10:48 +0300 | [diff] [blame] | 94 | |
| 95 | |
koder aka kdanilov | 22d134e | 2016-11-08 11:33:19 +0200 | [diff] [blame] | 96 | def log_nodes_statistic_stage(ctx: TestRun) -> None: |
koder aka kdanilov | 108ac36 | 2017-01-19 20:17:16 +0200 | [diff] [blame] | 97 | log_nodes_statistic(ctx.nodes) |
koder aka kdanilov | 0fdaaee | 2015-06-30 11:10:48 +0300 | [diff] [blame] | 98 | |
| 99 | |
| 100 | def parse_args(argv): |
| 101 | descr = "Disk io performance test suite" |
| 102 | parser = argparse.ArgumentParser(prog='wally', description=descr) |
koder aka kdanilov | 22d134e | 2016-11-08 11:33:19 +0200 | [diff] [blame] | 103 | parser.add_argument("-l", '--log-level', help="print some extra log info") |
koder aka kdanilov | 962ee5f | 2016-12-19 02:40:08 +0200 | [diff] [blame] | 104 | parser.add_argument("--ssh-key-passwd", default=None, help="Pass ssh key password") |
koder aka kdanilov | f286517 | 2016-12-30 03:35:11 +0200 | [diff] [blame] | 105 | parser.add_argument("--ssh-key-passwd-kbd", action="store_true", help="Enter ssh key password interactivelly") |
koder aka kdanilov | 3d2bc4f | 2016-11-12 18:31:18 +0200 | [diff] [blame] | 106 | parser.add_argument("-s", '--settings-dir', default=None, |
| 107 | help="Folder to store key/settings/history files") |
koder aka kdanilov | 0fdaaee | 2015-06-30 11:10:48 +0300 | [diff] [blame] | 108 | |
| 109 | subparsers = parser.add_subparsers(dest='subparser_name') |
| 110 | |
| 111 | # --------------------------------------------------------------------- |
koder aka kdanilov | 22d134e | 2016-11-08 11:33:19 +0200 | [diff] [blame] | 112 | report_parser = subparsers.add_parser('ls', help='list all results') |
koder aka kdanilov | 0fdaaee | 2015-06-30 11:10:48 +0300 | [diff] [blame] | 113 | report_parser.add_argument("result_storage", help="Folder with test results") |
| 114 | |
| 115 | # --------------------------------------------------------------------- |
| 116 | compare_help = 'compare two results' |
| 117 | report_parser = subparsers.add_parser('compare', help=compare_help) |
| 118 | report_parser.add_argument("data_path1", help="First folder with test results") |
| 119 | report_parser.add_argument("data_path2", help="Second folder with test results") |
| 120 | |
| 121 | # --------------------------------------------------------------------- |
| 122 | report_help = 'run report on previously obtained results' |
| 123 | report_parser = subparsers.add_parser('report', help=report_help) |
koder aka kdanilov | 0fdaaee | 2015-06-30 11:10:48 +0300 | [diff] [blame] | 124 | report_parser.add_argument("data_dir", help="folder with rest results") |
| 125 | |
| 126 | # --------------------------------------------------------------------- |
koder aka kdanilov | 108ac36 | 2017-01-19 20:17:16 +0200 | [diff] [blame] | 127 | ipython_help = 'run ipython in prepared environment' |
| 128 | ipython_parser = subparsers.add_parser('ipython', help=ipython_help) |
| 129 | ipython_parser.add_argument("storage_dir", help="Storage path") |
| 130 | # --------------------------------------------------------------------- |
| 131 | jupyter_help = 'run ipython in prepared environment' |
| 132 | jupyter_parser = subparsers.add_parser('jupyter', help=jupyter_help) |
| 133 | jupyter_parser.add_argument("storage_dir", help="Storage path") |
| 134 | |
| 135 | # --------------------------------------------------------------------- |
koder aka kdanilov | 0fdaaee | 2015-06-30 11:10:48 +0300 | [diff] [blame] | 136 | test_parser = subparsers.add_parser('test', help='run tests') |
koder aka kdanilov | 22d134e | 2016-11-08 11:33:19 +0200 | [diff] [blame] | 137 | test_parser.add_argument('--build-description', type=str, default="Build info") |
koder aka kdanilov | 0fdaaee | 2015-06-30 11:10:48 +0300 | [diff] [blame] | 138 | test_parser.add_argument('--build-id', type=str, default="id") |
| 139 | test_parser.add_argument('--build-type', type=str, default="GA") |
koder aka kdanilov | 39e449e | 2016-12-17 15:15:26 +0200 | [diff] [blame] | 140 | test_parser.add_argument('--dont-collect', action='store_true', help="Don't collect cluster info") |
koder aka kdanilov | 22d134e | 2016-11-08 11:33:19 +0200 | [diff] [blame] | 141 | test_parser.add_argument('-n', '--no-tests', action='store_true', help="Don't run tests") |
| 142 | test_parser.add_argument('--load-report', action='store_true') |
| 143 | test_parser.add_argument("-k", '--keep-vm', action='store_true', help="Don't remove test vm's") |
koder aka kdanilov | 0fdaaee | 2015-06-30 11:10:48 +0300 | [diff] [blame] | 144 | test_parser.add_argument("-d", '--dont-discover-nodes', action='store_true', |
koder aka kdanilov | 22d134e | 2016-11-08 11:33:19 +0200 | [diff] [blame] | 145 | help="Don't connect/discover fuel nodes") |
| 146 | test_parser.add_argument('--no-report', action='store_true', help="Skip report stages") |
koder aka kdanilov | 962ee5f | 2016-12-19 02:40:08 +0200 | [diff] [blame] | 147 | test_parser.add_argument('--result-dir', default=None, help="Save results to DIR", metavar="DIR") |
koder aka kdanilov | 0fdaaee | 2015-06-30 11:10:48 +0300 | [diff] [blame] | 148 | test_parser.add_argument("comment", help="Test information") |
koder aka kdanilov | 962ee5f | 2016-12-19 02:40:08 +0200 | [diff] [blame] | 149 | test_parser.add_argument("config_file", help="Yaml config file") |
koder aka kdanilov | 0fdaaee | 2015-06-30 11:10:48 +0300 | [diff] [blame] | 150 | |
| 151 | # --------------------------------------------------------------------- |
koder aka kdanilov | e7e1a4d | 2016-12-17 20:29:52 +0200 | [diff] [blame] | 152 | test_parser = subparsers.add_parser('resume', help='resume tests') |
| 153 | test_parser.add_argument("storage_dir", help="Path to test directory") |
koder aka kdanilov | 0fdaaee | 2015-06-30 11:10:48 +0300 | [diff] [blame] | 154 | |
koder aka kdanilov | 7f59d56 | 2016-12-26 01:34:23 +0200 | [diff] [blame] | 155 | # --------------------------------------------------------------------- |
koder aka kdanilov | 108ac36 | 2017-01-19 20:17:16 +0200 | [diff] [blame] | 156 | test_parser = subparsers.add_parser('db', help='Exec command on DB') |
koder aka kdanilov | 7f59d56 | 2016-12-26 01:34:23 +0200 | [diff] [blame] | 157 | test_parser.add_argument("cmd", choices=("show",), help="Command to execute") |
| 158 | test_parser.add_argument("params", nargs='*', help="Command params") |
| 159 | test_parser.add_argument("storage_dir", help="Storage path") |
| 160 | |
koder aka kdanilov | 0fdaaee | 2015-06-30 11:10:48 +0300 | [diff] [blame] | 161 | return parser.parse_args(argv[1:]) |
| 162 | |
| 163 | |
koder aka kdanilov | 3d2bc4f | 2016-11-12 18:31:18 +0200 | [diff] [blame] | 164 | def get_config_path(config: Config, opts_value: Optional[str]) -> str: |
| 165 | if opts_value is None and 'settings_dir' not in config: |
| 166 | val = "~/.wally" |
| 167 | elif opts_value is not None: |
| 168 | val = opts_value |
| 169 | else: |
| 170 | val = config.settings_dir |
| 171 | |
| 172 | return os.path.abspath(os.path.expanduser(val)) |
| 173 | |
| 174 | |
koder aka kdanilov | 962ee5f | 2016-12-19 02:40:08 +0200 | [diff] [blame] | 175 | def find_cfg_file(name: str, included_from: str = None) -> str: |
| 176 | paths = [".", os.path.expanduser('~/.wally')] |
| 177 | if included_from is not None: |
| 178 | paths.append(os.path.dirname(included_from)) |
| 179 | |
| 180 | search_paths = set(os.path.abspath(path) for path in paths if os.path.isdir(path)) |
| 181 | |
| 182 | for folder in search_paths: |
| 183 | path = os.path.join(folder, name) |
| 184 | if os.path.exists(path): |
| 185 | return path |
| 186 | |
| 187 | raise FileNotFoundError(name) |
| 188 | |
| 189 | |
| 190 | def load_config(path: str) -> Config: |
| 191 | path = os.path.abspath(path) |
| 192 | cfg_dict = yaml_load(open(path).read()) |
| 193 | |
| 194 | while 'include' in cfg_dict: |
| 195 | inc = cfg_dict.pop('include') |
| 196 | if isinstance(inc, str): |
| 197 | inc = [inc] |
| 198 | |
| 199 | for fname in inc: |
| 200 | inc_path = find_cfg_file(fname, path) |
| 201 | inc_dict = yaml_load(open(inc_path).read()) |
| 202 | inc_dict.update(cfg_dict) |
| 203 | cfg_dict = inc_dict |
| 204 | |
| 205 | return Config(cfg_dict) |
| 206 | |
| 207 | |
koder aka kdanilov | 7f59d56 | 2016-12-26 01:34:23 +0200 | [diff] [blame] | 208 | def get_run_stages() -> List[Stage]: |
| 209 | return [DiscoverCephStage(), |
| 210 | DiscoverOSStage(), |
| 211 | DiscoverFuelStage(), |
| 212 | ExplicitNodesStage(), |
| 213 | StartSensorsStage(), |
| 214 | RunTestsStage(), |
| 215 | CollectSensorsStage(), |
| 216 | ConnectStage(), |
| 217 | SleepStage(), |
| 218 | PrepareNodes()] |
| 219 | |
| 220 | |
koder aka kdanilov | 108ac36 | 2017-01-19 20:17:16 +0200 | [diff] [blame] | 221 | notebook_kern = """ |
| 222 | { |
| 223 | "cells": [ |
| 224 | { |
| 225 | "cell_type": "code", |
| 226 | "execution_count": 1, |
| 227 | "metadata": { |
| 228 | "collapsed": true |
| 229 | }, |
| 230 | "outputs": [], |
| 231 | "source": [ |
| 232 | "from wally.storage import make_storage\n", |
| 233 | "from wally.hlstorage import ResultStorage\n" |
| 234 | "storage = make_storage(\"$STORAGE\", existing=True)\n", |
| 235 | "rstorage = ResultStorage(storage=storage)\n" |
| 236 | ] |
| 237 | } |
| 238 | ], |
| 239 | "metadata": { |
| 240 | "kernelspec": { |
| 241 | "display_name": "Python 3", |
| 242 | "language": "python", |
| 243 | "name": "python3" |
| 244 | }, |
| 245 | "language_info": { |
| 246 | "codemirror_mode": { |
| 247 | "name": "ipython", |
| 248 | "version": 3 |
| 249 | }, |
| 250 | "file_extension": ".py", |
| 251 | "mimetype": "text/x-python", |
| 252 | "name": "python", |
| 253 | "nbconvert_exporter": "python", |
| 254 | "pygments_lexer": "ipython3", |
| 255 | "version": "3.5.2" |
| 256 | } |
| 257 | }, |
| 258 | "nbformat": 4, |
| 259 | "nbformat_minor": 2 |
| 260 | }""" |
| 261 | |
| 262 | |
koder aka kdanilov | 22d134e | 2016-11-08 11:33:19 +0200 | [diff] [blame] | 263 | def main(argv: List[str]) -> int: |
koder aka kdanilov | 0fdaaee | 2015-06-30 11:10:48 +0300 | [diff] [blame] | 264 | if faulthandler is not None: |
| 265 | faulthandler.register(signal.SIGUSR1, all_threads=True) |
| 266 | |
| 267 | opts = parse_args(argv) |
koder aka kdanilov | 39e449e | 2016-12-17 15:15:26 +0200 | [diff] [blame] | 268 | stages = [] # type: List[Stage] |
koder aka kdanilov | 22d134e | 2016-11-08 11:33:19 +0200 | [diff] [blame] | 269 | |
| 270 | # stop mypy from telling that config & storage might be undeclared |
| 271 | config = None # type: Config |
koder aka kdanilov | 3d2bc4f | 2016-11-12 18:31:18 +0200 | [diff] [blame] | 272 | storage = None # type: Storage |
koder aka kdanilov | 0fdaaee | 2015-06-30 11:10:48 +0300 | [diff] [blame] | 273 | |
| 274 | if opts.subparser_name == 'test': |
koder aka kdanilov | 962ee5f | 2016-12-19 02:40:08 +0200 | [diff] [blame] | 275 | config = load_config(opts.config_file) |
koder aka kdanilov | e7e1a4d | 2016-12-17 20:29:52 +0200 | [diff] [blame] | 276 | config.storage_url, config.run_uuid = utils.get_uniq_path_uuid(config.results_dir) |
| 277 | config.comment = opts.comment |
| 278 | config.keep_vm = opts.keep_vm |
| 279 | config.no_tests = opts.no_tests |
| 280 | config.dont_discover_nodes = opts.dont_discover_nodes |
| 281 | config.build_id = opts.build_id |
| 282 | config.build_description = opts.build_description |
| 283 | config.build_type = opts.build_type |
| 284 | config.settings_dir = get_config_path(config, opts.settings_dir) |
koder aka kdanilov | 0fdaaee | 2015-06-30 11:10:48 +0300 | [diff] [blame] | 285 | |
koder aka kdanilov | e7e1a4d | 2016-12-17 20:29:52 +0200 | [diff] [blame] | 286 | storage = make_storage(config.storage_url) |
koder aka kdanilov | 7f59d56 | 2016-12-26 01:34:23 +0200 | [diff] [blame] | 287 | storage.put(config, 'config') |
koder aka kdanilov | ffaf48d | 2016-12-27 02:25:29 +0200 | [diff] [blame] | 288 | |
koder aka kdanilov | 7f59d56 | 2016-12-26 01:34:23 +0200 | [diff] [blame] | 289 | stages.extend(get_run_stages()) |
koder aka kdanilov | ffaf48d | 2016-12-27 02:25:29 +0200 | [diff] [blame] | 290 | stages.append(SaveNodesStage()) |
koder aka kdanilov | 0fdaaee | 2015-06-30 11:10:48 +0300 | [diff] [blame] | 291 | |
koder aka kdanilov | 39e449e | 2016-12-17 15:15:26 +0200 | [diff] [blame] | 292 | if not opts.dont_collect: |
koder aka kdanilov | 962ee5f | 2016-12-19 02:40:08 +0200 | [diff] [blame] | 293 | stages.append(CollectInfoStage()) |
koder aka kdanilov | 0fdaaee | 2015-06-30 11:10:48 +0300 | [diff] [blame] | 294 | |
koder aka kdanilov | 962ee5f | 2016-12-19 02:40:08 +0200 | [diff] [blame] | 295 | argv2 = argv[:] |
| 296 | if '--ssh-key-passwd' in argv2: |
| 297 | # don't save ssh key password to storage |
| 298 | argv2[argv2.index("--ssh-key-passwd") + 1] = "<removed from output>" |
koder aka kdanilov | 7f59d56 | 2016-12-26 01:34:23 +0200 | [diff] [blame] | 299 | storage.put(argv2, 'cli') |
koder aka kdanilov | e7e1a4d | 2016-12-17 20:29:52 +0200 | [diff] [blame] | 300 | |
| 301 | elif opts.subparser_name == 'resume': |
koder aka kdanilov | 7f59d56 | 2016-12-26 01:34:23 +0200 | [diff] [blame] | 302 | opts.resumed = True |
koder aka kdanilov | e7e1a4d | 2016-12-17 20:29:52 +0200 | [diff] [blame] | 303 | storage = make_storage(opts.storage_dir, existing=True) |
| 304 | config = storage.load(Config, 'config') |
koder aka kdanilov | 7f59d56 | 2016-12-26 01:34:23 +0200 | [diff] [blame] | 305 | stages.extend(get_run_stages()) |
| 306 | stages.append(LoadStoredNodesStage()) |
koder aka kdanilov | 108ac36 | 2017-01-19 20:17:16 +0200 | [diff] [blame] | 307 | prev_opts = storage.get('cli') # type: List[str] |
| 308 | |
koder aka kdanilov | 7f59d56 | 2016-12-26 01:34:23 +0200 | [diff] [blame] | 309 | if '--ssh-key-passwd' in prev_opts and opts.ssh_key_passwd: |
| 310 | prev_opts[prev_opts.index("--ssh-key-passwd") + 1] = opts.ssh_key_passwd |
| 311 | |
| 312 | restored_opts = parse_args(prev_opts) |
| 313 | opts.__dict__.update(restored_opts.__dict__) |
| 314 | opts.subparser_name = 'resume' |
koder aka kdanilov | e7e1a4d | 2016-12-17 20:29:52 +0200 | [diff] [blame] | 315 | |
koder aka kdanilov | 0fdaaee | 2015-06-30 11:10:48 +0300 | [diff] [blame] | 316 | elif opts.subparser_name == 'ls': |
koder aka kdanilov | 22d134e | 2016-11-08 11:33:19 +0200 | [diff] [blame] | 317 | tab = texttable.Texttable(max_width=200) |
| 318 | tab.set_deco(tab.HEADER | tab.VLINES | tab.BORDER) |
| 319 | tab.set_cols_align(["l", "l", "l", "l"]) |
| 320 | tab.header(["Name", "Tests", "Run at", "Comment"]) |
| 321 | tab.add_rows(list_results(opts.result_storage)) |
| 322 | print(tab.draw()) |
koder aka kdanilov | 0fdaaee | 2015-06-30 11:10:48 +0300 | [diff] [blame] | 323 | return 0 |
| 324 | |
| 325 | elif opts.subparser_name == 'report': |
koder aka kdanilov | ffaf48d | 2016-12-27 02:25:29 +0200 | [diff] [blame] | 326 | if getattr(opts, "no_report", False): |
| 327 | print(" --no-report option can't be used with 'report' cmd") |
| 328 | return 1 |
koder aka kdanilov | 22d134e | 2016-11-08 11:33:19 +0200 | [diff] [blame] | 329 | storage = make_storage(opts.data_dir, existing=True) |
koder aka kdanilov | ffaf48d | 2016-12-27 02:25:29 +0200 | [diff] [blame] | 330 | config = storage.load(Config, 'config') |
| 331 | stages.append(LoadStoredNodesStage()) |
koder aka kdanilov | 0fdaaee | 2015-06-30 11:10:48 +0300 | [diff] [blame] | 332 | |
| 333 | elif opts.subparser_name == 'compare': |
koder aka kdanilov | 7308462 | 2016-11-16 21:51:08 +0200 | [diff] [blame] | 334 | # x = run_test.load_data_from_path(opts.data_path1) |
| 335 | # y = run_test.load_data_from_path(opts.data_path2) |
| 336 | # print(run_test.IOPerfTest.format_diff_for_console( |
| 337 | # [x['io'][0], y['io'][0]])) |
koder aka kdanilov | 0fdaaee | 2015-06-30 11:10:48 +0300 | [diff] [blame] | 338 | return 0 |
| 339 | |
koder aka kdanilov | 7f59d56 | 2016-12-26 01:34:23 +0200 | [diff] [blame] | 340 | elif opts.subparser_name == 'db': |
| 341 | storage = make_storage(opts.storage_dir, existing=True) |
| 342 | if opts.cmd == 'show': |
| 343 | if len(opts.params) != 1: |
| 344 | print("'show' command requires parameter - key to show") |
| 345 | return 1 |
| 346 | pprint.pprint(storage.get(opts.params[0])) |
| 347 | else: |
| 348 | print("Unknown/not_implemented command {!r}".format(opts.cmd)) |
| 349 | return 1 |
| 350 | return 0 |
koder aka kdanilov | 108ac36 | 2017-01-19 20:17:16 +0200 | [diff] [blame] | 351 | elif opts.subparser_name == 'ipython': |
| 352 | storage = make_storage(opts.storage_dir, existing=True) |
| 353 | from .hlstorage import ResultStorage |
| 354 | rstorage = ResultStorage(storage=storage) |
| 355 | |
| 356 | import IPython |
| 357 | IPython.embed() |
| 358 | |
| 359 | return 0 |
koder aka kdanilov | a732a60 | 2017-02-01 20:29:56 +0200 | [diff] [blame] | 360 | # elif opts.subparser_name == 'jupyter': |
| 361 | # with tempfile.NamedTemporaryFile() as fd: |
| 362 | # fd.write(notebook_kern.replace("$STORAGE", opts.storage_dir)) |
| 363 | # subprocess.call("jupyter notebook ", shell=True) |
| 364 | # return 0 |
| 365 | else: |
| 366 | print("Subparser {!r} is not supported".format(opts.subparser_name)) |
| 367 | return 1 |
koder aka kdanilov | 7f59d56 | 2016-12-26 01:34:23 +0200 | [diff] [blame] | 368 | |
koder aka kdanilov | 39e449e | 2016-12-17 15:15:26 +0200 | [diff] [blame] | 369 | report_stages = [] # type: List[Stage] |
koder aka kdanilov | 22d134e | 2016-11-08 11:33:19 +0200 | [diff] [blame] | 370 | if not getattr(opts, "no_report", False): |
koder aka kdanilov | 108ac36 | 2017-01-19 20:17:16 +0200 | [diff] [blame] | 371 | # report_stages.append(CalcStatisticStage()) |
| 372 | # report_stages.append(ConsoleReportStage()) |
koder aka kdanilov | 962ee5f | 2016-12-19 02:40:08 +0200 | [diff] [blame] | 373 | report_stages.append(HtmlReportStage()) |
koder aka kdanilov | 0fdaaee | 2015-06-30 11:10:48 +0300 | [diff] [blame] | 374 | |
koder aka kdanilov | 22d134e | 2016-11-08 11:33:19 +0200 | [diff] [blame] | 375 | # log level is not a part of config |
koder aka kdanilov | 0fdaaee | 2015-06-30 11:10:48 +0300 | [diff] [blame] | 376 | if opts.log_level is not None: |
| 377 | str_level = opts.log_level |
| 378 | else: |
koder aka kdanilov | 22d134e | 2016-11-08 11:33:19 +0200 | [diff] [blame] | 379 | str_level = config.get('logging/log_level', 'INFO') |
koder aka kdanilov | 0fdaaee | 2015-06-30 11:10:48 +0300 | [diff] [blame] | 380 | |
koder aka kdanilov | 962ee5f | 2016-12-19 02:40:08 +0200 | [diff] [blame] | 381 | log_config_file = config.get('logging/config', None) |
| 382 | |
| 383 | if log_config_file is not None: |
| 384 | log_config_file = find_cfg_file(log_config_file, opts.config_file) |
| 385 | |
| 386 | setup_loggers(getattr(logging, str_level), |
koder aka kdanilov | 7f59d56 | 2016-12-26 01:34:23 +0200 | [diff] [blame] | 387 | log_fd=storage.get_fd('log', "w"), |
koder aka kdanilov | 962ee5f | 2016-12-19 02:40:08 +0200 | [diff] [blame] | 388 | config_file=log_config_file) |
| 389 | |
koder aka kdanilov | 22d134e | 2016-11-08 11:33:19 +0200 | [diff] [blame] | 390 | logger.info("All info would be stored into %r", config.storage_url) |
| 391 | |
| 392 | ctx = TestRun(config, storage) |
koder aka kdanilov | e7e1a4d | 2016-12-17 20:29:52 +0200 | [diff] [blame] | 393 | ctx.rpc_code, ctx.default_rpc_plugins = node.get_rpc_server_code() |
koder aka kdanilov | 0fdaaee | 2015-06-30 11:10:48 +0300 | [diff] [blame] | 394 | |
koder aka kdanilov | 962ee5f | 2016-12-19 02:40:08 +0200 | [diff] [blame] | 395 | if opts.ssh_key_passwd is not None: |
| 396 | set_ssh_key_passwd(opts.ssh_key_passwd) |
koder aka kdanilov | f286517 | 2016-12-30 03:35:11 +0200 | [diff] [blame] | 397 | elif opts.ssh_key_passwd_kbd: |
| 398 | set_ssh_key_passwd(getpass.getpass("Ssh key password: ").strip()) |
koder aka kdanilov | 962ee5f | 2016-12-19 02:40:08 +0200 | [diff] [blame] | 399 | |
koder aka kdanilov | 39e449e | 2016-12-17 15:15:26 +0200 | [diff] [blame] | 400 | stages.sort(key=lambda x: x.priority) |
| 401 | |
koder aka kdanilov | 962ee5f | 2016-12-19 02:40:08 +0200 | [diff] [blame] | 402 | # TODO: run only stages, which have config |
koder aka kdanilov | 39e449e | 2016-12-17 15:15:26 +0200 | [diff] [blame] | 403 | failed = False |
| 404 | cleanup_stages = [] |
koder aka kdanilov | 7f59d56 | 2016-12-26 01:34:23 +0200 | [diff] [blame] | 405 | |
koder aka kdanilov | 0fdaaee | 2015-06-30 11:10:48 +0300 | [diff] [blame] | 406 | for stage in stages: |
koder aka kdanilov | e7e1a4d | 2016-12-17 20:29:52 +0200 | [diff] [blame] | 407 | if stage.config_block is not None: |
| 408 | if stage.config_block not in ctx.config: |
| 409 | continue |
| 410 | |
| 411 | cleanup_stages.append(stage) |
koder aka kdanilov | 39e449e | 2016-12-17 15:15:26 +0200 | [diff] [blame] | 412 | try: |
koder aka kdanilov | 39e449e | 2016-12-17 15:15:26 +0200 | [diff] [blame] | 413 | with log_stage(stage): |
| 414 | stage.run(ctx) |
koder aka kdanilov | 962ee5f | 2016-12-19 02:40:08 +0200 | [diff] [blame] | 415 | except (Exception, KeyboardInterrupt): |
koder aka kdanilov | 39e449e | 2016-12-17 15:15:26 +0200 | [diff] [blame] | 416 | failed = True |
koder aka kdanilov | 0fdaaee | 2015-06-30 11:10:48 +0300 | [diff] [blame] | 417 | break |
koder aka kdanilov | 7f59d56 | 2016-12-26 01:34:23 +0200 | [diff] [blame] | 418 | ctx.storage.sync() |
| 419 | ctx.storage.sync() |
koder aka kdanilov | 0fdaaee | 2015-06-30 11:10:48 +0300 | [diff] [blame] | 420 | |
koder aka kdanilov | 39e449e | 2016-12-17 15:15:26 +0200 | [diff] [blame] | 421 | logger.debug("Start cleanup") |
| 422 | cleanup_failed = False |
| 423 | for stage in cleanup_stages[::-1]: |
| 424 | try: |
koder aka kdanilov | 962ee5f | 2016-12-19 02:40:08 +0200 | [diff] [blame] | 425 | with log_stage(stage, cleanup=True): |
koder aka kdanilov | 39e449e | 2016-12-17 15:15:26 +0200 | [diff] [blame] | 426 | stage.cleanup(ctx) |
| 427 | except: |
| 428 | cleanup_failed = True |
koder aka kdanilov | 7f59d56 | 2016-12-26 01:34:23 +0200 | [diff] [blame] | 429 | ctx.storage.sync() |
koder aka kdanilov | 0fdaaee | 2015-06-30 11:10:48 +0300 | [diff] [blame] | 430 | |
koder aka kdanilov | 39e449e | 2016-12-17 15:15:26 +0200 | [diff] [blame] | 431 | if not failed: |
koder aka kdanilov | 0fdaaee | 2015-06-30 11:10:48 +0300 | [diff] [blame] | 432 | for report_stage in report_stages: |
| 433 | with log_stage(report_stage): |
koder aka kdanilov | ffaf48d | 2016-12-27 02:25:29 +0200 | [diff] [blame] | 434 | try: |
| 435 | report_stage.run(ctx) |
| 436 | except utils.StopTestError: |
| 437 | logger.error("Report stage %s requested stop execution", report_stage.name()) |
| 438 | failed = True |
| 439 | break |
koder aka kdanilov | 0fdaaee | 2015-06-30 11:10:48 +0300 | [diff] [blame] | 440 | |
koder aka kdanilov | 7f59d56 | 2016-12-26 01:34:23 +0200 | [diff] [blame] | 441 | ctx.storage.sync() |
| 442 | |
koder aka kdanilov | 22d134e | 2016-11-08 11:33:19 +0200 | [diff] [blame] | 443 | logger.info("All info is stored into %r", config.storage_url) |
koder aka kdanilov | 0fdaaee | 2015-06-30 11:10:48 +0300 | [diff] [blame] | 444 | |
koder aka kdanilov | 39e449e | 2016-12-17 15:15:26 +0200 | [diff] [blame] | 445 | if failed or cleanup_failed: |
koder aka kdanilov | 22d134e | 2016-11-08 11:33:19 +0200 | [diff] [blame] | 446 | logger.error("Tests are failed. See error details in log above") |
koder aka kdanilov | 0fdaaee | 2015-06-30 11:10:48 +0300 | [diff] [blame] | 447 | return 1 |
koder aka kdanilov | 39e449e | 2016-12-17 15:15:26 +0200 | [diff] [blame] | 448 | else: |
| 449 | logger.info("Tests finished successfully") |
| 450 | return 0 |