Dennis Dmitriev | 6f59add | 2016-10-18 13:45:27 +0300 | [diff] [blame] | 1 | # Copyright 2016 Mirantis, Inc. |
| 2 | # |
| 3 | # Licensed under the Apache License, Version 2.0 (the "License"); you may |
| 4 | # not use this file except in compliance with the License. You may obtain |
| 5 | # a copy of the License at |
| 6 | # |
| 7 | # http://www.apache.org/licenses/LICENSE-2.0 |
| 8 | # |
| 9 | # Unless required by applicable law or agreed to in writing, software |
| 10 | # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT |
| 11 | # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the |
| 12 | # License for the specific language governing permissions and limitations |
| 13 | # under the License. |
Dennis Dmitriev | 9cc4ca3 | 2016-11-03 13:50:45 +0200 | [diff] [blame] | 14 | import datetime |
| 15 | import json |
| 16 | |
Dennis Dmitriev | 2d643bc | 2017-12-04 12:23:47 +0200 | [diff] [blame] | 17 | from devops import error |
| 18 | from functools32 import lru_cache |
Dennis Dmitriev | 6f59add | 2016-10-18 13:45:27 +0300 | [diff] [blame] | 19 | |
| 20 | from tcp_tests import logger |
| 21 | from tcp_tests import settings |
| 22 | |
| 23 | |
| 24 | LOG = logger.logger |
| 25 | |
| 26 | |
| 27 | class RallyManager(object): |
| 28 | """docstring for RallyManager""" |
| 29 | |
Dennis Dmitriev | 2d643bc | 2017-12-04 12:23:47 +0200 | [diff] [blame] | 30 | image_name = ( |
| 31 | 'docker-prod-virtual.docker.mirantis.net/' |
| 32 | 'mirantis/oscore/rally-tempest') |
| 33 | image_version = 'latest' |
| 34 | tempest_tag = "16.0.0" |
| 35 | designate_tag = "0.2.0" |
Dennis Dmitriev | 6f59add | 2016-10-18 13:45:27 +0300 | [diff] [blame] | 36 | |
Dennis Dmitriev | 2d643bc | 2017-12-04 12:23:47 +0200 | [diff] [blame] | 37 | def __init__(self, underlay, rally_node='gtw01.'): |
Dennis Dmitriev | 6f59add | 2016-10-18 13:45:27 +0300 | [diff] [blame] | 38 | super(RallyManager, self).__init__() |
Dennis Dmitriev | 6f59add | 2016-10-18 13:45:27 +0300 | [diff] [blame] | 39 | self._underlay = underlay |
Dennis Dmitriev | 2d643bc | 2017-12-04 12:23:47 +0200 | [diff] [blame] | 40 | self._node_name = self.get_target_node(target=rally_node) |
Dennis Dmitriev | 6f59add | 2016-10-18 13:45:27 +0300 | [diff] [blame] | 41 | |
Dennis Dmitriev | 2d643bc | 2017-12-04 12:23:47 +0200 | [diff] [blame] | 42 | @property |
| 43 | @lru_cache(maxsize=None) |
| 44 | def image_id(self): |
| 45 | LOG.info("Getting image id") |
| 46 | cmd = ("docker images | grep {0}| grep {1}| awk '{{print $3}}'" |
| 47 | .format(self.image_name, self.image_version)) |
| 48 | res = self._underlay.check_call(cmd, node_name=self._node_name) |
Dmitry Tyzhnenko | ccb4287 | 2018-08-28 13:41:15 +0300 | [diff] [blame] | 49 | LOG.debug(res['stdout']) |
Dennis Dmitriev | 2d643bc | 2017-12-04 12:23:47 +0200 | [diff] [blame] | 50 | image_id = res['stdout'][0].strip() |
| 51 | LOG.info("Image ID is {}".format(image_id)) |
| 52 | return image_id |
Dennis Dmitriev | 6f59add | 2016-10-18 13:45:27 +0300 | [diff] [blame] | 53 | |
Dennis Dmitriev | 2d643bc | 2017-12-04 12:23:47 +0200 | [diff] [blame] | 54 | @property |
| 55 | @lru_cache(maxsize=None) |
| 56 | def docker_id(self): |
| 57 | cmd = ("docker ps | grep {image_id} | " |
| 58 | "awk '{{print $1}}'| head -1").format( |
| 59 | image_id=self.image_id) |
| 60 | LOG.info("Getting container id") |
| 61 | res = self._underlay.check_call(cmd, node_name=self._node_name) |
| 62 | docker_id = res['stdout'][0].strip() |
| 63 | LOG.info("Container ID is {}".format(docker_id)) |
| 64 | return docker_id |
Dennis Dmitriev | 6f59add | 2016-10-18 13:45:27 +0300 | [diff] [blame] | 65 | |
Dennis Dmitriev | 2d643bc | 2017-12-04 12:23:47 +0200 | [diff] [blame] | 66 | # Move method to underlay |
| 67 | def get_target_node(self, target='gtw01.'): |
| 68 | return [node_name for node_name |
| 69 | in self._underlay.node_names() |
| 70 | if node_name.startswith(target)][0] |
Dennis Dmitriev | 6f59add | 2016-10-18 13:45:27 +0300 | [diff] [blame] | 71 | |
Dennis Dmitriev | 2d643bc | 2017-12-04 12:23:47 +0200 | [diff] [blame] | 72 | def _docker_exec(self, cmd, timeout=None, verbose=False): |
| 73 | docker_cmd = ('docker exec -i {docker_id} bash -c "{cmd}"' |
| 74 | .format(cmd=cmd, docker_id=self.docker_id)) |
| 75 | LOG.info("Executing: {docker_cmd}".format(docker_cmd=docker_cmd)) |
Dennis Dmitriev | b8115f5 | 2017-12-15 13:09:56 +0200 | [diff] [blame] | 76 | return self._underlay.check_call(docker_cmd, node_name=self._node_name, |
| 77 | verbose=verbose, timeout=timeout) |
Dennis Dmitriev | 6f59add | 2016-10-18 13:45:27 +0300 | [diff] [blame] | 78 | |
Dennis Dmitriev | 2d643bc | 2017-12-04 12:23:47 +0200 | [diff] [blame] | 79 | def _run(self): |
| 80 | """Start the rally container in the background""" |
| 81 | with self._underlay.remote(node_name=self._node_name) as remote: |
| 82 | cmd = ("docker run --net host -v /root/rally:/home/rally/.rally " |
| 83 | "-v /etc/ssl/certs/:/etc/ssl/certs/ " |
| 84 | "-tid -u root --entrypoint /bin/bash {image_id}" |
| 85 | .format(image_id=self.image_id)) |
Dennis Dmitriev | 6f59add | 2016-10-18 13:45:27 +0300 | [diff] [blame] | 86 | LOG.info("Run Rally container") |
| 87 | remote.check_call(cmd) |
| 88 | |
Dennis Dmitriev | 2d643bc | 2017-12-04 12:23:47 +0200 | [diff] [blame] | 89 | def run_container(self, version=None): |
| 90 | """Install docker, configure and run rally container""" |
| 91 | version = version or self.image_version |
| 92 | image = self.image_name |
| 93 | LOG.info("Pull {image}:{version}".format(image=image, |
| 94 | version=version)) |
Tatyana Leontovich | a4dff86 | 2018-07-20 17:16:26 +0300 | [diff] [blame] | 95 | try: |
| 96 | cmd = ("apt-get -y install docker-ce &&" |
| 97 | " docker pull {image}:{version}".format(image=image, |
| 98 | version=version)) |
| 99 | self._underlay.check_call(cmd, node_name=self._node_name) |
Dennis Dmitriev | b6bcc5c | 2018-09-26 11:07:53 +0000 | [diff] [blame] | 100 | except Exception: |
Tatyana Leontovich | a4dff86 | 2018-07-20 17:16:26 +0300 | [diff] [blame] | 101 | LOG.debug('Cannot install docker-ce') |
| 102 | cmd = ("apt-get -y install docker.io &&" |
| 103 | " docker pull {image}:{version}".format(image=image, |
| 104 | version=version)) |
| 105 | self._underlay.check_call(cmd, node_name=self._node_name) |
Dennis Dmitriev | 6f59add | 2016-10-18 13:45:27 +0300 | [diff] [blame] | 106 | |
Tatyana Leontovich | d6bcbc9 | 2018-03-23 15:02:28 +0200 | [diff] [blame] | 107 | cmd_iptables = "iptables --policy FORWARD ACCEPT" |
| 108 | self._underlay.check_call(cmd_iptables, node_name=self._node_name) |
| 109 | |
Dennis Dmitriev | 2d643bc | 2017-12-04 12:23:47 +0200 | [diff] [blame] | 110 | LOG.info("Create rally workdir") |
| 111 | cmd = 'mkdir -p /root/rally; chown 65500 /root/rally' |
| 112 | self._underlay.check_call(cmd, node_name=self._node_name) |
| 113 | |
| 114 | LOG.info("Copy keystonercv3") |
Dmitry Tyzhnenko | 6d77ce4 | 2018-06-06 18:39:31 +0300 | [diff] [blame] | 115 | tgt = self._node_name.split('.')[0] |
| 116 | cmd = "scp -3 ctl01:/root/keystonercv3 " \ |
| 117 | "{tgt}:/root/rally/keystonercv3".format( |
| 118 | tgt=tgt) |
| 119 | domain = '.'.join(self._node_name.split('.')[1:]) |
| 120 | self._underlay.check_call(cmd, node_name="cfg01.{domain}".format( |
| 121 | domain=domain)) |
Dennis Dmitriev | 2d643bc | 2017-12-04 12:23:47 +0200 | [diff] [blame] | 122 | self._run() |
| 123 | |
| 124 | LOG.info("Create rally deployment") |
| 125 | self._docker_exec("rally-manage db recreate") |
| 126 | self._docker_exec("source /home/rally/.rally/keystonercv3;" |
| 127 | "rally deployment create --fromenv --name=Abathur") |
| 128 | self._docker_exec("rally deployment list") |
| 129 | |
| 130 | def prepare_rally_task(self, target_node='ctl01.'): |
| 131 | """Prepare cirros image and private network for rally task""" |
| 132 | ctl_node_name = self._underlay.get_target_node_names( |
| 133 | target=target_node)[0] |
| 134 | cmds = [ |
| 135 | ". keystonercv3 ; openstack flavor create --public m1.tiny", |
| 136 | ("wget http://download.cirros-cloud.net/0.3.4/" |
| 137 | "cirros-0.3.4-i386-disk.img"), |
| 138 | (". /root/keystonercv3; glance --timeout 120 image-create " |
| 139 | "--name cirros-disk --visibility public --disk-format qcow2 " |
| 140 | "--container-format bare --progress " |
| 141 | "< /root/cirros-0.3.4-i386-disk.img"), |
| 142 | ". /root/keystonercv3; neutron net-create net04", |
Dennis Dmitriev | 6f59add | 2016-10-18 13:45:27 +0300 | [diff] [blame] | 143 | ] |
Dennis Dmitriev | 9cc4ca3 | 2016-11-03 13:50:45 +0200 | [diff] [blame] | 144 | |
Dennis Dmitriev | 2d643bc | 2017-12-04 12:23:47 +0200 | [diff] [blame] | 145 | for cmd in cmds: |
| 146 | self._underlay.check_call(cmd, node_name=ctl_node_name) |
Dennis Dmitriev | 9cc4ca3 | 2016-11-03 13:50:45 +0200 | [diff] [blame] | 147 | |
Dennis Dmitriev | 2d643bc | 2017-12-04 12:23:47 +0200 | [diff] [blame] | 148 | def prepare_tempest_task(self): |
| 149 | """Configure rally.conf for tempest tests""" |
| 150 | pass |
| 151 | # LOG.info("Modify rally.conf") |
| 152 | # cmd = ("sed -i 's|#swift_operator_role = Member|" |
| 153 | # "swift_operator_role=SwiftOperator|g' " |
| 154 | # "/etc/rally/rally.conf") |
| 155 | # self._docker_exec(cmd) |
Dennis Dmitriev | 9cc4ca3 | 2016-11-03 13:50:45 +0200 | [diff] [blame] | 156 | |
Dennis Dmitriev | 2d643bc | 2017-12-04 12:23:47 +0200 | [diff] [blame] | 157 | def create_rally_task(self, task_path, task_content): |
| 158 | """Create a file with rally task definition |
Dennis Dmitriev | 9cc4ca3 | 2016-11-03 13:50:45 +0200 | [diff] [blame] | 159 | |
Dennis Dmitriev | 2d643bc | 2017-12-04 12:23:47 +0200 | [diff] [blame] | 160 | :param task_path: path to JSON or YAML file on target node |
| 161 | :task_content: string with json or yaml content to store in file |
| 162 | """ |
Dmitry Tyzhnenko | bc1133a | 2017-12-20 14:23:53 +0200 | [diff] [blame] | 163 | cmd = ("mkdir -p $(dirname {task_path}) && " |
| 164 | "cat > {task_path} << EOF\n{task_content}\nEOF").format( |
Dennis Dmitriev | 2d643bc | 2017-12-04 12:23:47 +0200 | [diff] [blame] | 165 | task_path=task_path, task_content=task_content) |
| 166 | self._underlay.check_call(cmd, node_name=self._node_name) |
Dennis Dmitriev | 9cc4ca3 | 2016-11-03 13:50:45 +0200 | [diff] [blame] | 167 | |
Dennis Dmitriev | b8115f5 | 2017-12-15 13:09:56 +0200 | [diff] [blame] | 168 | def run_task(self, task='', timeout=None, raise_on_timeout=True, |
| 169 | verbose=False): |
Dennis Dmitriev | 2d643bc | 2017-12-04 12:23:47 +0200 | [diff] [blame] | 170 | """Run rally task |
Dennis Dmitriev | 9cc4ca3 | 2016-11-03 13:50:45 +0200 | [diff] [blame] | 171 | |
Dennis Dmitriev | 2d643bc | 2017-12-04 12:23:47 +0200 | [diff] [blame] | 172 | :param taks: path to json or yaml file with the task definition |
| 173 | :param raise_on_timeout: bool, ignore TimeoutError if False |
Dennis Dmitriev | b8115f5 | 2017-12-15 13:09:56 +0200 | [diff] [blame] | 174 | :param verbose: show rally output to console if True |
Dennis Dmitriev | 2d643bc | 2017-12-04 12:23:47 +0200 | [diff] [blame] | 175 | """ |
| 176 | try: |
Dennis Dmitriev | b8115f5 | 2017-12-15 13:09:56 +0200 | [diff] [blame] | 177 | res = self._docker_exec( |
| 178 | "rally task start {task}".format(task=task), |
| 179 | timeout=timeout, |
| 180 | verbose=verbose) |
Dennis Dmitriev | 2d643bc | 2017-12-04 12:23:47 +0200 | [diff] [blame] | 181 | except error.TimeoutError: |
| 182 | if raise_on_timeout: |
| 183 | raise |
| 184 | else: |
Dennis Dmitriev | b8115f5 | 2017-12-15 13:09:56 +0200 | [diff] [blame] | 185 | res = None |
Dennis Dmitriev | 2d643bc | 2017-12-04 12:23:47 +0200 | [diff] [blame] | 186 | pass |
Dennis Dmitriev | b8115f5 | 2017-12-15 13:09:56 +0200 | [diff] [blame] | 187 | return res |
Dennis Dmitriev | 9cc4ca3 | 2016-11-03 13:50:45 +0200 | [diff] [blame] | 188 | |
Dennis Dmitriev | 2d643bc | 2017-12-04 12:23:47 +0200 | [diff] [blame] | 189 | # Updated to replace the OpenStackManager method run_tempest |
| 190 | def run_tempest(self, conf_name='/var/lib/lvm_mcp.conf', |
| 191 | pattern='set=smoke', concurrency=0, timeout=None, |
Tatyana Leontovich | c72604d | 2018-01-04 17:58:00 +0200 | [diff] [blame] | 192 | report_prefix='', report_types=None, |
| 193 | designate_plugin=True): |
Dennis Dmitriev | 2d643bc | 2017-12-04 12:23:47 +0200 | [diff] [blame] | 194 | """Run tempest tests |
Dennis Dmitriev | 9cc4ca3 | 2016-11-03 13:50:45 +0200 | [diff] [blame] | 195 | |
Dennis Dmitriev | 2d643bc | 2017-12-04 12:23:47 +0200 | [diff] [blame] | 196 | :param conf_name: tempest config placed in the rally container |
| 197 | :param pattern: tempest testcase name or one of existing 'set=...' |
| 198 | :param concurrency: how many threads to use in parallel. 0 means |
| 199 | to take the amount of the cores on the node |
| 200 | <self._node_name>. |
| 201 | :param timeout: stop tempest tests after specified timeout. |
Tatyana Leontovich | c72604d | 2018-01-04 17:58:00 +0200 | [diff] [blame] | 202 | :param designate_plugin: enabled by default plugin for designate |
Dennis Dmitriev | 2d643bc | 2017-12-04 12:23:47 +0200 | [diff] [blame] | 203 | :param report_prefix: str, prefix for report filenames. Usually the |
| 204 | output of the fixture 'func_name' |
| 205 | :param report_types: list of the report types that need to download |
| 206 | from the environment: ['html', 'xml', 'json']. |
| 207 | None by default. |
| 208 | """ |
| 209 | report_types = report_types or [] |
Tatyana Leontovich | c72604d | 2018-01-04 17:58:00 +0200 | [diff] [blame] | 210 | if not designate_plugin: |
| 211 | cmd = ( |
| 212 | "cat > /root/rally/install_tempest.sh << EOF\n" |
| 213 | "rally verify create-verifier" |
| 214 | " --type tempest " |
| 215 | " --name tempest-verifier" |
| 216 | " --source /var/lib/tempest" |
| 217 | " --version {tempest_tag}" |
| 218 | " --system-wide\n" |
| 219 | "rally verify configure-verifier --extend {tempest_conf}\n" |
| 220 | "rally verify configure-verifier --show\n" |
| 221 | "EOF".format(tempest_tag=self.tempest_tag, |
| 222 | tempest_conf=conf_name)) |
| 223 | else: |
| 224 | cmd = ( |
| 225 | "cat > /root/rally/install_tempest.sh << EOF\n" |
| 226 | "rally verify create-verifier" |
| 227 | " --type tempest " |
| 228 | " --name tempest-verifier" |
| 229 | " --source /var/lib/tempest" |
| 230 | " --version {tempest_tag}" |
| 231 | " --system-wide\n" |
| 232 | "rally verify add-verifier-ext" |
| 233 | " --source /var/lib/designate-tempest-plugin" |
| 234 | " --version {designate_tag}\n" |
| 235 | "rally verify configure-verifier --extend {tempest_conf}\n" |
| 236 | "rally verify configure-verifier --show\n" |
| 237 | "EOF".format(tempest_tag=self.tempest_tag, |
| 238 | designate_tag=self.designate_tag, |
| 239 | tempest_conf=conf_name)) |
Dennis Dmitriev | 2d643bc | 2017-12-04 12:23:47 +0200 | [diff] [blame] | 240 | with self._underlay.remote(node_name=self._node_name) as remote: |
| 241 | LOG.info("Create install_tempest.sh") |
| 242 | remote.check_call(cmd) |
| 243 | remote.check_call("chmod +x /root/rally/install_tempest.sh") |
| 244 | |
| 245 | LOG.info("Run tempest inside Rally container") |
| 246 | self._docker_exec("/home/rally/.rally/install_tempest.sh") |
| 247 | self._docker_exec( |
| 248 | ("source /home/rally/.rally/keystonercv3 && " |
| 249 | "rally verify start --skip-list /var/lib/mcp_skip.list " |
| 250 | " --concurrency {concurrency} --pattern {pattern}" |
| 251 | .format(concurrency=concurrency, pattern=pattern)), |
| 252 | timeout=timeout, verbose=True) |
| 253 | if report_prefix: |
| 254 | report_filename = '{0}_report_{1}'.format( |
| 255 | report_prefix, |
| 256 | datetime.datetime.now().strftime('%Y%m%d_%H%M%S')) |
| 257 | else: |
| 258 | report_filename = 'report_{1}'.format( |
| 259 | datetime.datetime.now().strftime('%Y%m%d_%H%M%S')) |
| 260 | docker_file_prefix = '/home/rally/.rally/' + report_filename |
| 261 | |
| 262 | # Create reports |
| 263 | if 'xml' in report_types: |
| 264 | self._docker_exec( |
| 265 | "rally verify report --type junit-xml --to {0}.xml" |
| 266 | .format(docker_file_prefix)) |
| 267 | if 'html' in report_types: |
| 268 | self._docker_exec( |
| 269 | "rally verify report --type html --to {0}.html" |
| 270 | .format(docker_file_prefix)) |
| 271 | # Always create report in JSON to return results into test case |
| 272 | # However, it won't be downloaded until ('json' in report_prefix) |
| 273 | self._docker_exec("rally verify report --type json --to {0}.json" |
| 274 | .format(docker_file_prefix)) |
| 275 | |
| 276 | # Download reports to the settings.LOGS_DIR |
| 277 | file_src_prefix = '/root/rally/{0}'.format(report_filename) |
| 278 | file_dst_prefix = '{0}/{1}'.format(settings.LOGS_DIR, report_filename) |
| 279 | with self._underlay.remote(node_name=self._node_name) as remote: |
| 280 | for suffix in report_types: |
| 281 | remote.download(file_src_prefix + '.' + suffix, |
| 282 | file_dst_prefix + '.' + suffix) |
| 283 | res = json.load(remote.open(file_src_prefix + '.json')) |
| 284 | |
| 285 | # Get latest verification ID to find the lates testcases in the report |
| 286 | vtime = {vdata['finished_at']: vid |
| 287 | for vid, vdata in res['verifications'].items()} |
| 288 | vlatest_id = vtime[max(vtime.keys())] |
| 289 | |
| 290 | # Each status has the dict with pairs: |
| 291 | # <status>: { |
| 292 | # <case_name>: <case_details>, |
| 293 | # } |
| 294 | formatted_tc = { |
| 295 | 'success': {}, |
| 296 | 'fail': {}, |
| 297 | 'xfail': {}, |
| 298 | 'skip': {} |
| 299 | } |
| 300 | |
| 301 | for tname, tdata in res['tests'].items(): |
| 302 | status = tdata['by_verification'][vlatest_id]['status'] |
| 303 | details = tdata['by_verification'][vlatest_id].get('details', '') |
| 304 | if status not in formatted_tc: |
| 305 | # Fail if tempest return a new status that may be |
| 306 | # necessary to take into account in test cases |
| 307 | raise Exception("Unknown testcase {0} status: {1} " |
| 308 | .format(tname, status)) |
| 309 | formatted_tc[status][tdata['name']] = details |
| 310 | LOG.debug("Formatted testcases: {0}".format(formatted_tc)) |
| 311 | return formatted_tc |