blob: 589e1ee7755f34bec0c627e11430b5718cc02f11 [file] [log] [blame]
Dennis Dmitriev6f59add2016-10-18 13:45:27 +03001# Copyright 2016 Mirantis, Inc.
2#
3# Licensed under the Apache License, Version 2.0 (the "License"); you may
4# not use this file except in compliance with the License. You may obtain
5# a copy of the License at
6#
7# http://www.apache.org/licenses/LICENSE-2.0
8#
9# Unless required by applicable law or agreed to in writing, software
10# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
11# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
12# License for the specific language governing permissions and limitations
13# under the License.
Dennis Dmitriev9cc4ca32016-11-03 13:50:45 +020014import datetime
15import json
16
Dennis Dmitriev2d643bc2017-12-04 12:23:47 +020017from devops import error
18from functools32 import lru_cache
Dennis Dmitriev6f59add2016-10-18 13:45:27 +030019
20from tcp_tests import logger
21from tcp_tests import settings
22
23
24LOG = logger.logger
25
26
27class RallyManager(object):
28 """docstring for RallyManager"""
29
Dennis Dmitriev2d643bc2017-12-04 12:23:47 +020030 image_name = (
31 'docker-prod-virtual.docker.mirantis.net/'
32 'mirantis/oscore/rally-tempest')
33 image_version = 'latest'
34 tempest_tag = "16.0.0"
35 designate_tag = "0.2.0"
Dennis Dmitriev6f59add2016-10-18 13:45:27 +030036
Dennis Dmitriev2d643bc2017-12-04 12:23:47 +020037 def __init__(self, underlay, rally_node='gtw01.'):
Dennis Dmitriev6f59add2016-10-18 13:45:27 +030038 super(RallyManager, self).__init__()
Dennis Dmitriev6f59add2016-10-18 13:45:27 +030039 self._underlay = underlay
Dennis Dmitriev2d643bc2017-12-04 12:23:47 +020040 self._node_name = self.get_target_node(target=rally_node)
Dennis Dmitriev6f59add2016-10-18 13:45:27 +030041
Dennis Dmitriev2d643bc2017-12-04 12:23:47 +020042 @property
43 @lru_cache(maxsize=None)
44 def image_id(self):
45 LOG.info("Getting image id")
46 cmd = ("docker images | grep {0}| grep {1}| awk '{{print $3}}'"
47 .format(self.image_name, self.image_version))
48 res = self._underlay.check_call(cmd, node_name=self._node_name)
Dmitry Tyzhnenkoccb42872018-08-28 13:41:15 +030049 LOG.debug(res['stdout'])
Dennis Dmitriev2d643bc2017-12-04 12:23:47 +020050 image_id = res['stdout'][0].strip()
51 LOG.info("Image ID is {}".format(image_id))
52 return image_id
Dennis Dmitriev6f59add2016-10-18 13:45:27 +030053
Dennis Dmitriev2d643bc2017-12-04 12:23:47 +020054 @property
55 @lru_cache(maxsize=None)
56 def docker_id(self):
57 cmd = ("docker ps | grep {image_id} | "
58 "awk '{{print $1}}'| head -1").format(
59 image_id=self.image_id)
60 LOG.info("Getting container id")
61 res = self._underlay.check_call(cmd, node_name=self._node_name)
62 docker_id = res['stdout'][0].strip()
63 LOG.info("Container ID is {}".format(docker_id))
64 return docker_id
Dennis Dmitriev6f59add2016-10-18 13:45:27 +030065
Dennis Dmitriev2d643bc2017-12-04 12:23:47 +020066 # Move method to underlay
67 def get_target_node(self, target='gtw01.'):
68 return [node_name for node_name
69 in self._underlay.node_names()
70 if node_name.startswith(target)][0]
Dennis Dmitriev6f59add2016-10-18 13:45:27 +030071
Dennis Dmitriev2d643bc2017-12-04 12:23:47 +020072 def _docker_exec(self, cmd, timeout=None, verbose=False):
73 docker_cmd = ('docker exec -i {docker_id} bash -c "{cmd}"'
74 .format(cmd=cmd, docker_id=self.docker_id))
75 LOG.info("Executing: {docker_cmd}".format(docker_cmd=docker_cmd))
Dennis Dmitrievb8115f52017-12-15 13:09:56 +020076 return self._underlay.check_call(docker_cmd, node_name=self._node_name,
77 verbose=verbose, timeout=timeout)
Dennis Dmitriev6f59add2016-10-18 13:45:27 +030078
Dennis Dmitriev2d643bc2017-12-04 12:23:47 +020079 def _run(self):
80 """Start the rally container in the background"""
81 with self._underlay.remote(node_name=self._node_name) as remote:
82 cmd = ("docker run --net host -v /root/rally:/home/rally/.rally "
83 "-v /etc/ssl/certs/:/etc/ssl/certs/ "
84 "-tid -u root --entrypoint /bin/bash {image_id}"
85 .format(image_id=self.image_id))
Dennis Dmitriev6f59add2016-10-18 13:45:27 +030086 LOG.info("Run Rally container")
87 remote.check_call(cmd)
88
Dennis Dmitriev2d643bc2017-12-04 12:23:47 +020089 def run_container(self, version=None):
90 """Install docker, configure and run rally container"""
91 version = version or self.image_version
92 image = self.image_name
93 LOG.info("Pull {image}:{version}".format(image=image,
94 version=version))
Tatyana Leontovicha4dff862018-07-20 17:16:26 +030095 try:
96 cmd = ("apt-get -y install docker-ce &&"
97 " docker pull {image}:{version}".format(image=image,
98 version=version))
99 self._underlay.check_call(cmd, node_name=self._node_name)
Dennis Dmitrievb6bcc5c2018-09-26 11:07:53 +0000100 except Exception:
Tatyana Leontovicha4dff862018-07-20 17:16:26 +0300101 LOG.debug('Cannot install docker-ce')
102 cmd = ("apt-get -y install docker.io &&"
103 " docker pull {image}:{version}".format(image=image,
104 version=version))
105 self._underlay.check_call(cmd, node_name=self._node_name)
Dennis Dmitriev6f59add2016-10-18 13:45:27 +0300106
Tatyana Leontovichd6bcbc92018-03-23 15:02:28 +0200107 cmd_iptables = "iptables --policy FORWARD ACCEPT"
108 self._underlay.check_call(cmd_iptables, node_name=self._node_name)
109
Dennis Dmitriev2d643bc2017-12-04 12:23:47 +0200110 LOG.info("Create rally workdir")
111 cmd = 'mkdir -p /root/rally; chown 65500 /root/rally'
112 self._underlay.check_call(cmd, node_name=self._node_name)
113
114 LOG.info("Copy keystonercv3")
Dmitry Tyzhnenko6d77ce42018-06-06 18:39:31 +0300115 tgt = self._node_name.split('.')[0]
116 cmd = "scp -3 ctl01:/root/keystonercv3 " \
117 "{tgt}:/root/rally/keystonercv3".format(
118 tgt=tgt)
119 domain = '.'.join(self._node_name.split('.')[1:])
120 self._underlay.check_call(cmd, node_name="cfg01.{domain}".format(
121 domain=domain))
Dennis Dmitriev2d643bc2017-12-04 12:23:47 +0200122 self._run()
123
124 LOG.info("Create rally deployment")
125 self._docker_exec("rally-manage db recreate")
126 self._docker_exec("source /home/rally/.rally/keystonercv3;"
127 "rally deployment create --fromenv --name=Abathur")
128 self._docker_exec("rally deployment list")
129
130 def prepare_rally_task(self, target_node='ctl01.'):
131 """Prepare cirros image and private network for rally task"""
132 ctl_node_name = self._underlay.get_target_node_names(
133 target=target_node)[0]
134 cmds = [
135 ". keystonercv3 ; openstack flavor create --public m1.tiny",
136 ("wget http://download.cirros-cloud.net/0.3.4/"
137 "cirros-0.3.4-i386-disk.img"),
138 (". /root/keystonercv3; glance --timeout 120 image-create "
139 "--name cirros-disk --visibility public --disk-format qcow2 "
140 "--container-format bare --progress "
141 "< /root/cirros-0.3.4-i386-disk.img"),
142 ". /root/keystonercv3; neutron net-create net04",
Dennis Dmitriev6f59add2016-10-18 13:45:27 +0300143 ]
Dennis Dmitriev9cc4ca32016-11-03 13:50:45 +0200144
Dennis Dmitriev2d643bc2017-12-04 12:23:47 +0200145 for cmd in cmds:
146 self._underlay.check_call(cmd, node_name=ctl_node_name)
Dennis Dmitriev9cc4ca32016-11-03 13:50:45 +0200147
Dennis Dmitriev2d643bc2017-12-04 12:23:47 +0200148 def prepare_tempest_task(self):
149 """Configure rally.conf for tempest tests"""
150 pass
151# LOG.info("Modify rally.conf")
152# cmd = ("sed -i 's|#swift_operator_role = Member|"
153# "swift_operator_role=SwiftOperator|g' "
154# "/etc/rally/rally.conf")
155# self._docker_exec(cmd)
Dennis Dmitriev9cc4ca32016-11-03 13:50:45 +0200156
Dennis Dmitriev2d643bc2017-12-04 12:23:47 +0200157 def create_rally_task(self, task_path, task_content):
158 """Create a file with rally task definition
Dennis Dmitriev9cc4ca32016-11-03 13:50:45 +0200159
Dennis Dmitriev2d643bc2017-12-04 12:23:47 +0200160 :param task_path: path to JSON or YAML file on target node
161 :task_content: string with json or yaml content to store in file
162 """
Dmitry Tyzhnenkobc1133a2017-12-20 14:23:53 +0200163 cmd = ("mkdir -p $(dirname {task_path}) && "
164 "cat > {task_path} << EOF\n{task_content}\nEOF").format(
Dennis Dmitriev2d643bc2017-12-04 12:23:47 +0200165 task_path=task_path, task_content=task_content)
166 self._underlay.check_call(cmd, node_name=self._node_name)
Dennis Dmitriev9cc4ca32016-11-03 13:50:45 +0200167
Dennis Dmitrievb8115f52017-12-15 13:09:56 +0200168 def run_task(self, task='', timeout=None, raise_on_timeout=True,
169 verbose=False):
Dennis Dmitriev2d643bc2017-12-04 12:23:47 +0200170 """Run rally task
Dennis Dmitriev9cc4ca32016-11-03 13:50:45 +0200171
Dennis Dmitriev2d643bc2017-12-04 12:23:47 +0200172 :param taks: path to json or yaml file with the task definition
173 :param raise_on_timeout: bool, ignore TimeoutError if False
Dennis Dmitrievb8115f52017-12-15 13:09:56 +0200174 :param verbose: show rally output to console if True
Dennis Dmitriev2d643bc2017-12-04 12:23:47 +0200175 """
176 try:
Dennis Dmitrievb8115f52017-12-15 13:09:56 +0200177 res = self._docker_exec(
178 "rally task start {task}".format(task=task),
179 timeout=timeout,
180 verbose=verbose)
Dennis Dmitriev2d643bc2017-12-04 12:23:47 +0200181 except error.TimeoutError:
182 if raise_on_timeout:
183 raise
184 else:
Dennis Dmitrievb8115f52017-12-15 13:09:56 +0200185 res = None
Dennis Dmitriev2d643bc2017-12-04 12:23:47 +0200186 pass
Dennis Dmitrievb8115f52017-12-15 13:09:56 +0200187 return res
Dennis Dmitriev9cc4ca32016-11-03 13:50:45 +0200188
Dennis Dmitriev2d643bc2017-12-04 12:23:47 +0200189 # Updated to replace the OpenStackManager method run_tempest
190 def run_tempest(self, conf_name='/var/lib/lvm_mcp.conf',
191 pattern='set=smoke', concurrency=0, timeout=None,
Tatyana Leontovichc72604d2018-01-04 17:58:00 +0200192 report_prefix='', report_types=None,
193 designate_plugin=True):
Dennis Dmitriev2d643bc2017-12-04 12:23:47 +0200194 """Run tempest tests
Dennis Dmitriev9cc4ca32016-11-03 13:50:45 +0200195
Dennis Dmitriev2d643bc2017-12-04 12:23:47 +0200196 :param conf_name: tempest config placed in the rally container
197 :param pattern: tempest testcase name or one of existing 'set=...'
198 :param concurrency: how many threads to use in parallel. 0 means
199 to take the amount of the cores on the node
200 <self._node_name>.
201 :param timeout: stop tempest tests after specified timeout.
Tatyana Leontovichc72604d2018-01-04 17:58:00 +0200202 :param designate_plugin: enabled by default plugin for designate
Dennis Dmitriev2d643bc2017-12-04 12:23:47 +0200203 :param report_prefix: str, prefix for report filenames. Usually the
204 output of the fixture 'func_name'
205 :param report_types: list of the report types that need to download
206 from the environment: ['html', 'xml', 'json'].
207 None by default.
208 """
209 report_types = report_types or []
Tatyana Leontovichc72604d2018-01-04 17:58:00 +0200210 if not designate_plugin:
211 cmd = (
212 "cat > /root/rally/install_tempest.sh << EOF\n"
213 "rally verify create-verifier"
214 " --type tempest "
215 " --name tempest-verifier"
216 " --source /var/lib/tempest"
217 " --version {tempest_tag}"
218 " --system-wide\n"
219 "rally verify configure-verifier --extend {tempest_conf}\n"
220 "rally verify configure-verifier --show\n"
221 "EOF".format(tempest_tag=self.tempest_tag,
222 tempest_conf=conf_name))
223 else:
224 cmd = (
225 "cat > /root/rally/install_tempest.sh << EOF\n"
226 "rally verify create-verifier"
227 " --type tempest "
228 " --name tempest-verifier"
229 " --source /var/lib/tempest"
230 " --version {tempest_tag}"
231 " --system-wide\n"
232 "rally verify add-verifier-ext"
233 " --source /var/lib/designate-tempest-plugin"
234 " --version {designate_tag}\n"
235 "rally verify configure-verifier --extend {tempest_conf}\n"
236 "rally verify configure-verifier --show\n"
237 "EOF".format(tempest_tag=self.tempest_tag,
238 designate_tag=self.designate_tag,
239 tempest_conf=conf_name))
Dennis Dmitriev2d643bc2017-12-04 12:23:47 +0200240 with self._underlay.remote(node_name=self._node_name) as remote:
241 LOG.info("Create install_tempest.sh")
242 remote.check_call(cmd)
243 remote.check_call("chmod +x /root/rally/install_tempest.sh")
244
245 LOG.info("Run tempest inside Rally container")
246 self._docker_exec("/home/rally/.rally/install_tempest.sh")
247 self._docker_exec(
248 ("source /home/rally/.rally/keystonercv3 && "
249 "rally verify start --skip-list /var/lib/mcp_skip.list "
250 " --concurrency {concurrency} --pattern {pattern}"
251 .format(concurrency=concurrency, pattern=pattern)),
252 timeout=timeout, verbose=True)
253 if report_prefix:
254 report_filename = '{0}_report_{1}'.format(
255 report_prefix,
256 datetime.datetime.now().strftime('%Y%m%d_%H%M%S'))
257 else:
258 report_filename = 'report_{1}'.format(
259 datetime.datetime.now().strftime('%Y%m%d_%H%M%S'))
260 docker_file_prefix = '/home/rally/.rally/' + report_filename
261
262 # Create reports
263 if 'xml' in report_types:
264 self._docker_exec(
265 "rally verify report --type junit-xml --to {0}.xml"
266 .format(docker_file_prefix))
267 if 'html' in report_types:
268 self._docker_exec(
269 "rally verify report --type html --to {0}.html"
270 .format(docker_file_prefix))
271 # Always create report in JSON to return results into test case
272 # However, it won't be downloaded until ('json' in report_prefix)
273 self._docker_exec("rally verify report --type json --to {0}.json"
274 .format(docker_file_prefix))
275
276 # Download reports to the settings.LOGS_DIR
277 file_src_prefix = '/root/rally/{0}'.format(report_filename)
278 file_dst_prefix = '{0}/{1}'.format(settings.LOGS_DIR, report_filename)
279 with self._underlay.remote(node_name=self._node_name) as remote:
280 for suffix in report_types:
281 remote.download(file_src_prefix + '.' + suffix,
282 file_dst_prefix + '.' + suffix)
283 res = json.load(remote.open(file_src_prefix + '.json'))
284
285 # Get latest verification ID to find the lates testcases in the report
286 vtime = {vdata['finished_at']: vid
287 for vid, vdata in res['verifications'].items()}
288 vlatest_id = vtime[max(vtime.keys())]
289
290 # Each status has the dict with pairs:
291 # <status>: {
292 # <case_name>: <case_details>,
293 # }
294 formatted_tc = {
295 'success': {},
296 'fail': {},
297 'xfail': {},
298 'skip': {}
299 }
300
301 for tname, tdata in res['tests'].items():
302 status = tdata['by_verification'][vlatest_id]['status']
303 details = tdata['by_verification'][vlatest_id].get('details', '')
304 if status not in formatted_tc:
305 # Fail if tempest return a new status that may be
306 # necessary to take into account in test cases
307 raise Exception("Unknown testcase {0} status: {1} "
308 .format(tname, status))
309 formatted_tc[status][tdata['name']] = details
310 LOG.debug("Formatted testcases: {0}".format(formatted_tc))
311 return formatted_tc