blob: 2c9ed550c9f8f625d3624a457e3483346c7d10ad [file] [log] [blame]
Dennis Dmitriev6f59add2016-10-18 13:45:27 +03001# Copyright 2016 Mirantis, Inc.
2#
3# Licensed under the Apache License, Version 2.0 (the "License"); you may
4# not use this file except in compliance with the License. You may obtain
5# a copy of the License at
6#
7# http://www.apache.org/licenses/LICENSE-2.0
8#
9# Unless required by applicable law or agreed to in writing, software
10# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
11# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
12# License for the specific language governing permissions and limitations
13# under the License.
14
Tatyana Leontovichab47e162017-10-06 16:53:30 +030015import os
Dennis Dmitriev6f59add2016-10-18 13:45:27 +030016import random
Artem Panchenkodb0a97f2017-06-27 19:09:13 +030017import StringIO
Dennis Dmitriev6f59add2016-10-18 13:45:27 +030018
19from devops.helpers import helpers
20from devops.helpers import ssh_client
Dennis Dmitriev2dfb8ef2017-07-21 20:19:38 +030021from devops.helpers import subprocess_runner
Dennis Dmitriev6f59add2016-10-18 13:45:27 +030022from paramiko import rsakey
Dennis Dmitriev99b26fe2017-04-26 12:34:44 +030023import yaml
Dennis Dmitriev6f59add2016-10-18 13:45:27 +030024
25from tcp_tests import logger
Tatyana Leontovichab47e162017-10-06 16:53:30 +030026from tcp_tests.helpers import ext
Dennis Dmitriev6f59add2016-10-18 13:45:27 +030027from tcp_tests.helpers import utils
28
29LOG = logger.logger
30
31
32class UnderlaySSHManager(object):
33 """Keep the list of SSH access credentials to Underlay nodes.
34
35 This object is initialized using config.underlay.ssh.
36
37 :param config_ssh: JSONList of SSH access credentials for nodes:
38 [
39 {
40 node_name: node1,
41 address_pool: 'public-pool01',
42 host: ,
43 port: ,
44 keys: [],
45 keys_source_host: None,
46 login: ,
47 password: ,
Dennis Dmitriev474e3f72016-10-21 16:46:09 +030048 roles: [],
Dennis Dmitriev6f59add2016-10-18 13:45:27 +030049 },
50 {
51 node_name: node1,
52 address_pool: 'private-pool01',
53 host:
54 port:
55 keys: []
56 keys_source_host: None,
57 login:
58 password:
Dennis Dmitriev474e3f72016-10-21 16:46:09 +030059 roles: [],
Dennis Dmitriev6f59add2016-10-18 13:45:27 +030060 },
61 {
62 node_name: node2,
63 address_pool: 'public-pool01',
64 keys_source_host: node1
65 ...
66 }
67 ,
68 ...
69 ]
70
71 self.node_names(): list of node names registered in underlay.
72 self.remote(): SSHClient object by a node name (w/wo address pool)
73 or by a hostname.
74 """
Dennis Dmitriev2a13a132016-11-04 00:56:23 +020075 __config = None
Dennis Dmitriev6f59add2016-10-18 13:45:27 +030076 config_ssh = None
77 config_lvm = None
78
Dennis Dmitriev2a13a132016-11-04 00:56:23 +020079 def __init__(self, config):
Dennis Dmitriev6f59add2016-10-18 13:45:27 +030080 """Read config.underlay.ssh object
81
82 :param config_ssh: dict
83 """
Dennis Dmitriev2a13a132016-11-04 00:56:23 +020084 self.__config = config
Dennis Dmitriev6f59add2016-10-18 13:45:27 +030085 if self.config_ssh is None:
86 self.config_ssh = []
87
88 if self.config_lvm is None:
89 self.config_lvm = {}
90
Dennis Dmitriev2a13a132016-11-04 00:56:23 +020091 self.add_config_ssh(self.__config.underlay.ssh)
Dennis Dmitriev6f59add2016-10-18 13:45:27 +030092
93 def add_config_ssh(self, config_ssh):
94
95 if config_ssh is None:
96 config_ssh = []
97
98 for ssh in config_ssh:
99 ssh_data = {
100 # Required keys:
101 'node_name': ssh['node_name'],
102 'host': ssh['host'],
103 'login': ssh['login'],
104 'password': ssh['password'],
105 # Optional keys:
106 'address_pool': ssh.get('address_pool', None),
107 'port': ssh.get('port', None),
108 'keys': ssh.get('keys', []),
Dennis Dmitriev474e3f72016-10-21 16:46:09 +0300109 'roles': ssh.get('roles', []),
Dennis Dmitriev6f59add2016-10-18 13:45:27 +0300110 }
111
112 if 'keys_source_host' in ssh:
113 node_name = ssh['keys_source_host']
114 remote = self.remote(node_name)
115 keys = self.__get_keys(remote)
116 ssh_data['keys'].extend(keys)
117
118 self.config_ssh.append(ssh_data)
119
120 def remove_config_ssh(self, config_ssh):
121 if config_ssh is None:
122 config_ssh = []
123
124 for ssh in config_ssh:
125 ssh_data = {
126 # Required keys:
127 'node_name': ssh['node_name'],
128 'host': ssh['host'],
129 'login': ssh['login'],
130 'password': ssh['password'],
131 # Optional keys:
132 'address_pool': ssh.get('address_pool', None),
133 'port': ssh.get('port', None),
134 'keys': ssh.get('keys', []),
Dennis Dmitriev474e3f72016-10-21 16:46:09 +0300135 'roles': ssh.get('roles', []),
Dennis Dmitriev6f59add2016-10-18 13:45:27 +0300136 }
137 self.config_ssh.remove(ssh_data)
138
139 def __get_keys(self, remote):
140 keys = []
141 remote.execute('cd ~')
142 key_string = './.ssh/id_rsa'
143 if remote.exists(key_string):
144 with remote.open(key_string) as f:
145 keys.append(rsakey.RSAKey.from_private_key(f))
146 return keys
147
Tatyana Leontovichecd491d2017-09-13 13:51:12 +0300148 def __ssh_data(self, node_name=None, host=None, address_pool=None,
149 node_role=None):
Dennis Dmitriev6f59add2016-10-18 13:45:27 +0300150
151 ssh_data = None
152
153 if host is not None:
154 for ssh in self.config_ssh:
155 if host == ssh['host']:
156 ssh_data = ssh
157 break
158
159 elif node_name is not None:
160 for ssh in self.config_ssh:
161 if node_name == ssh['node_name']:
162 if address_pool is not None:
163 if address_pool == ssh['address_pool']:
164 ssh_data = ssh
165 break
166 else:
167 ssh_data = ssh
Tatyana Leontovichecd491d2017-09-13 13:51:12 +0300168 elif node_role is not None:
169 for ssh in self.config_ssh:
170 if node_role in ssh['roles']:
171 if address_pool is not None:
172 if address_pool == ssh['address_pool']:
173 ssh_data = ssh
174 break
175 else:
176 ssh_data = ssh
Dennis Dmitriev6f59add2016-10-18 13:45:27 +0300177 if ssh_data is None:
Dmitry Tyzhnenkob610afd2018-02-19 15:43:45 +0200178 LOG.debug("config_ssh - {}".format(self.config_ssh))
Dennis Dmitriev6f59add2016-10-18 13:45:27 +0300179 raise Exception('Auth data for node was not found using '
180 'node_name="{}" , host="{}" , address_pool="{}"'
181 .format(node_name, host, address_pool))
182 return ssh_data
183
184 def node_names(self):
185 """Get list of node names registered in config.underlay.ssh"""
186
187 names = [] # List is used to keep the original order of names
188 for ssh in self.config_ssh:
189 if ssh['node_name'] not in names:
190 names.append(ssh['node_name'])
191 return names
192
193 def enable_lvm(self, lvmconfig):
194 """Method for enabling lvm oh hosts in environment
195
196 :param lvmconfig: dict with ids or device' names of lvm storage
197 :raises: devops.error.DevopsCalledProcessError,
198 devops.error.TimeoutError, AssertionError, ValueError
199 """
200 def get_actions(lvm_id):
201 return [
202 "systemctl enable lvm2-lvmetad.service",
203 "systemctl enable lvm2-lvmetad.socket",
204 "systemctl start lvm2-lvmetad.service",
205 "systemctl start lvm2-lvmetad.socket",
206 "pvcreate {} && pvs".format(lvm_id),
207 "vgcreate default {} && vgs".format(lvm_id),
208 "lvcreate -L 1G -T default/pool && lvs",
209 ]
210 lvmpackages = ["lvm2", "liblvm2-dev", "thin-provisioning-tools"]
211 for node_name in self.node_names():
212 lvm = lvmconfig.get(node_name, None)
213 if not lvm:
214 continue
215 if 'id' in lvm:
216 lvmdevice = '/dev/disk/by-id/{}'.format(lvm['id'])
217 elif 'device' in lvm:
218 lvmdevice = '/dev/{}'.format(lvm['device'])
219 else:
220 raise ValueError("Unknown LVM device type")
221 if lvmdevice:
222 self.apt_install_package(
223 packages=lvmpackages, node_name=node_name, verbose=True)
224 for command in get_actions(lvmdevice):
225 self.sudo_check_call(command, node_name=node_name,
226 verbose=True)
227 self.config_lvm = dict(lvmconfig)
228
229 def host_by_node_name(self, node_name, address_pool=None):
230 ssh_data = self.__ssh_data(node_name=node_name,
231 address_pool=address_pool)
232 return ssh_data['host']
233
Tatyana Leontovichecd491d2017-09-13 13:51:12 +0300234 def host_by_node_role(self, node_role, address_pool=None):
235 ssh_data = self.__ssh_data(node_role=node_role,
236 address_pool=address_pool)
237 return ssh_data['host']
238
Dennis Dmitriev6f59add2016-10-18 13:45:27 +0300239 def remote(self, node_name=None, host=None, address_pool=None):
240 """Get SSHClient by a node name or hostname.
241
242 One of the following arguments should be specified:
243 - host (str): IP address or hostname. If specified, 'node_name' is
244 ignored.
245 - node_name (str): Name of the node stored to config.underlay.ssh
246 - address_pool (str): optional for node_name.
247 If None, use the first matched node_name.
248 """
249 ssh_data = self.__ssh_data(node_name=node_name, host=host,
250 address_pool=address_pool)
Dmitry Tyzhnenko5a5d8da2017-12-14 14:14:42 +0200251 ssh_auth = ssh_client.SSHAuth(
252 username=ssh_data['login'],
253 password=ssh_data['password'],
254 keys=[rsakey.RSAKey(file_obj=StringIO.StringIO(key))
255 for key in ssh_data['keys']])
256
Dennis Dmitriev6f59add2016-10-18 13:45:27 +0300257 return ssh_client.SSHClient(
258 host=ssh_data['host'],
259 port=ssh_data['port'] or 22,
Dmitry Tyzhnenko5a5d8da2017-12-14 14:14:42 +0200260 auth=ssh_auth)
Dennis Dmitriev6f59add2016-10-18 13:45:27 +0300261
Dennis Dmitriev2dfb8ef2017-07-21 20:19:38 +0300262 def local(self):
263 """Get Subprocess instance for local operations like:
264
265 underlay.local.execute(command, verbose=False, timeout=None)
266 underlay.local.check_call(
267 command, verbose=False, timeout=None,
268 error_info=None, expected=None, raise_on_err=True)
269 underlay.local.check_stderr(
270 command, verbose=False, timeout=None,
271 error_info=None, raise_on_err=True)
272 """
273 return subprocess_runner.Subprocess()
274
Dennis Dmitriev6f59add2016-10-18 13:45:27 +0300275 def check_call(
276 self, cmd,
277 node_name=None, host=None, address_pool=None,
278 verbose=False, timeout=None,
279 error_info=None,
280 expected=None, raise_on_err=True):
281 """Execute command on the node_name/host and check for exit code
282
283 :type cmd: str
284 :type node_name: str
285 :type host: str
286 :type verbose: bool
287 :type timeout: int
288 :type error_info: str
289 :type expected: list
290 :type raise_on_err: bool
291 :rtype: list stdout
292 :raises: devops.error.DevopsCalledProcessError
293 """
294 remote = self.remote(node_name=node_name, host=host,
295 address_pool=address_pool)
296 return remote.check_call(
297 command=cmd, verbose=verbose, timeout=timeout,
298 error_info=error_info, expected=expected,
299 raise_on_err=raise_on_err)
300
301 def apt_install_package(self, packages=None, node_name=None, host=None,
302 **kwargs):
303 """Method to install packages on ubuntu nodes
304
305 :type packages: list
306 :type node_name: str
307 :type host: str
308 :raises: devops.error.DevopsCalledProcessError,
309 devops.error.TimeoutError, AssertionError, ValueError
310
311 Other params of check_call and sudo_check_call are allowed
312 """
313 expected = kwargs.pop('expected', None)
314 if not packages or not isinstance(packages, list):
315 raise ValueError("packages list should be provided!")
316 install = "apt-get install -y {}".format(" ".join(packages))
317 # Should wait until other 'apt' jobs are finished
318 pgrep_expected = [0, 1]
319 pgrep_command = "pgrep -a -f apt"
320 helpers.wait(
321 lambda: (self.check_call(
322 pgrep_command, expected=pgrep_expected, host=host,
323 node_name=node_name, **kwargs).exit_code == 1
324 ), interval=30, timeout=1200,
325 timeout_msg="Timeout reached while waiting for apt lock"
326 )
327 # Install packages
328 self.sudo_check_call("apt-get update", node_name=node_name, host=host,
329 **kwargs)
330 self.sudo_check_call(install, expected=expected, node_name=node_name,
331 host=host, **kwargs)
332
333 def sudo_check_call(
334 self, cmd,
335 node_name=None, host=None, address_pool=None,
336 verbose=False, timeout=None,
337 error_info=None,
338 expected=None, raise_on_err=True):
339 """Execute command with sudo on node_name/host and check for exit code
340
341 :type cmd: str
342 :type node_name: str
343 :type host: str
344 :type verbose: bool
345 :type timeout: int
346 :type error_info: str
347 :type expected: list
348 :type raise_on_err: bool
349 :rtype: list stdout
350 :raises: devops.error.DevopsCalledProcessError
351 """
352 remote = self.remote(node_name=node_name, host=host,
353 address_pool=address_pool)
354 with remote.get_sudo(remote):
355 return remote.check_call(
356 command=cmd, verbose=verbose, timeout=timeout,
357 error_info=error_info, expected=expected,
358 raise_on_err=raise_on_err)
359
360 def dir_upload(self, host, source, destination):
361 """Upload local directory content to remote host
362
363 :param host: str, remote node name
364 :param source: str, local directory path
365 :param destination: str, local directory path
366 """
367 with self.remote(node_name=host) as remote:
368 remote.upload(source, destination)
369
Dennis Dmitriev0f08d9a2017-12-19 02:27:59 +0200370 def get_random_node(self, node_names=None):
Dennis Dmitriev6f59add2016-10-18 13:45:27 +0300371 """Get random node name
372
Dennis Dmitriev0f08d9a2017-12-19 02:27:59 +0200373 :param node_names: list of strings
Dennis Dmitriev6f59add2016-10-18 13:45:27 +0300374 :return: str, name of node
375 """
Dennis Dmitriev0f08d9a2017-12-19 02:27:59 +0200376 return random.choice(node_names or self.node_names())
Dennis Dmitriev6f59add2016-10-18 13:45:27 +0300377
378 def yaml_editor(self, file_path, node_name=None, host=None,
379 address_pool=None):
380 """Returns an initialized YamlEditor instance for context manager
381
382 Usage (with 'underlay' fixture):
383
384 # Local YAML file
385 with underlay.yaml_editor('/path/to/file') as editor:
386 editor.content[key] = "value"
387
388 # Remote YAML file on TCP host
389 with underlay.yaml_editor('/path/to/file',
390 host=config.tcp.tcp_host) as editor:
391 editor.content[key] = "value"
392 """
393 # Local YAML file
394 if node_name is None and host is None:
395 return utils.YamlEditor(file_path=file_path)
396
397 # Remote YAML file
398 ssh_data = self.__ssh_data(node_name=node_name, host=host,
399 address_pool=address_pool)
400 return utils.YamlEditor(
401 file_path=file_path,
402 host=ssh_data['host'],
403 port=ssh_data['port'] or 22,
404 username=ssh_data['login'],
405 password=ssh_data['password'],
406 private_keys=ssh_data['keys'])
Dennis Dmitriev010f4cd2016-11-01 20:43:51 +0200407
Dennis Dmitriev99b26fe2017-04-26 12:34:44 +0300408 def read_template(self, file_path):
409 """Read yaml as a jinja template"""
410 options = {
411 'config': self.__config,
412 }
413 template = utils.render_template(file_path, options=options)
414 return yaml.load(template)
Tatyana Leontovichab47e162017-10-06 16:53:30 +0300415
416 def get_logs(self, artifact_name,
417 node_role=ext.UNDERLAY_NODE_ROLES.salt_master):
Dennis Dmitriev21369672018-03-24 14:50:18 +0200418
419 # Prefix each '$' symbol with backslash '\' to disable
420 # early interpolation of environment variables on cfg01 node only
Dennis Dmitrievf0b2afe2018-02-28 13:25:02 +0200421 dump_commands = (
Dennis Dmitriev21369672018-03-24 14:50:18 +0200422 "mkdir /root/\$(hostname -f)/;"
423 "rsync -aruv /var/log/ /root/\$(hostname -f)/;"
424 "dpkg -l > /root/\$(hostname -f)/dump_dpkg_l.txt;"
425 "df -h > /root/\$(hostname -f)/dump_df.txt;"
426 "mount > /root/\$(hostname -f)/dump_mount.txt;"
427 "blkid -o list > /root/\$(hostname -f)/dump_blkid_o_list.txt;"
428 "iptables -t nat -S > /root/\$(hostname -f)/dump_iptables_nat.txt;"
429 "iptables -S > /root/\$(hostname -f)/dump_iptables.txt;"
430 "ps auxwwf > /root/\$(hostname -f)/dump_ps.txt;"
431 "docker images > /root/\$(hostname -f)/dump_docker_images.txt;"
432 "docker ps > /root/\$(hostname -f)/dump_docker_ps.txt;"
Mikhail Ivanov82da3392018-03-15 22:19:26 +0400433 "docker service ls > "
Dennis Dmitriev21369672018-03-24 14:50:18 +0200434 " /root/\$(hostname -f)/dump_docker_services_ls.txt;"
435 "for SERVICE in \$(docker service ls | awk '{ print $2 }'); "
436 " do docker service ps --no-trunc 2>&1 \$SERVICE >> "
437 " /root/\$(hostname -f)/dump_docker_service_ps.txt;"
Dennis Dmitriev01d5e372018-03-15 23:29:29 +0200438 " done;"
Dennis Dmitriev21369672018-03-24 14:50:18 +0200439 "for SERVICE in \$(docker service ls | awk '{ print $2 }'); "
440 " do docker service logs 2>&1 \$SERVICE > "
441 " /root/\$(hostname -f)/dump_docker_service_\${SERVICE}_logs;"
Dennis Dmitriev01d5e372018-03-15 23:29:29 +0200442 " done;"
Dennis Dmitriev21369672018-03-24 14:50:18 +0200443 "vgdisplay > /root/\$(hostname -f)/dump_vgdisplay.txt;"
444 "lvdisplay > /root/\$(hostname -f)/dump_lvdisplay.txt;"
445 "ip a > /root/\$(hostname -f)/dump_ip_a.txt;"
446 "ip r > /root/\$(hostname -f)/dump_ip_r.txt;"
447 "netstat -anp > /root/\$(hostname -f)/dump_netstat.txt;"
448 "brctl show > /root/\$(hostname -f)/dump_brctl_show.txt;"
449 "arp -an > /root/\$(hostname -f)/dump_arp.txt;"
450 "uname -a > /root/\$(hostname -f)/dump_uname_a.txt;"
451 "lsmod > /root/\$(hostname -f)/dump_lsmod.txt;"
452 "cat /proc/interrupts > /root/\$(hostname -f)/dump_interrupts.txt;"
453 "cat /etc/*-release > /root/\$(hostname -f)/dump_release.txt;"
Dennis Dmitrievf0b2afe2018-02-28 13:25:02 +0200454 # OpenStack specific, will fail on other nodes
Dennis Dmitriev21369672018-03-24 14:50:18 +0200455 # "rabbitmqctl report > "
456 # " /root/\$(hostname -f)/dump_rabbitmqctl.txt;"
Tatyana Leontovichab47e162017-10-06 16:53:30 +0300457
Dennis Dmitriev21369672018-03-24 14:50:18 +0200458 # "ceph health > /root/\$(hostname -f)/dump_ceph_health.txt;"
459 # "ceph -s > /root/\$(hostname -f)/dump_ceph_s.txt;"
460 # "ceph osd tree > /root/\$(hostname -f)/dump_ceph_osd_tree.txt;"
Dennis Dmitriev0bc485b2017-12-13 12:49:54 +0200461
Dennis Dmitriev21369672018-03-24 14:50:18 +0200462 # "for ns in \$(ip netns list);"
463 # " do echo Namespace: \${ns}; ip netns exec \${ns} ip a;"
464 # "done > /root/\$(hostname -f)/dump_ip_a_ns.txt;"
Dennis Dmitrievf0b2afe2018-02-28 13:25:02 +0200465
Dennis Dmitriev21369672018-03-24 14:50:18 +0200466 # "for ns in \$(ip netns list);"
467 # " do echo Namespace: \${ns}; ip netns exec \${ns} ip r;"
468 # "done > /root/\$(hostname -f)/dump_ip_r_ns.txt;"
Dennis Dmitrievf0b2afe2018-02-28 13:25:02 +0200469
Dennis Dmitriev21369672018-03-24 14:50:18 +0200470 # "for ns in \$(ip netns list);"
471 # " do echo Namespace: \${ns}; ip netns exec \${ns} netstat -anp;"
472 # "done > /root/\$(hostname -f)/dump_netstat_ns.txt;"
Dennis Dmitrievf0b2afe2018-02-28 13:25:02 +0200473
474 "/usr/bin/haproxy-status.sh > "
Dennis Dmitriev21369672018-03-24 14:50:18 +0200475 " /root/\$(hostname -f)/dump_haproxy.txt;"
Dennis Dmitrievf0b2afe2018-02-28 13:25:02 +0200476
477 # Archive the files
478 "cd /root/; tar --absolute-names --warning=no-file-changed "
Dennis Dmitriev21369672018-03-24 14:50:18 +0200479 " -czf \$(hostname -f).tar.gz ./\$(hostname -f)/;"
Dennis Dmitrievf0b2afe2018-02-28 13:25:02 +0200480 )
481
482 master_host = self.__config.salt.salt_master_host
483 with self.remote(host=master_host) as master:
484 # dump files
485 LOG.info("Archive artifacts on all nodes")
Dennis Dmitrievc83b3d42018-03-16 00:59:18 +0200486 master.check_call('salt "*" cmd.run "{0}"'.format(dump_commands),
Dennis Dmitrievf0b2afe2018-02-28 13:25:02 +0200487 raise_on_err=False)
488
489 # create target dir for archives
490 master.check_call("mkdir /root/dump/")
491
492 # get archived artifacts to the master node
493 for node in self.config_ssh:
494 LOG.info("Getting archived artifacts from the node {0}"
Dennis Dmitriev0bc485b2017-12-13 12:49:54 +0200495 .format(node['node_name']))
Dennis Dmitrievf0b2afe2018-02-28 13:25:02 +0200496 master.check_call("rsync -aruv {0}:/root/*.tar.gz "
497 "/root/dump/".format(node['node_name']),
Dennis Dmitriev8feb2522018-03-28 19:17:04 +0300498 raise_on_err=False,
499 timeout=120)
Dennis Dmitriev0bc485b2017-12-13 12:49:54 +0200500
Dennis Dmitrievf0b2afe2018-02-28 13:25:02 +0200501 destination_name = '/root/{0}_dump.tar.gz'.format(artifact_name)
502 # Archive the artifacts from all nodes
503 master.check_call(
504 'cd /root/dump/;'
505 'tar --absolute-names --warning=no-file-changed -czf '
506 ' {0} ./'.format(destination_name))
Tatyana Leontovichab47e162017-10-06 16:53:30 +0300507
Dennis Dmitrievf0b2afe2018-02-28 13:25:02 +0200508 # Download the artifact to the host
Dennis Dmitriev2d643bc2017-12-04 12:23:47 +0200509 LOG.info("Downloading the artifact {0}".format(destination_name))
Dennis Dmitrievf0b2afe2018-02-28 13:25:02 +0200510 master.download(destination=destination_name, target=os.getcwd())
Dennis Dmitriev2d643bc2017-12-04 12:23:47 +0200511
512 def delayed_call(
513 self, cmd,
514 node_name=None, host=None, address_pool=None,
515 verbose=True, timeout=5,
516 delay_min=None, delay_max=None):
517 """Delayed call of the specified command in background
518
519 :param delay_min: minimum delay in minutes before run
520 the command
521 :param delay_max: maximum delay in minutes before run
522 the command
523 The command will be started at random time in the range
524 from delay_min to delay_max in minutes from 'now'
525 using the command 'at'.
526
527 'now' is rounded to integer by 'at' command, i.e.:
528 now(28 min 59 sec) == 28 min 00 sec.
529
530 So, if delay_min=1 , the command may start in range from
531 1 sec to 60 sec.
532
533 If delay_min and delay_max are None, then the command will
534 be executed in the background right now.
535 """
536 time_min = delay_min or delay_max
537 time_max = delay_max or delay_min
538
539 delay = None
540 if time_min is not None and time_max is not None:
541 delay = random.randint(time_min, time_max)
542
543 delay_str = ''
544 if delay:
545 delay_str = " + {0} min".format(delay)
546
547 delay_cmd = "cat << EOF | at now {0}\n{1}\nEOF".format(delay_str, cmd)
548
549 self.check_call(delay_cmd, node_name=node_name, host=host,
550 address_pool=address_pool, verbose=verbose,
551 timeout=timeout)
552
553 def get_target_node_names(self, target='gtw01.'):
554 """Get all node names which names starts with <target>"""
555 return [node_name for node_name
556 in self.node_names()
557 if node_name.startswith(target)]