Bootstrap with opened SSH on nodes
master_config.sh script for cfg01-day01 image requires working
SSH service to perform 'ssh-keyscan' to prepare Jenkins.
1. To wait the end of the bootstrap process with opened SSH,
two flags added:
/is_cloud_init_started - file on the node, indicates that the
bootstrap process should wait for the flag /is_cloud_init_finished
/is_cloud_init_finished - file on the node, indicates that the
bootstrap process can be finished successfully
2. Backward compatibility:
- if the SSH service is available, but /is_cloud_init_started not
found, then the bootstrap process will be finished as successful.
- if any of (AuthenticationException, BadAuthenticationType)
exceptions caused, the bootstrap process will be finished as
successful.
3. For each node, at least 2 successful (in terms of #1 or #2 above)
SSH checks should pass before the bootstrap is finished.
That allows to avoid intermediate allowness of ssh service during
bootstrap using the second check.
It is necessary for cases when ssh service is started by the
preparation process but is stopped for a while until the
cloud-init script is finished or the node is rebooted.
Change-Id: I82fb10efa8a67d080b725a66a3185fc845d2b1a0
diff --git a/tcp_tests/managers/envmanager_devops.py b/tcp_tests/managers/envmanager_devops.py
index d17c6bd..d02cff5 100644
--- a/tcp_tests/managers/envmanager_devops.py
+++ b/tcp_tests/managers/envmanager_devops.py
@@ -307,29 +307,58 @@
raise exceptions.EnvironmentIsNotSet()
self.__env.start()
LOG.info('Environment "{0}" started'.format(self.__env.name))
+ check_cloudinit_started = '[ -f /is_cloud_init_started ]'
+ check_cloudinit_finished = '[ -f /is_cloud_init_finished ]'
+ passed = {}
for node in self.__env.get_nodes(role__in=underlay_node_roles):
LOG.info("Waiting for SSH on node '{0}' / {1} ...".format(
node.name, self.node_ip(node)))
- def _ssh_wait(host,
- port,
- username=settings.SSH_NODE_CREDENTIALS['login'],
- password=settings.SSH_NODE_CREDENTIALS['password'],
- timeout=0):
+ def _ssh_check(host,
+ port,
+ username=settings.SSH_NODE_CREDENTIALS['login'],
+ password=settings.SSH_NODE_CREDENTIALS['password'],
+ timeout=0):
try:
ssh = ssh_client.SSHClient(
host=host, port=port,
auth=ssh_client.SSHAuth(
username=username,
password=password))
- except AuthenticationException:
- return True
- except BadAuthenticationType:
+
+ # If '/is_cloud_init_started' exists, then wait for
+ # the flag /is_cloud_init_finished
+ if ssh.execute(check_cloudinit_started)['exit_code'] == 0:
+ status = ssh.execute(
+ check_cloudinit_finished)['exit_code'] == 0
+ # Else, just wait for SSH
+ else:
+ status = ssh.execute('echo ok')['exit_code'] == 0
+ return status
+
+ except (AuthenticationException, BadAuthenticationType):
return True
except Exception:
return False
- return ssh.execute('echo ok')['exit_code'] == 0
+ def _ssh_wait(host,
+ port,
+ username=settings.SSH_NODE_CREDENTIALS['login'],
+ password=settings.SSH_NODE_CREDENTIALS['password'],
+ timeout=0):
+
+ if host in passed and passed[host] >= 2:
+ # host already passed the check
+ return True
+
+ for node in self.__env.get_nodes(role__in=underlay_node_roles):
+ ip = self.node_ip(node)
+ if ip not in passed:
+ passed[ip] = 0
+ if _ssh_check(ip, port):
+ passed[ip] += 1
+ else:
+ passed[ip] = 0
helpers.wait(
lambda: _ssh_wait(self.node_ip(node), 22),