Add log info for tempest SSH connection issues
The patch does the following:
1. Add in extra information
2. Logs current attempts
3. Waits a little longer prior to the next SSH attempt
Related-bug: #1253896
Change-Id: I878362899b4cf81143c02c15c55936b33cf65ec8
diff --git a/tempest/common/ssh.py b/tempest/common/ssh.py
index 0ed9b82..c772ce9 100644
--- a/tempest/common/ssh.py
+++ b/tempest/common/ssh.py
@@ -49,7 +49,7 @@
self.channel_timeout = float(channel_timeout)
self.buf_size = 1024
- def _get_ssh_connection(self, sleep=1.5, backoff=1.01):
+ def _get_ssh_connection(self, sleep=1.5, backoff=1):
"""Returns an ssh connection to the specified host."""
bsleep = sleep
ssh = paramiko.SSHClient()
@@ -76,19 +76,21 @@
self.username, self.host)
return ssh
except (socket.error,
- paramiko.SSHException):
- attempts += 1
- time.sleep(bsleep)
- bsleep *= backoff
- if not self._is_timed_out(_start_time):
- continue
- else:
+ paramiko.SSHException) as e:
+ if self._is_timed_out(_start_time):
LOG.exception("Failed to establish authenticated ssh"
" connection to %s@%s after %d attempts",
self.username, self.host, attempts)
raise exceptions.SSHTimeout(host=self.host,
user=self.username,
password=self.password)
+ bsleep += backoff
+ attempts += 1
+ LOG.warning("Failed to establish authenticated ssh"
+ " connection to %s@%s (%s). Number attempts: %s."
+ " Retry after %d seconds.",
+ self.username, self.host, e, attempts, bsleep)
+ time.sleep(bsleep)
def _is_timed_out(self, start_time):
return (time.time() - self.timeout) > start_time
diff --git a/tempest/tests/test_ssh.py b/tempest/tests/test_ssh.py
index 429ed56..a6eedc4 100644
--- a/tempest/tests/test_ssh.py
+++ b/tempest/tests/test_ssh.py
@@ -88,15 +88,17 @@
client_mock.connect.side_effect = [socket.error, socket.error, True]
t_mock.side_effect = [
1000, # Start time
+ 1000, # LOG.warning() calls time.time() loop 1
1001, # Sleep loop 1
+ 1001, # LOG.warning() calls time.time() loop 2
1002 # Sleep loop 2
]
client._get_ssh_connection(sleep=1)
expected_sleeps = [
- mock.call(1),
- mock.call(1.01)
+ mock.call(2),
+ mock.call(3)
]
self.assertEqual(expected_sleeps, s_mock.mock_calls)
@@ -111,7 +113,9 @@
]
t_mock.side_effect = [
1000, # Start time
+ 1000, # LOG.warning() calls time.time() loop 1
1001, # Sleep loop 1
+ 1001, # LOG.warning() calls time.time() loop 2
1002, # Sleep loop 2
1003, # Sleep loop 3
1004 # LOG.error() calls time.time()