Add retry decorator to SSH "execute" method
In case of SSH timeout (TimeoutException, TimeoutError), the
tenacity.retry decorator retries the execution of the SSH
"execute" method up to 10 times.
Some SSH execute calls, related to QoS scenario tests, have been
enhanced by setting a relatively small timeout value. The commands
executed should be quick enough to be executed in this amount of time.
In case of timeout (due to communication problems), the retry decorator
will send again the command to be executed.
Change-Id: Idc0d55b776f499a4bc5d8c9d9a549f0af8f3fac0
Closes-Bug: #1844516
diff --git a/neutron_tempest_plugin/common/ssh.py b/neutron_tempest_plugin/common/ssh.py
index ea30a28..96f0ef9 100644
--- a/neutron_tempest_plugin/common/ssh.py
+++ b/neutron_tempest_plugin/common/ssh.py
@@ -14,12 +14,15 @@
import locale
import os
+import socket
import time
from oslo_log import log
import paramiko
+import six
from tempest.lib.common import ssh
from tempest.lib import exceptions
+import tenacity
from neutron_tempest_plugin import config
from neutron_tempest_plugin import exceptions as exc
@@ -29,6 +32,16 @@
LOG = log.getLogger(__name__)
+RETRY_EXCEPTIONS = (exceptions.TimeoutException, paramiko.SSHException,
+ socket.error)
+if six.PY2:
+ # NOTE(ralonsoh): TimeoutError was added in 3.3 and corresponds to
+ # OSError(errno.ETIMEDOUT)
+ RETRY_EXCEPTIONS += (OSError, )
+else:
+ RETRY_EXCEPTIONS += (TimeoutError, )
+
+
class Client(ssh.Client):
default_ssh_lang = 'en_US.UTF-8'
@@ -179,6 +192,11 @@
user=self.username,
password=self.password)
+ @tenacity.retry(
+ stop=tenacity.stop_after_attempt(10),
+ wait=tenacity.wait_fixed(1),
+ retry=tenacity.retry_if_exception_type(RETRY_EXCEPTIONS),
+ reraise=True)
def exec_command(self, cmd, encoding="utf-8", timeout=None):
if timeout:
original_timeout = self.timeout