commit c3e8673eaf4cab8f38b2a1d4a1926d72a341da9f
author Zuul <zuul@review.opendev.org> Mon Sep 23 12:39:13 2019 +0000
committer Gerrit Code Review <review@openstack.org> Mon Sep 23 12:39:13 2019 +0000
tree f71099a70e0028b498d47fdda35f835a7a9ffbac
parent 2f467caa9821a9e9fbe7d77069a40115337af275
parent aa65dfb5265536eca40dbaf9b1826f8bf5148f80

Merge "Add retry decorator to SSH "execute" method"

neutron_tempest_plugin/common/ssh.py

diff --git a/neutron_tempest_plugin/common/ssh.py b/neutron_tempest_plugin/common/ssh.py
index ea30a28..96f0ef9 100644
--- a/neutron_tempest_plugin/common/ssh.py
+++ b/neutron_tempest_plugin/common/ssh.py
@@ -14,12 +14,15 @@
 
 import locale
 import os
+import socket
 import time
 
 from oslo_log import log
 import paramiko
+import six
 from tempest.lib.common import ssh
 from tempest.lib import exceptions
+import tenacity
 
 from neutron_tempest_plugin import config
 from neutron_tempest_plugin import exceptions as exc
@@ -29,6 +32,16 @@
 LOG = log.getLogger(__name__)
 
 
+RETRY_EXCEPTIONS = (exceptions.TimeoutException, paramiko.SSHException,
+                    socket.error)
+if six.PY2:
+    # NOTE(ralonsoh): TimeoutError was added in 3.3 and corresponds to
+    # OSError(errno.ETIMEDOUT)
+    RETRY_EXCEPTIONS += (OSError, )
+else:
+    RETRY_EXCEPTIONS += (TimeoutError, )
+
+
 class Client(ssh.Client):
 
     default_ssh_lang = 'en_US.UTF-8'
@@ -179,6 +192,11 @@
                                         user=self.username,
                                         password=self.password)
 
+    @tenacity.retry(
+        stop=tenacity.stop_after_attempt(10),
+        wait=tenacity.wait_fixed(1),
+        retry=tenacity.retry_if_exception_type(RETRY_EXCEPTIONS),
+        reraise=True)
     def exec_command(self, cmd, encoding="utf-8", timeout=None):
         if timeout:
             original_timeout = self.timeout

neutron_tempest_plugin/scenario/test_qos.py

diff --git a/neutron_tempest_plugin/scenario/test_qos.py b/neutron_tempest_plugin/scenario/test_qos.py
index 82a5391..ba8cc88 100644
--- a/neutron_tempest_plugin/scenario/test_qos.py
+++ b/neutron_tempest_plugin/scenario/test_qos.py
@@ -85,9 +85,9 @@
         cmd = ("(dd if=/dev/zero bs=%(bs)d count=%(count)d of=%(file_path)s) "
                % {'bs': self.BUFFER_SIZE, 'count': self.COUNT,
                'file_path': self.FILE_PATH})
-        ssh_client.exec_command(cmd)
+        ssh_client.exec_command(cmd, timeout=5)
         cmd = "stat -c %%s %s" % self.FILE_PATH
-        filesize = ssh_client.exec_command(cmd)
+        filesize = ssh_client.exec_command(cmd, timeout=5)
         if int(filesize.strip()) != self.FILE_SIZE:
             raise sc_exceptions.FileCreationFailedException(
                 file=self.FILE_PATH)
@@ -96,7 +96,7 @@
     def _kill_nc_process(ssh_client):
         cmd = "killall -q nc"
         try:
-            ssh_client.exec_command(cmd)
+            ssh_client.exec_command(cmd, timeout=5)
         except exceptions.SSHExecCommandFailed:
             pass
 
@@ -104,7 +104,7 @@
         self._kill_nc_process(ssh_client)
         cmd = ("(nc -ll -p %(port)d < %(file_path)s > /dev/null &)" % {
                 'port': port, 'file_path': self.FILE_PATH})
-        ssh_client.exec_command(cmd)
+        ssh_client.exec_command(cmd, timeout=5)
 
         # Open TCP socket to remote VM and download big file
         start_time = time.time()

requirements.txt

diff --git a/requirements.txt b/requirements.txt
index bb836d1..2febb7e 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -13,6 +13,7 @@
 paramiko>=2.0.0 # LGPLv2.1+
 six>=1.10.0 # MIT
 tempest>=17.1.0 # Apache-2.0
+tenacity>=3.2.1 # Apache-2.0
 ddt>=1.0.1 # MIT
 testtools>=2.2.0 # MIT
 testscenarios>=0.4 # Apache-2.0/BSD