Merge "Tempest test for Neutron extension: extraroute-atomic"
diff --git a/.zuul.yaml b/.zuul.yaml
index 00be4ed..44de1bf 100644
--- a/.zuul.yaml
+++ b/.zuul.yaml
@@ -846,8 +846,6 @@
         - sfc
       devstack_localrc:
         NETWORK_API_EXTENSIONS: "{{ (network_api_extensions_common + network_api_extensions_sfc) | join(',') }}"
-    files:
-      - ^neutron_tempest_plugin/sfc/.*$
 
 - job:
     name: neutron-tempest-plugin-bgpvpn-bagpipe
@@ -892,8 +890,6 @@
         - fwaas_v2
       devstack_localrc:
         NETWORK_API_EXTENSIONS: "{{ (network_api_extensions_common + network_api_extensions_fwaas) | join(',') }}"
-    files:
-      - ^neutron_tempest_plugin/fwaas/.*$
 
 - project-template:
     name: neutron-tempest-plugin-jobs
diff --git a/neutron_tempest_plugin/common/ssh.py b/neutron_tempest_plugin/common/ssh.py
index ea30a28..96f0ef9 100644
--- a/neutron_tempest_plugin/common/ssh.py
+++ b/neutron_tempest_plugin/common/ssh.py
@@ -14,12 +14,15 @@
 
 import locale
 import os
+import socket
 import time
 
 from oslo_log import log
 import paramiko
+import six
 from tempest.lib.common import ssh
 from tempest.lib import exceptions
+import tenacity
 
 from neutron_tempest_plugin import config
 from neutron_tempest_plugin import exceptions as exc
@@ -29,6 +32,16 @@
 LOG = log.getLogger(__name__)
 
 
+RETRY_EXCEPTIONS = (exceptions.TimeoutException, paramiko.SSHException,
+                    socket.error)
+if six.PY2:
+    # NOTE(ralonsoh): TimeoutError was added in 3.3 and corresponds to
+    # OSError(errno.ETIMEDOUT)
+    RETRY_EXCEPTIONS += (OSError, )
+else:
+    RETRY_EXCEPTIONS += (TimeoutError, )
+
+
 class Client(ssh.Client):
 
     default_ssh_lang = 'en_US.UTF-8'
@@ -179,6 +192,11 @@
                                         user=self.username,
                                         password=self.password)
 
+    @tenacity.retry(
+        stop=tenacity.stop_after_attempt(10),
+        wait=tenacity.wait_fixed(1),
+        retry=tenacity.retry_if_exception_type(RETRY_EXCEPTIONS),
+        reraise=True)
     def exec_command(self, cmd, encoding="utf-8", timeout=None):
         if timeout:
             original_timeout = self.timeout
diff --git a/neutron_tempest_plugin/scenario/test_multicast.py b/neutron_tempest_plugin/scenario/test_multicast.py
index cfaa73f..9b79582 100644
--- a/neutron_tempest_plugin/scenario/test_multicast.py
+++ b/neutron_tempest_plugin/scenario/test_multicast.py
@@ -22,11 +22,13 @@
 from neutron_tempest_plugin.common import ssh
 from neutron_tempest_plugin.common import utils
 from neutron_tempest_plugin import config
+from neutron_tempest_plugin import exceptions
 from neutron_tempest_plugin.scenario import base
 
 
 CONF = config.CONF
 LOG = log.getLogger(__name__)
+PYTHON3_BIN = "python3"
 
 
 def get_receiver_script(group, port, hello_message, ack_message, result_file):
@@ -192,20 +194,27 @@
         port = self.client.list_ports(
             network_id=self.network['id'], device_id=server['id'])['ports'][0]
         server['fip'] = self.create_floatingip(port=port)
+        server['ssh_client'] = ssh.Client(server['fip']['floating_ip_address'],
+                                          self.username,
+                                          pkey=self.keypair['private_key'])
+        self._check_python_installed_on_server(server['ssh_client'],
+                                               server['id'])
         return server
 
+    def _check_python_installed_on_server(self, ssh_client, server_id):
+        try:
+            ssh_client.execute_script('which %s' % PYTHON3_BIN)
+        except exceptions.SSHScriptFailed:
+            raise self.skipException(
+                "%s is not available on server %s" % (PYTHON3_BIN, server_id))
+
     def _prepare_sender(self, server, mcast_address):
         check_script = get_sender_script(
             group=mcast_address, port=self.multicast_port,
             message=self.multicast_message,
             result_file=self.sender_output_file)
-        ssh_client = ssh.Client(server['fip']['floating_ip_address'],
-                                self.username,
-                                pkey=self.keypair['private_key'])
-
-        ssh_client.execute_script(
+        server['ssh_client'].execute_script(
             'echo "%s" > ~/multicast_traffic_sender.py' % check_script)
-        return ssh_client
 
     def _prepare_receiver(self, server, mcast_address):
         check_script = get_receiver_script(
@@ -216,9 +225,9 @@
             server['fip']['floating_ip_address'],
             self.username,
             pkey=self.keypair['private_key'])
-        ssh_client.execute_script(
+        self._check_python_installed_on_server(ssh_client, server['id'])
+        server['ssh_client'].execute_script(
             'echo "%s" > ~/multicast_traffic_receiver.py' % check_script)
-        return ssh_client
 
     @decorators.idempotent_id('113486fc-24c9-4be4-8361-03b1c9892867')
     def test_multicast_between_vms_on_same_network(self):
@@ -246,41 +255,39 @@
                     path=file_path))
             return msg in result
 
-        sender_ssh_client = self._prepare_sender(sender, mcast_address)
-        receiver_ssh_clients = []
+        self._prepare_sender(sender, mcast_address)
         receiver_ids = []
         for receiver in receivers:
-            receiver_ssh_client = self._prepare_receiver(
-                receiver, mcast_address)
-            receiver_ssh_client.execute_script(
-                "python3 ~/multicast_traffic_receiver.py &", shell="bash")
+            self._prepare_receiver(receiver, mcast_address)
+            receiver['ssh_client'].execute_script(
+                "%s ~/multicast_traffic_receiver.py &" % PYTHON3_BIN,
+                shell="bash")
             utils.wait_until_true(
                 lambda: _message_received(
-                    receiver_ssh_client, self.hello_message,
+                    receiver['ssh_client'], self.hello_message,
                     self.receiver_output_file),
                 exception=RuntimeError(
                     "Receiver script didn't start properly on server "
                     "{!r}.".format(receiver['id'])))
 
-            receiver_ssh_clients.append(receiver_ssh_client)
             receiver_ids.append(receiver['id'])
 
         # Now lets run scripts on sender
-        sender_ssh_client.execute_script(
-            "python3 ~/multicast_traffic_sender.py")
+        sender['ssh_client'].execute_script(
+            "%s ~/multicast_traffic_sender.py" % PYTHON3_BIN)
 
         # And check if message was received
-        for receiver_ssh_client in receiver_ssh_clients:
+        for receiver in receivers:
             utils.wait_until_true(
                 lambda: _message_received(
-                    receiver_ssh_client, self.multicast_message,
+                    receiver['ssh_client'], self.multicast_message,
                     self.receiver_output_file),
                 exception=RuntimeError(
                     "Receiver {!r} didn't get multicast message".format(
                         receiver['id'])))
 
         # TODO(slaweq): add validation of answears on sended server
-        replies_result = sender_ssh_client.execute_script(
+        replies_result = sender['ssh_client'].execute_script(
             "cat {path} || echo '{path} not exists yet'".format(
                 path=self.sender_output_file))
         for receiver_id in receiver_ids:
diff --git a/neutron_tempest_plugin/scenario/test_qos.py b/neutron_tempest_plugin/scenario/test_qos.py
index 82a5391..ba8cc88 100644
--- a/neutron_tempest_plugin/scenario/test_qos.py
+++ b/neutron_tempest_plugin/scenario/test_qos.py
@@ -85,9 +85,9 @@
         cmd = ("(dd if=/dev/zero bs=%(bs)d count=%(count)d of=%(file_path)s) "
                % {'bs': self.BUFFER_SIZE, 'count': self.COUNT,
                'file_path': self.FILE_PATH})
-        ssh_client.exec_command(cmd)
+        ssh_client.exec_command(cmd, timeout=5)
         cmd = "stat -c %%s %s" % self.FILE_PATH
-        filesize = ssh_client.exec_command(cmd)
+        filesize = ssh_client.exec_command(cmd, timeout=5)
         if int(filesize.strip()) != self.FILE_SIZE:
             raise sc_exceptions.FileCreationFailedException(
                 file=self.FILE_PATH)
@@ -96,7 +96,7 @@
     def _kill_nc_process(ssh_client):
         cmd = "killall -q nc"
         try:
-            ssh_client.exec_command(cmd)
+            ssh_client.exec_command(cmd, timeout=5)
         except exceptions.SSHExecCommandFailed:
             pass
 
@@ -104,7 +104,7 @@
         self._kill_nc_process(ssh_client)
         cmd = ("(nc -ll -p %(port)d < %(file_path)s > /dev/null &)" % {
                 'port': port, 'file_path': self.FILE_PATH})
-        ssh_client.exec_command(cmd)
+        ssh_client.exec_command(cmd, timeout=5)
 
         # Open TCP socket to remote VM and download big file
         start_time = time.time()
diff --git a/requirements.txt b/requirements.txt
index bb836d1..2febb7e 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -13,6 +13,7 @@
 paramiko>=2.0.0 # LGPLv2.1+
 six>=1.10.0 # MIT
 tempest>=17.1.0 # Apache-2.0
+tenacity>=3.2.1 # Apache-2.0
 ddt>=1.0.1 # MIT
 testtools>=2.2.0 # MIT
 testscenarios>=0.4 # Apache-2.0/BSD