Merge "Check VM's console log before trying to SSH to it."
diff --git a/neutron_tempest_plugin/common/ip.py b/neutron_tempest_plugin/common/ip.py
index 83cd3d9..d981770 100644
--- a/neutron_tempest_plugin/common/ip.py
+++ b/neutron_tempest_plugin/common/ip.py
@@ -36,13 +36,19 @@
sudo = 'sudo'
ip_path = '/sbin/ip'
- def __init__(self, ssh_client=None, timeout=None):
+ def __init__(self, ssh_client=None, timeout=None, namespace=None):
self.ssh_client = ssh_client
self.timeout = timeout
+ self.namespace = namespace
def get_command(self, obj, *command):
- command_line = '{sudo!s} {ip_path!r} {object!s} {command!s}'.format(
- sudo=self.sudo, ip_path=self.ip_path, object=obj,
+ command_line = '{sudo!s} {ip_path!r} '.format(sudo=self.sudo,
+ ip_path=self.ip_path)
+ if self.namespace:
+ command_line += 'netns exec {ns_name!s} {ip_path!r} '.format(
+ ns_name=self.namespace, ip_path=self.ip_path)
+ command_line += '{object!s} {command!s}'.format(
+ object=obj,
command=subprocess.list2cmdline([str(c) for c in command]))
return command_line
@@ -84,6 +90,13 @@
self.add_address(address=subport_ip, device=subport_device)
return subport_device
+ def list_namespaces(self):
+ namespaces_output = self.execute("netns")
+ ns_list = []
+ for ns_line in namespaces_output.split("\n"):
+ ns_list.append(ns_line.split(" ", 1)[0])
+ return ns_list
+
def list_addresses(self, device=None, ip_addresses=None, port=None,
subnets=None):
command = ['list']
@@ -308,20 +321,24 @@
netaddr.IPNetwork(subnet['cidr']).prefixlen)
-def arp_table():
+def arp_table(namespace=None):
# 192.168.0.16 0x1 0x2 dc:a6:32:06:56:51 * enp0s31f6
regex_str = (r"([^ ]+)\s+(0x\d+)\s+(0x\d+)\s+(\w{2}\:\w{2}\:\w{2}\:\w{2}\:"
r"\w{2}\:\w{2})\s+([\w+\*]+)\s+([\-\w]+)")
regex = re.compile(regex_str)
arp_table = []
- with open('/proc/net/arp', 'r') as proc_file:
- for line in proc_file.readlines():
- m = regex.match(line)
- if m:
- arp_table.append(ARPregister(
- ip_address=m.group(1), hw_type=m.group(2),
- flags=m.group(3), mac_address=m.group(4),
- mask=m.group(5), device=m.group(6)))
+ cmd = ""
+ if namespace:
+ cmd = "sudo ip netns exec %s " % namespace
+ cmd += "cat /proc/net/arp"
+ arp_entries = shell.execute(cmd).stdout.split("\n")
+ for line in arp_entries:
+ m = regex.match(line)
+ if m:
+ arp_table.append(ARPregister(
+ ip_address=m.group(1), hw_type=m.group(2),
+ flags=m.group(3), mac_address=m.group(4),
+ mask=m.group(5), device=m.group(6)))
return arp_table
diff --git a/neutron_tempest_plugin/scenario/base.py b/neutron_tempest_plugin/scenario/base.py
index 8481edc..2eb45f8 100644
--- a/neutron_tempest_plugin/scenario/base.py
+++ b/neutron_tempest_plugin/scenario/base.py
@@ -21,6 +21,7 @@
from neutron_lib.api import validators
from neutron_lib import constants as neutron_lib_constants
from oslo_log import log
+from paramiko import ssh_exception as ssh_exc
from tempest.common.utils import net_utils
from tempest.common import waiters
from tempest.lib.common.utils import data_utils
@@ -269,7 +270,7 @@
pkey=ssh_key, timeout=ssh_timeout)
try:
ssh_client.test_connection_auth()
- except lib_exc.SSHTimeout as ssh_e:
+ except (lib_exc.SSHTimeout, ssh_exc.AuthenticationException) as ssh_e:
LOG.debug(ssh_e)
self._log_console_output(servers)
self._log_local_network_status()
@@ -299,10 +300,20 @@
"for the console log", server['id'])
def _log_local_network_status(self):
- local_routes = ip_utils.IPCommand().list_routes()
- LOG.debug('Local routes:\n%s', '\n'.join(str(r) for r in local_routes))
+ self._log_ns_network_status()
+ for ns_name in ip_utils.IPCommand().list_namespaces():
+ self._log_ns_network_status(ns_name=ns_name)
+
+ def _log_ns_network_status(self, ns_name=None):
+ local_ips = ip_utils.IPCommand(namespace=ns_name).list_addresses()
+ LOG.debug('Namespace %s; IP Addresses:\n%s',
+ ns_name, '\n'.join(str(r) for r in local_ips))
+ local_routes = ip_utils.IPCommand(namespace=ns_name).list_routes()
+ LOG.debug('Namespace %s; Local routes:\n%s',
+ ns_name, '\n'.join(str(r) for r in local_routes))
arp_table = ip_utils.arp_table()
- LOG.debug('Local ARP table:\n%s', '\n'.join(str(r) for r in arp_table))
+ LOG.debug('Namespace %s; Local ARP table:\n%s',
+ ns_name, '\n'.join(str(r) for r in arp_table))
def _check_remote_connectivity(self, source, dest, count,
should_succeed=True,
@@ -384,12 +395,14 @@
fragmentation,
timeout=timeout, pattern=pattern,
forbid_packet_loss=forbid_packet_loss))
- except lib_exc.SSHTimeout as ssh_e:
+ except (lib_exc.SSHTimeout, ssh_exc.AuthenticationException) as ssh_e:
LOG.debug(ssh_e)
self._log_console_output(servers)
+ self._log_local_network_status()
raise
except AssertionError:
self._log_console_output(servers)
+ self._log_local_network_status()
raise
def ping_ip_address(self, ip_address, should_succeed=True,
@@ -499,15 +512,17 @@
**kwargs)
self.assertIn(server['name'],
ssh_client.exec_command('hostname'))
- except lib_exc.SSHTimeout as ssh_e:
+ except (lib_exc.SSHTimeout, ssh_exc.AuthenticationException) as ssh_e:
LOG.debug(ssh_e)
if log_errors:
self._log_console_output(servers)
+ self._log_local_network_status()
raise
except AssertionError as assert_e:
LOG.debug(assert_e)
if log_errors:
self._log_console_output(servers)
+ self._log_local_network_status()
raise
def ensure_nc_listen(self, ssh_client, port, protocol, echo_msg=None,
@@ -532,9 +547,10 @@
return ssh_client.execute_script(
get_ncat_server_cmd(port, protocol, echo_msg),
become_root=True, combine_stderr=True)
- except lib_exc.SSHTimeout as ssh_e:
+ except (lib_exc.SSHTimeout, ssh_exc.AuthenticationException) as ssh_e:
LOG.debug(ssh_e)
self._log_console_output(servers)
+ self._log_local_network_status()
raise
def nc_client(self, ip_address, port, protocol):
diff --git a/neutron_tempest_plugin/scenario/test_ipv6.py b/neutron_tempest_plugin/scenario/test_ipv6.py
index 741945a..732c96d 100644
--- a/neutron_tempest_plugin/scenario/test_ipv6.py
+++ b/neutron_tempest_plugin/scenario/test_ipv6.py
@@ -15,6 +15,7 @@
from neutron_lib import constants as lib_constants
from oslo_log import log
+from paramiko import ssh_exception as ssh_exc
from tempest.common import utils as tempest_utils
from tempest.lib.common.utils import data_utils
from tempest.lib import decorators
@@ -108,16 +109,22 @@
return True
return False
- # Set NIC with IPv6 to be UP and wait until IPv6 address will be
- # configured on this NIC
- turn_nic6_on(ssh_client, ipv6_port)
- # And check if IPv6 address will be properly configured on this NIC
- utils.wait_until_true(
- lambda: guest_has_address(ipv6_address),
- timeout=120,
- exception=RuntimeError(
- "Timed out waiting for IP address {!r} to be configured in "
- "the VM {!r}.".format(ipv6_address, vm['id'])))
+ try:
+ # Set NIC with IPv6 to be UP and wait until IPv6 address will be
+ # configured on this NIC
+ turn_nic6_on(ssh_client, ipv6_port)
+ # And check if IPv6 address will be properly configured on this NIC
+ utils.wait_until_true(
+ lambda: guest_has_address(ipv6_address),
+ timeout=120,
+ exception=RuntimeError(
+ "Timed out waiting for IP address {!r} to be configured "
+ "in the VM {!r}.".format(ipv6_address, vm['id'])))
+ except (lib_exc.SSHTimeout, ssh_exc.AuthenticationException) as ssh_e:
+ LOG.debug(ssh_e)
+ self._log_console_output([vm])
+ self._log_local_network_status()
+ raise
def _test_ipv6_hotplug(self, ra_mode, address_mode):
ipv6_networks = [self.create_network() for _ in range(2)]
diff --git a/neutron_tempest_plugin/scenario/test_security_groups.py b/neutron_tempest_plugin/scenario/test_security_groups.py
index 23a5224..9059a2f 100644
--- a/neutron_tempest_plugin/scenario/test_security_groups.py
+++ b/neutron_tempest_plugin/scenario/test_security_groups.py
@@ -33,13 +33,15 @@
required_extensions = ['router', 'security-group']
def _verify_http_connection(self, ssh_client, ssh_server,
- test_ip, test_port, should_pass=True):
+ test_ip, test_port, servers, should_pass=True):
"""Verify if HTTP connection works using remote hosts.
:param ssh.Client ssh_client: The client host active SSH client.
:param ssh.Client ssh_server: The HTTP server host active SSH client.
:param string test_ip: IP address of HTTP server
:param string test_port: Port of HTTP server
+ :param list servers: List of servers for which console output will be
+ logged in case when test case
:param bool should_pass: Wheter test should pass or not.
:return: if passed or not
@@ -57,6 +59,8 @@
except Exception as e:
if not should_pass:
return
+ self._log_console_output(servers)
+ self._log_local_network_status()
raise e
@classmethod
@@ -378,6 +382,7 @@
ssh_clients[0],
ssh_clients[2],
test_ip, port,
+ servers,
should_pass=False)
# add two remote-group rules with port-ranges
@@ -399,7 +404,8 @@
self._verify_http_connection(
ssh_clients[0],
ssh_clients[2],
- test_ip, port)
+ test_ip, port,
+ servers)
@decorators.idempotent_id('f07d0159-8f9e-4faa-87f5-a869ab0ad490')
def test_intra_sg_isolation(self):