Merge "Use different job definition for linuxbridge jobs runs on Xenial"
diff --git a/neutron_tempest_plugin/api/admin/test_tag.py b/neutron_tempest_plugin/api/admin/test_tag.py
index 7879b4c..eae7977 100644
--- a/neutron_tempest_plugin/api/admin/test_tag.py
+++ b/neutron_tempest_plugin/api/admin/test_tag.py
@@ -28,13 +28,13 @@
def _get_and_compare_tags(self, tags):
res_body = self.client.get_tags(self.resource, self.res_id)
- self.assertItemsEqual(tags, res_body['tags'])
+ self.assertCountEqual(tags, res_body['tags'])
def _test_tag_operations(self):
# create and get tags
tags = ['red', 'blue']
res_body = self.client.update_tags(self.resource, self.res_id, tags)
- self.assertItemsEqual(tags, res_body['tags'])
+ self.assertCountEqual(tags, res_body['tags'])
self._get_and_compare_tags(tags)
# add a tag
@@ -52,7 +52,7 @@
# replace tags
tags = ['red', 'yellow', 'purple']
res_body = self.client.update_tags(self.resource, self.res_id, tags)
- self.assertItemsEqual(tags, res_body['tags'])
+ self.assertCountEqual(tags, res_body['tags'])
self._get_and_compare_tags(tags)
# get tag
@@ -477,7 +477,7 @@
# nothing specific about networks here, just a resource that is
# available in all setups
res_body = self.client.get_tags('networks', res_id)
- self.assertItemsEqual(tags, res_body['tags'])
+ self.assertCountEqual(tags, res_body['tags'])
@decorators.attr(type='smoke')
@decorators.idempotent_id('74c56fb1-a3b1-4a62-a8d2-d04dca6bd4cd')
diff --git a/neutron_tempest_plugin/api/test_allowed_address_pair.py b/neutron_tempest_plugin/api/test_allowed_address_pair.py
index dd48382..7b11638 100644
--- a/neutron_tempest_plugin/api/test_allowed_address_pair.py
+++ b/neutron_tempest_plugin/api/test_allowed_address_pair.py
@@ -81,7 +81,7 @@
body = self.client.update_port(
port_id, allowed_address_pairs=allowed_address_pairs)
allowed_address_pair = body['port']['allowed_address_pairs']
- self.assertItemsEqual(allowed_address_pair, allowed_address_pairs)
+ self.assertCountEqual(allowed_address_pair, allowed_address_pairs)
@decorators.idempotent_id('9599b337-272c-47fd-b3cf-509414414ac4')
def test_update_port_with_address_pair(self):
diff --git a/neutron_tempest_plugin/common/ip.py b/neutron_tempest_plugin/common/ip.py
index 83cd3d9..d981770 100644
--- a/neutron_tempest_plugin/common/ip.py
+++ b/neutron_tempest_plugin/common/ip.py
@@ -36,13 +36,19 @@
sudo = 'sudo'
ip_path = '/sbin/ip'
- def __init__(self, ssh_client=None, timeout=None):
+ def __init__(self, ssh_client=None, timeout=None, namespace=None):
self.ssh_client = ssh_client
self.timeout = timeout
+ self.namespace = namespace
def get_command(self, obj, *command):
- command_line = '{sudo!s} {ip_path!r} {object!s} {command!s}'.format(
- sudo=self.sudo, ip_path=self.ip_path, object=obj,
+ command_line = '{sudo!s} {ip_path!r} '.format(sudo=self.sudo,
+ ip_path=self.ip_path)
+ if self.namespace:
+ command_line += 'netns exec {ns_name!s} {ip_path!r} '.format(
+ ns_name=self.namespace, ip_path=self.ip_path)
+ command_line += '{object!s} {command!s}'.format(
+ object=obj,
command=subprocess.list2cmdline([str(c) for c in command]))
return command_line
@@ -84,6 +90,13 @@
self.add_address(address=subport_ip, device=subport_device)
return subport_device
+ def list_namespaces(self):
+ namespaces_output = self.execute("netns")
+ ns_list = []
+ for ns_line in namespaces_output.split("\n"):
+ ns_list.append(ns_line.split(" ", 1)[0])
+ return ns_list
+
def list_addresses(self, device=None, ip_addresses=None, port=None,
subnets=None):
command = ['list']
@@ -308,20 +321,24 @@
netaddr.IPNetwork(subnet['cidr']).prefixlen)
-def arp_table():
+def arp_table(namespace=None):
# 192.168.0.16 0x1 0x2 dc:a6:32:06:56:51 * enp0s31f6
regex_str = (r"([^ ]+)\s+(0x\d+)\s+(0x\d+)\s+(\w{2}\:\w{2}\:\w{2}\:\w{2}\:"
r"\w{2}\:\w{2})\s+([\w+\*]+)\s+([\-\w]+)")
regex = re.compile(regex_str)
arp_table = []
- with open('/proc/net/arp', 'r') as proc_file:
- for line in proc_file.readlines():
- m = regex.match(line)
- if m:
- arp_table.append(ARPregister(
- ip_address=m.group(1), hw_type=m.group(2),
- flags=m.group(3), mac_address=m.group(4),
- mask=m.group(5), device=m.group(6)))
+ cmd = ""
+ if namespace:
+ cmd = "sudo ip netns exec %s " % namespace
+ cmd += "cat /proc/net/arp"
+ arp_entries = shell.execute(cmd).stdout.split("\n")
+ for line in arp_entries:
+ m = regex.match(line)
+ if m:
+ arp_table.append(ARPregister(
+ ip_address=m.group(1), hw_type=m.group(2),
+ flags=m.group(3), mac_address=m.group(4),
+ mask=m.group(5), device=m.group(6)))
return arp_table
diff --git a/neutron_tempest_plugin/scenario/base.py b/neutron_tempest_plugin/scenario/base.py
index 402a901..2eb45f8 100644
--- a/neutron_tempest_plugin/scenario/base.py
+++ b/neutron_tempest_plugin/scenario/base.py
@@ -21,6 +21,7 @@
from neutron_lib.api import validators
from neutron_lib import constants as neutron_lib_constants
from oslo_log import log
+from paramiko import ssh_exception as ssh_exc
from tempest.common.utils import net_utils
from tempest.common import waiters
from tempest.lib.common.utils import data_utils
@@ -121,9 +122,11 @@
if not kwargs.get('security_groups'):
kwargs['security_groups'] = [{'name': 'default'}]
- client = self.os_primary.servers_client
- if kwargs.get('availability_zone'):
- client = self.os_admin.servers_client
+ client = kwargs.pop('client', None)
+ if client is None:
+ client = self.os_primary.servers_client
+ if kwargs.get('availability_zone'):
+ client = self.os_admin.servers_client
server = client.create_server(
flavorRef=flavor_ref,
@@ -139,6 +142,10 @@
self.addCleanup(test_utils.call_and_ignore_notfound_exc,
client.delete_server,
server['server']['id'])
+
+ self.wait_for_server_active(server['server'], client=client)
+ self.wait_for_guest_os_ready(server['server'], client=client)
+
return server
@classmethod
@@ -252,7 +259,6 @@
server_kwargs['name'] = server_name
self.server = self.create_server(**server_kwargs)
- self.wait_for_server_active(self.server['server'])
self.port = self.client.list_ports(network_id=self.network['id'],
device_id=self.server[
'server']['id'])['ports'][0]
@@ -264,7 +270,7 @@
pkey=ssh_key, timeout=ssh_timeout)
try:
ssh_client.test_connection_auth()
- except lib_exc.SSHTimeout as ssh_e:
+ except (lib_exc.SSHTimeout, ssh_exc.AuthenticationException) as ssh_e:
LOG.debug(ssh_e)
self._log_console_output(servers)
self._log_local_network_status()
@@ -294,10 +300,20 @@
"for the console log", server['id'])
def _log_local_network_status(self):
- local_routes = ip_utils.IPCommand().list_routes()
- LOG.debug('Local routes:\n%s', '\n'.join(str(r) for r in local_routes))
+ self._log_ns_network_status()
+ for ns_name in ip_utils.IPCommand().list_namespaces():
+ self._log_ns_network_status(ns_name=ns_name)
+
+ def _log_ns_network_status(self, ns_name=None):
+ local_ips = ip_utils.IPCommand(namespace=ns_name).list_addresses()
+ LOG.debug('Namespace %s; IP Addresses:\n%s',
+ ns_name, '\n'.join(str(r) for r in local_ips))
+ local_routes = ip_utils.IPCommand(namespace=ns_name).list_routes()
+ LOG.debug('Namespace %s; Local routes:\n%s',
+ ns_name, '\n'.join(str(r) for r in local_routes))
arp_table = ip_utils.arp_table()
- LOG.debug('Local ARP table:\n%s', '\n'.join(str(r) for r in arp_table))
+ LOG.debug('Namespace %s; Local ARP table:\n%s',
+ ns_name, '\n'.join(str(r) for r in arp_table))
def _check_remote_connectivity(self, source, dest, count,
should_succeed=True,
@@ -379,12 +395,14 @@
fragmentation,
timeout=timeout, pattern=pattern,
forbid_packet_loss=forbid_packet_loss))
- except lib_exc.SSHTimeout as ssh_e:
+ except (lib_exc.SSHTimeout, ssh_exc.AuthenticationException) as ssh_e:
LOG.debug(ssh_e)
self._log_console_output(servers)
+ self._log_local_network_status()
raise
except AssertionError:
self._log_console_output(servers)
+ self._log_local_network_status()
raise
def ping_ip_address(self, ip_address, should_succeed=True,
@@ -452,6 +470,28 @@
self.wait_for_server_status(
server, constants.SERVER_STATUS_ACTIVE, client)
+ def wait_for_guest_os_ready(self, server, client=None):
+ if not CONF.compute_feature_enabled.console_output:
+ LOG.debug('Console output not supported, cannot check if server '
+ '%s is ready.', server['server']['id'])
+ return
+
+ client = client or self.os_primary.servers_client
+
+ def system_booted():
+ console_output = client.get_console_output(server['id'])['output']
+ for line in console_output.split('\n'):
+ if 'login:' in line.lower():
+ return True
+ return False
+
+ try:
+ utils.wait_until_true(system_booted, sleep=5)
+ except utils.WaitTimeout:
+ LOG.debug("No correct output in console of server %s found. "
+ "Guest operating system status can't be checked.",
+ server['id'])
+
def check_servers_hostnames(self, servers, timeout=None, log_errors=True,
external_port=None):
"""Compare hostnames of given servers with their names."""
@@ -472,15 +512,17 @@
**kwargs)
self.assertIn(server['name'],
ssh_client.exec_command('hostname'))
- except lib_exc.SSHTimeout as ssh_e:
+ except (lib_exc.SSHTimeout, ssh_exc.AuthenticationException) as ssh_e:
LOG.debug(ssh_e)
if log_errors:
self._log_console_output(servers)
+ self._log_local_network_status()
raise
except AssertionError as assert_e:
LOG.debug(assert_e)
if log_errors:
self._log_console_output(servers)
+ self._log_local_network_status()
raise
def ensure_nc_listen(self, ssh_client, port, protocol, echo_msg=None,
@@ -505,9 +547,10 @@
return ssh_client.execute_script(
get_ncat_server_cmd(port, protocol, echo_msg),
become_root=True, combine_stderr=True)
- except lib_exc.SSHTimeout as ssh_e:
+ except (lib_exc.SSHTimeout, ssh_exc.AuthenticationException) as ssh_e:
LOG.debug(ssh_e)
self._log_console_output(servers)
+ self._log_local_network_status()
raise
def nc_client(self, ip_address, port, protocol):
diff --git a/neutron_tempest_plugin/scenario/test_connectivity.py b/neutron_tempest_plugin/scenario/test_connectivity.py
index 1a7468a..66c4789 100644
--- a/neutron_tempest_plugin/scenario/test_connectivity.py
+++ b/neutron_tempest_plugin/scenario/test_connectivity.py
@@ -65,6 +65,7 @@
for vm in vms:
self.wait_for_server_active(vm['server'])
+ self.wait_for_guest_os_ready(vm['server'])
return vms
@@ -231,6 +232,7 @@
networks=[{'uuid': network['id']}],
security_groups=[{'name': self.secgroup['name']}])
self.wait_for_server_active(vm['server'])
+ self.wait_for_guest_os_ready(vm['server'])
vm_port = self.client.list_ports(
network_id=network['id'], device_id=vm['server']['id'])['ports'][0]
diff --git a/neutron_tempest_plugin/scenario/test_floatingip.py b/neutron_tempest_plugin/scenario/test_floatingip.py
index 6d4d830..8c8d163 100644
--- a/neutron_tempest_plugin/scenario/test_floatingip.py
+++ b/neutron_tempest_plugin/scenario/test_floatingip.py
@@ -417,6 +417,7 @@
servers.append(server)
for server in servers:
self.wait_for_server_active(server)
+ self.wait_for_guest_os_ready(server)
self.fip = self.create_floatingip(port=ports[0])
self.check_connectivity(self.fip['floating_ip_address'],
diff --git a/neutron_tempest_plugin/scenario/test_internal_dns.py b/neutron_tempest_plugin/scenario/test_internal_dns.py
index d19286c..c620233 100644
--- a/neutron_tempest_plugin/scenario/test_internal_dns.py
+++ b/neutron_tempest_plugin/scenario/test_internal_dns.py
@@ -52,6 +52,7 @@
{'name': self.security_groups[-1]['name']}],
name='leia')
self.wait_for_server_active(leia['server'])
+ self.wait_for_guest_os_ready(leia['server'])
ssh_client = ssh.Client(
self.fip['floating_ip_address'],
diff --git a/neutron_tempest_plugin/scenario/test_ipv6.py b/neutron_tempest_plugin/scenario/test_ipv6.py
index 02e2846..732c96d 100644
--- a/neutron_tempest_plugin/scenario/test_ipv6.py
+++ b/neutron_tempest_plugin/scenario/test_ipv6.py
@@ -15,6 +15,7 @@
from neutron_lib import constants as lib_constants
from oslo_log import log
+from paramiko import ssh_exception as ssh_exc
from tempest.common import utils as tempest_utils
from tempest.lib.common.utils import data_utils
from tempest.lib import decorators
@@ -108,16 +109,22 @@
return True
return False
- # Set NIC with IPv6 to be UP and wait until IPv6 address will be
- # configured on this NIC
- turn_nic6_on(ssh_client, ipv6_port)
- # And check if IPv6 address will be properly configured on this NIC
- utils.wait_until_true(
- lambda: guest_has_address(ipv6_address),
- timeout=120,
- exception=RuntimeError(
- "Timed out waiting for IP address {!r} to be configured in "
- "the VM {!r}.".format(ipv6_address, vm['id'])))
+ try:
+ # Set NIC with IPv6 to be UP and wait until IPv6 address will be
+ # configured on this NIC
+ turn_nic6_on(ssh_client, ipv6_port)
+ # And check if IPv6 address will be properly configured on this NIC
+ utils.wait_until_true(
+ lambda: guest_has_address(ipv6_address),
+ timeout=120,
+ exception=RuntimeError(
+ "Timed out waiting for IP address {!r} to be configured "
+ "in the VM {!r}.".format(ipv6_address, vm['id'])))
+ except (lib_exc.SSHTimeout, ssh_exc.AuthenticationException) as ssh_e:
+ LOG.debug(ssh_e)
+ self._log_console_output([vm])
+ self._log_local_network_status()
+ raise
def _test_ipv6_hotplug(self, ra_mode, address_mode):
ipv6_networks = [self.create_network() for _ in range(2)]
@@ -138,6 +145,7 @@
}
vm = self.create_server(**server_kwargs)['server']
self.wait_for_server_active(vm)
+ self.wait_for_guest_os_ready(vm)
ipv4_port = self.client.list_ports(
network_id=self.network['id'],
device_id=vm['id'])['ports'][0]
diff --git a/neutron_tempest_plugin/scenario/test_metadata.py b/neutron_tempest_plugin/scenario/test_metadata.py
index c78ff69..3897183 100644
--- a/neutron_tempest_plugin/scenario/test_metadata.py
+++ b/neutron_tempest_plugin/scenario/test_metadata.py
@@ -126,6 +126,7 @@
vm = self._create_server_with_network(
self.network, use_advanced_image=use_advanced_image)
self.wait_for_server_active(server=vm.server)
+ self.wait_for_guest_os_ready(vm.server)
self._assert_has_ssh_connectivity(vm.ssh_client)
interface = self._get_primary_interface(vm.ssh_client)
diff --git a/neutron_tempest_plugin/scenario/test_multicast.py b/neutron_tempest_plugin/scenario/test_multicast.py
index b2ea8ae..79f835e 100644
--- a/neutron_tempest_plugin/scenario/test_multicast.py
+++ b/neutron_tempest_plugin/scenario/test_multicast.py
@@ -201,6 +201,7 @@
security_groups=[{'name': self.secgroup['security_group']['name']}]
)['server']
self.wait_for_server_active(server)
+ self.wait_for_guest_os_ready(server)
port = self.client.list_ports(
network_id=self.network['id'], device_id=server['id'])['ports'][0]
server['fip'] = self.create_floatingip(port=port)
diff --git a/neutron_tempest_plugin/scenario/test_port_forwardings.py b/neutron_tempest_plugin/scenario/test_port_forwardings.py
index 4080bca..fb6410c 100644
--- a/neutron_tempest_plugin/scenario/test_port_forwardings.py
+++ b/neutron_tempest_plugin/scenario/test_port_forwardings.py
@@ -70,6 +70,7 @@
networks=[{'port': port['id']}])['server']
server['name'] = name
self.wait_for_server_active(server)
+ self.wait_for_guest_os_ready(server)
server['port_forwarding_tcp'] = self.create_port_forwarding(
self.fip['id'],
internal_port_id=port['id'],
@@ -255,6 +256,7 @@
name=name, networks=[{'port': port['id']}])['server']
server['name'] = name
self.wait_for_server_active(server)
+ self.wait_for_guest_os_ready(server)
# Add a second fixed_ip address to port (same subnet)
internal_subnet_id = port['fixed_ips'][0]['subnet_id']
diff --git a/neutron_tempest_plugin/scenario/test_qos.py b/neutron_tempest_plugin/scenario/test_qos.py
index 938d2b0..6cd6b25 100644
--- a/neutron_tempest_plugin/scenario/test_qos.py
+++ b/neutron_tempest_plugin/scenario/test_qos.py
@@ -175,6 +175,7 @@
server = self.create_server(**server_kwargs)
self.wait_for_server_active(server['server'])
+ self.wait_for_guest_os_ready(server['server'])
self.check_connectivity(self.fip['floating_ip_address'],
CONF.validation.image_ssh_user,
keypair['private_key'])
diff --git a/neutron_tempest_plugin/scenario/test_security_groups.py b/neutron_tempest_plugin/scenario/test_security_groups.py
index 23a5224..9059a2f 100644
--- a/neutron_tempest_plugin/scenario/test_security_groups.py
+++ b/neutron_tempest_plugin/scenario/test_security_groups.py
@@ -33,13 +33,15 @@
required_extensions = ['router', 'security-group']
def _verify_http_connection(self, ssh_client, ssh_server,
- test_ip, test_port, should_pass=True):
+ test_ip, test_port, servers, should_pass=True):
"""Verify if HTTP connection works using remote hosts.
:param ssh.Client ssh_client: The client host active SSH client.
:param ssh.Client ssh_server: The HTTP server host active SSH client.
:param string test_ip: IP address of HTTP server
:param string test_port: Port of HTTP server
+ :param list servers: List of servers for which console output will be
+ logged in case when test case
:param bool should_pass: Wheter test should pass or not.
:return: if passed or not
@@ -57,6 +59,8 @@
except Exception as e:
if not should_pass:
return
+ self._log_console_output(servers)
+ self._log_local_network_status()
raise e
@classmethod
@@ -378,6 +382,7 @@
ssh_clients[0],
ssh_clients[2],
test_ip, port,
+ servers,
should_pass=False)
# add two remote-group rules with port-ranges
@@ -399,7 +404,8 @@
self._verify_http_connection(
ssh_clients[0],
ssh_clients[2],
- test_ip, port)
+ test_ip, port,
+ servers)
@decorators.idempotent_id('f07d0159-8f9e-4faa-87f5-a869ab0ad490')
def test_intra_sg_isolation(self):
diff --git a/neutron_tempest_plugin/scenario/test_trunk.py b/neutron_tempest_plugin/scenario/test_trunk.py
index 585af06..03bdd11 100644
--- a/neutron_tempest_plugin/scenario/test_trunk.py
+++ b/neutron_tempest_plugin/scenario/test_trunk.py
@@ -161,6 +161,7 @@
def _configure_vlan_subport(self, vm, vlan_tag, vlan_subnet):
self.wait_for_server_active(server=vm.server)
+ self.wait_for_guest_os_ready(vm.server)
self._wait_for_trunk(trunk=vm.trunk)
self._wait_for_port(port=vm.port)
self._wait_for_port(port=vm.subport)
@@ -199,6 +200,7 @@
vm2 = self._create_server_with_trunk_port()
for vm in (vm1, vm2):
self.wait_for_server_active(server=vm.server)
+ self.wait_for_guest_os_ready(vm.server)
self._wait_for_trunk(vm.trunk)
self._assert_has_ssh_connectivity(vm.ssh_client)
@@ -325,6 +327,7 @@
use_advanced_image=use_advanced_image)
for role in ['migrate', 'connection_test']:
self.wait_for_server_active(servers[role].server)
+ self.wait_for_guest_os_ready(servers[role].server)
self._configure_vlan_subport(vm=servers[role],
vlan_tag=vlan_tag,
vlan_subnet=vlan_subnet)
@@ -377,6 +380,7 @@
vlan_subnet=vlan_subnet)
for vm in vms:
self.wait_for_server_active(vm.server)
+ self.wait_for_guest_os_ready(vm.server)
# allow ICMP traffic
sg_rule = self.create_pingable_secgroup_rule(self.security_group['id'])