|  | # Copyright 2016 Red Hat, Inc. | 
|  | # All Rights Reserved. | 
|  | # | 
|  | #    Licensed under the Apache License, Version 2.0 (the "License"); you may | 
|  | #    not use this file except in compliance with the License. You may obtain | 
|  | #    a copy of the License at | 
|  | # | 
|  | #         http://www.apache.org/licenses/LICENSE-2.0 | 
|  | # | 
|  | #    Unless required by applicable law or agreed to in writing, software | 
|  | #    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT | 
|  | #    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the | 
|  | #    License for the specific language governing permissions and limitations | 
|  | #    under the License. | 
|  |  | 
|  | import re | 
|  | import subprocess | 
|  |  | 
|  | from debtcollector import removals | 
|  | import netaddr | 
|  | from neutron_lib.api import validators | 
|  | from neutron_lib import constants as neutron_lib_constants | 
|  | from oslo_log import log | 
|  | from packaging import version as packaging_version | 
|  | from paramiko import ssh_exception as ssh_exc | 
|  | from tempest.common.utils import net_utils | 
|  | from tempest.common import waiters | 
|  | from tempest.lib.common.utils import data_utils | 
|  | from tempest.lib.common.utils import test_utils | 
|  | from tempest.lib import exceptions as lib_exc | 
|  |  | 
|  | from neutron_tempest_plugin.api import base as base_api | 
|  | from neutron_tempest_plugin.common import ip as ip_utils | 
|  | from neutron_tempest_plugin.common import shell | 
|  | from neutron_tempest_plugin.common import ssh | 
|  | from neutron_tempest_plugin.common import utils | 
|  | from neutron_tempest_plugin import config | 
|  | from neutron_tempest_plugin import exceptions | 
|  | from neutron_tempest_plugin.scenario import constants | 
|  |  | 
|  | CONF = config.CONF | 
|  | LOG = log.getLogger(__name__) | 
|  | SSH_EXC_TUPLE = (lib_exc.SSHTimeout, | 
|  | ssh_exc.AuthenticationException, | 
|  | ssh_exc.NoValidConnectionsError, | 
|  | ConnectionResetError) | 
|  |  | 
|  |  | 
|  | def get_ncat_version(ssh_client=None): | 
|  | cmd = "ncat --version 2>&1" | 
|  | try: | 
|  | version_result = shell.execute(cmd, ssh_client=ssh_client).stdout | 
|  | except exceptions.ShellCommandFailed: | 
|  | m = None | 
|  | else: | 
|  | m = re.match(r"Ncat: Version ([\d.]+) *.", version_result) | 
|  | # NOTE(slaweq): by default lets assume we have ncat 7.60 which is in Ubuntu | 
|  | # 18.04 which is used on u/s gates | 
|  | return packaging_version.Version(m.group(1) if m else '7.60') | 
|  |  | 
|  |  | 
|  | def get_ncat_server_cmd(port, protocol, msg=None): | 
|  | udp = '' | 
|  | if protocol.lower() == neutron_lib_constants.PROTO_NAME_UDP: | 
|  | udp = '-u' | 
|  | cmd = "nc %(udp)s -p %(port)s -lk " % { | 
|  | 'udp': udp, 'port': port} | 
|  | if msg: | 
|  | if CONF.neutron_plugin_options.default_image_is_advanced: | 
|  | cmd += "-c 'echo %s' " % msg | 
|  | else: | 
|  | cmd += "-e echo %s " % msg | 
|  | cmd += "< /dev/zero &{0}sleep 0.1{0}".format('\n') | 
|  | return cmd | 
|  |  | 
|  |  | 
|  | def get_ncat_client_cmd(ip_address, port, protocol, ssh_client=None): | 
|  | cmd = 'echo "knock knock" | nc ' | 
|  | ncat_version = get_ncat_version(ssh_client=ssh_client) | 
|  | if ncat_version > packaging_version.Version('7.60'): | 
|  | cmd += '-d 1 ' | 
|  | if protocol.lower() == neutron_lib_constants.PROTO_NAME_UDP: | 
|  | cmd += '-u ' | 
|  | if ncat_version > packaging_version.Version('7.60'): | 
|  | cmd += '-z ' | 
|  | cmd += '-w 1 %(host)s %(port)s' % {'host': ip_address, 'port': port} | 
|  | return cmd | 
|  |  | 
|  |  | 
|  | class BaseTempestTestCase(base_api.BaseNetworkTest): | 
|  |  | 
|  | def create_server(self, flavor_ref, image_ref, key_name, networks, | 
|  | **kwargs): | 
|  | """Create a server using tempest lib | 
|  |  | 
|  | All the parameters are the ones used in Compute API | 
|  | * - Kwargs that require admin privileges | 
|  |  | 
|  | Args: | 
|  | flavor_ref(str): The flavor of the server to be provisioned. | 
|  | image_ref(str):  The image of the server to be provisioned. | 
|  | key_name(str): SSH key to to be used to connect to the | 
|  | provisioned server. | 
|  | networks(list): List of dictionaries where each represent | 
|  | an interface to be attached to the server. For network | 
|  | it should be {'uuid': network_uuid} and for port it should | 
|  | be {'port': port_uuid} | 
|  | kwargs: | 
|  | name(str): Name of the server to be provisioned. | 
|  | security_groups(list): List of dictionaries where | 
|  | the keys is 'name' and the value is the name of | 
|  | the security group. If it's not passed the default | 
|  | security group will be used. | 
|  | availability_zone(str)*: The availability zone that | 
|  | the instance will be in. | 
|  | You can request a specific az without actually creating one, | 
|  | Just pass 'X:Y' where X is the default availability | 
|  | zone, and Y is the compute host name. | 
|  | """ | 
|  |  | 
|  | kwargs.setdefault('name', data_utils.rand_name('server-test')) | 
|  |  | 
|  | # We cannot use setdefault() here because caller could have passed | 
|  | # security_groups=None and we don't want to pass None to | 
|  | # client.create_server() | 
|  | if not kwargs.get('security_groups'): | 
|  | kwargs['security_groups'] = [{'name': 'default'}] | 
|  |  | 
|  | client = kwargs.pop('client', None) | 
|  | if client is None: | 
|  | client = self.os_primary.servers_client | 
|  | if kwargs.get('availability_zone'): | 
|  | client = self.os_admin.servers_client | 
|  |  | 
|  | server = client.create_server( | 
|  | flavorRef=flavor_ref, | 
|  | imageRef=image_ref, | 
|  | key_name=key_name, | 
|  | networks=networks, | 
|  | **kwargs) | 
|  |  | 
|  | self.addCleanup(test_utils.call_and_ignore_notfound_exc, | 
|  | waiters.wait_for_server_termination, | 
|  | client, | 
|  | server['server']['id']) | 
|  | self.addCleanup(test_utils.call_and_ignore_notfound_exc, | 
|  | client.delete_server, | 
|  | server['server']['id']) | 
|  |  | 
|  | self.wait_for_server_active(server['server'], client=client) | 
|  | self.wait_for_guest_os_ready(server['server'], client=client) | 
|  |  | 
|  | return server | 
|  |  | 
|  | @classmethod | 
|  | def create_secgroup_rules(cls, rule_list, secgroup_id=None, | 
|  | client=None): | 
|  | client = client or cls.os_primary.network_client | 
|  | if not secgroup_id: | 
|  | sgs = client.list_security_groups()['security_groups'] | 
|  | for sg in sgs: | 
|  | if sg['name'] == constants.DEFAULT_SECURITY_GROUP: | 
|  | secgroup_id = sg['id'] | 
|  | break | 
|  | resp = [] | 
|  | for rule in rule_list: | 
|  | direction = rule.pop('direction') | 
|  | resp.append(client.create_security_group_rule( | 
|  | direction=direction, | 
|  | security_group_id=secgroup_id, | 
|  | **rule)['security_group_rule']) | 
|  | return resp | 
|  |  | 
|  | @classmethod | 
|  | def create_loginable_secgroup_rule(cls, secgroup_id=None, | 
|  | client=None): | 
|  | """This rule is intended to permit inbound ssh | 
|  |  | 
|  | Allowing ssh traffic from all sources, so no group_id is | 
|  | provided. | 
|  | Setting a group_id would only permit traffic from ports | 
|  | belonging to the same security group. | 
|  | """ | 
|  | return cls.create_security_group_rule( | 
|  | security_group_id=secgroup_id, | 
|  | client=client, | 
|  | protocol=neutron_lib_constants.PROTO_NAME_TCP, | 
|  | direction=neutron_lib_constants.INGRESS_DIRECTION, | 
|  | port_range_min=22, | 
|  | port_range_max=22) | 
|  |  | 
|  | @classmethod | 
|  | def create_ingress_metadata_secgroup_rule(cls, secgroup_id=None): | 
|  | """This rule is intended to permit inbound metadata traffic | 
|  |  | 
|  | Allowing ingress traffic from metadata server, required only for | 
|  | stateless security groups. | 
|  | """ | 
|  | # NOTE(slaweq): in case of stateless security groups, there is no | 
|  | # "related" or "established" traffic matching at all so even if | 
|  | # egress traffic to 169.254.169.254 is allowed by default SG, we | 
|  | # need to explicitly allow ingress traffic from the metadata server | 
|  | # to be able to receive responses in the guest vm | 
|  | cls.create_security_group_rule( | 
|  | security_group_id=secgroup_id, | 
|  | direction=neutron_lib_constants.INGRESS_DIRECTION, | 
|  | protocol=neutron_lib_constants.PROTO_NAME_TCP, | 
|  | remote_ip_prefix='169.254.169.254/32', | 
|  | description='metadata out' | 
|  | ) | 
|  |  | 
|  | @classmethod | 
|  | def create_pingable_secgroup_rule(cls, secgroup_id=None, | 
|  | client=None): | 
|  | """This rule is intended to permit inbound ping | 
|  |  | 
|  | """ | 
|  | return cls.create_security_group_rule( | 
|  | security_group_id=secgroup_id, client=client, | 
|  | protocol=neutron_lib_constants.PROTO_NAME_ICMP, | 
|  | direction=neutron_lib_constants.INGRESS_DIRECTION) | 
|  |  | 
|  | @classmethod | 
|  | def create_router_by_client(cls, is_admin=False, **kwargs): | 
|  | kwargs.update({'router_name': data_utils.rand_name('router'), | 
|  | 'admin_state_up': True, | 
|  | 'external_network_id': CONF.network.public_network_id}) | 
|  | if not is_admin: | 
|  | router = cls.create_router(**kwargs) | 
|  | else: | 
|  | router = cls.create_admin_router(**kwargs) | 
|  | LOG.debug("Created router %s", router['name']) | 
|  | cls._wait_for_router_ha_active(router['id']) | 
|  | return router | 
|  |  | 
|  | @classmethod | 
|  | def _wait_for_router_ha_active(cls, router_id): | 
|  | router = cls.os_admin.network_client.show_router(router_id)['router'] | 
|  | if not router.get('ha') or cls.is_driver_ovn: | 
|  | return | 
|  |  | 
|  | def _router_active_on_l3_agent(): | 
|  | agents = cls.os_admin.network_client.list_l3_agents_hosting_router( | 
|  | router_id)['agents'] | 
|  | return "active" in [agent['ha_state'] for agent in agents] | 
|  |  | 
|  | error_msg = ( | 
|  | "Router %s is not active on any of the L3 agents" % router_id) | 
|  | # NOTE(slaweq): timeout here should be lower for sure, but due to | 
|  | # the bug https://launchpad.net/bugs/1923633 let's wait even 10 | 
|  | # minutes until router will be active on some of the L3 agents | 
|  | utils.wait_until_true(_router_active_on_l3_agent, | 
|  | timeout=600, sleep=5, | 
|  | exception=lib_exc.TimeoutException(error_msg)) | 
|  |  | 
|  | @classmethod | 
|  | def skip_if_no_extension_enabled_in_l3_agents(cls, extension): | 
|  | l3_agents = cls.os_admin.network_client.list_agents( | 
|  | binary='neutron-l3-agent')['agents'] | 
|  | if not l3_agents: | 
|  | # the tests should not be skipped when neutron-l3-agent does not | 
|  | # exist (this validation doesn't apply to the setups like | 
|  | # e.g. ML2/OVN) | 
|  | return | 
|  | for agent in l3_agents: | 
|  | if extension in agent['configurations'].get('extensions', []): | 
|  | return | 
|  | raise cls.skipTest("No L3 agent with '%s' extension enabled found." % | 
|  | extension) | 
|  |  | 
|  | @removals.remove(version='Stein', | 
|  | message="Please use create_floatingip method instead of " | 
|  | "create_and_associate_floatingip.") | 
|  | def create_and_associate_floatingip(self, port_id, client=None): | 
|  | client = client or self.os_primary.network_client | 
|  | return self.create_floatingip(port_id=port_id, client=client) | 
|  |  | 
|  | def create_interface(cls, server_id, port_id, client=None): | 
|  | client = client or cls.os_primary.interfaces_client | 
|  | body = client.create_interface(server_id, port_id=port_id) | 
|  | return body['interfaceAttachment'] | 
|  |  | 
|  | def delete_interface(cls, server_id, port_id, client=None): | 
|  | client = client or cls.os_primary.interfaces_client | 
|  | client.delete_interface(server_id, port_id=port_id) | 
|  |  | 
|  | def setup_network_and_server(self, router=None, server_name=None, | 
|  | network=None, use_stateless_sg=False, | 
|  | create_fip=True, router_client=None, | 
|  | **kwargs): | 
|  | """Create network resources and a server. | 
|  |  | 
|  | Creating a network, subnet, router, keypair, security group | 
|  | and a server. | 
|  | """ | 
|  | self.network = network or self.create_network() | 
|  | LOG.debug("Created network %s", self.network['name']) | 
|  | self.subnet = self.create_subnet(self.network) | 
|  | LOG.debug("Created subnet %s", self.subnet['id']) | 
|  |  | 
|  | sg_args = { | 
|  | 'name': data_utils.rand_name('secgroup') | 
|  | } | 
|  | if use_stateless_sg: | 
|  | sg_args['stateful'] = False | 
|  | secgroup = self.os_primary.network_client.create_security_group( | 
|  | **sg_args) | 
|  | LOG.debug("Created security group %s", | 
|  | secgroup['security_group']['name']) | 
|  | self.security_groups.append(secgroup['security_group']) | 
|  | if not router: | 
|  | router = self.create_router_by_client(**kwargs) | 
|  | self.create_router_interface(router['id'], self.subnet['id'], | 
|  | client=router_client) | 
|  | self.keypair = self.create_keypair() | 
|  | self.create_loginable_secgroup_rule( | 
|  | secgroup_id=secgroup['security_group']['id']) | 
|  | if use_stateless_sg: | 
|  | self.create_ingress_metadata_secgroup_rule( | 
|  | secgroup_id=secgroup['security_group']['id']) | 
|  |  | 
|  | server_kwargs = { | 
|  | 'flavor_ref': CONF.compute.flavor_ref, | 
|  | 'image_ref': CONF.compute.image_ref, | 
|  | 'key_name': self.keypair['name'], | 
|  | 'networks': [{'uuid': self.network['id']}], | 
|  | 'security_groups': [{'name': secgroup['security_group']['name']}], | 
|  | } | 
|  | if server_name is not None: | 
|  | server_kwargs['name'] = server_name | 
|  |  | 
|  | self.server = self.create_server(**server_kwargs) | 
|  | self.port = self.client.list_ports(network_id=self.network['id'], | 
|  | device_id=self.server[ | 
|  | 'server']['id'])['ports'][0] | 
|  |  | 
|  | if create_fip: | 
|  | self.fip = self.create_floatingip(port=self.port) | 
|  |  | 
|  | def check_connectivity(self, host=None, ssh_user=None, ssh_key=None, | 
|  | servers=None, ssh_timeout=None, ssh_client=None): | 
|  | # Either ssh_client or ssh_user+ssh_key is mandatory. | 
|  | if ssh_client is None: | 
|  | ssh_client = ssh.Client(host, ssh_user, | 
|  | pkey=ssh_key, timeout=ssh_timeout) | 
|  | try: | 
|  | ssh_client.test_connection_auth() | 
|  | except SSH_EXC_TUPLE as ssh_e: | 
|  | LOG.debug(ssh_e) | 
|  | self._log_console_output(servers) | 
|  | self._log_local_network_status() | 
|  | raise | 
|  |  | 
|  | def _log_console_output(self, servers=None): | 
|  | if not CONF.compute_feature_enabled.console_output: | 
|  | LOG.debug('Console output not supported, cannot log') | 
|  | return | 
|  | if not servers: | 
|  | servers = self.os_primary.servers_client.list_servers() | 
|  | servers = servers['servers'] | 
|  | for server in servers: | 
|  | # NOTE(slaweq): sometimes servers are passed in dictionary with | 
|  | # "server" key as first level key and in other cases it may be that | 
|  | # it is just the "inner" dict without "server" key. Lets try to | 
|  | # handle both cases | 
|  | server = server.get("server") or server | 
|  | try: | 
|  | console_output = ( | 
|  | self.os_primary.servers_client.get_console_output( | 
|  | server['id'])['output']) | 
|  | LOG.debug('Console output for %s\nbody=\n%s', | 
|  | server['id'], console_output) | 
|  | except lib_exc.NotFound: | 
|  | LOG.debug("Server %s disappeared(deleted) while looking " | 
|  | "for the console log", server['id']) | 
|  |  | 
|  | def _log_local_network_status(self): | 
|  | self._log_ns_network_status() | 
|  | for ns_name in ip_utils.IPCommand().list_namespaces(): | 
|  | self._log_ns_network_status(ns_name=ns_name) | 
|  |  | 
|  | def _log_ns_network_status(self, ns_name=None): | 
|  | try: | 
|  | local_ips = ip_utils.IPCommand(namespace=ns_name).list_addresses() | 
|  | local_routes = ip_utils.IPCommand(namespace=ns_name).list_routes() | 
|  | arp_table = ip_utils.arp_table(namespace=ns_name) | 
|  | iptables = ip_utils.list_iptables(namespace=ns_name) | 
|  | lsockets = ip_utils.list_listening_sockets(namespace=ns_name) | 
|  | except exceptions.ShellCommandFailed: | 
|  | LOG.debug('Namespace %s has been deleted synchronously during the ' | 
|  | 'host network collection process', ns_name) | 
|  | return | 
|  |  | 
|  | LOG.debug('Namespace %s; IP Addresses:\n%s', | 
|  | ns_name, '\n'.join(str(r) for r in local_ips)) | 
|  | LOG.debug('Namespace %s; Local routes:\n%s', | 
|  | ns_name, '\n'.join(str(r) for r in local_routes)) | 
|  | LOG.debug('Namespace %s; Local ARP table:\n%s', | 
|  | ns_name, '\n'.join(str(r) for r in arp_table)) | 
|  | LOG.debug('Namespace %s; Local iptables:\n%s', ns_name, iptables) | 
|  | LOG.debug('Namespace %s; Listening sockets:\n%s', ns_name, lsockets) | 
|  |  | 
|  | def _check_remote_connectivity(self, source, dest, count, | 
|  | should_succeed=True, | 
|  | nic=None, mtu=None, fragmentation=True, | 
|  | timeout=None, pattern=None, | 
|  | forbid_packet_loss=False, | 
|  | check_response_ip=True): | 
|  | """check ping server via source ssh connection | 
|  |  | 
|  | :param source: RemoteClient: an ssh connection from which to ping | 
|  | :param dest: and IP to ping against | 
|  | :param count: Number of ping packet(s) to send | 
|  | :param should_succeed: boolean should ping succeed or not | 
|  | :param nic: specific network interface to ping from | 
|  | :param mtu: mtu size for the packet to be sent | 
|  | :param fragmentation: Flag for packet fragmentation | 
|  | :param timeout: Timeout for all ping packet(s) to succeed | 
|  | :param pattern: hex digits included in ICMP messages | 
|  | :param forbid_packet_loss: forbid or allow some lost packets | 
|  | :param check_response_ip: check response ip | 
|  | :returns: boolean -- should_succeed == ping | 
|  | :returns: ping is false if ping failed | 
|  | """ | 
|  | def ping_host(source, host, count, | 
|  | size=CONF.validation.ping_size, nic=None, mtu=None, | 
|  | fragmentation=True, pattern=None): | 
|  | IP_VERSION_4 = neutron_lib_constants.IP_VERSION_4 | 
|  | IP_VERSION_6 = neutron_lib_constants.IP_VERSION_6 | 
|  |  | 
|  | # Use 'ping6' for IPv6 addresses, 'ping' for IPv4 and hostnames | 
|  | ip_version = ( | 
|  | IP_VERSION_6 if netaddr.valid_ipv6(host) else IP_VERSION_4) | 
|  | cmd = ( | 
|  | 'ping6' if ip_version == IP_VERSION_6 else 'ping') | 
|  | if nic: | 
|  | cmd = 'sudo {cmd} -I {nic}'.format(cmd=cmd, nic=nic) | 
|  | if mtu: | 
|  | if not fragmentation: | 
|  | cmd += ' -M do' | 
|  | size = str(net_utils.get_ping_payload_size( | 
|  | mtu=mtu, ip_version=ip_version)) | 
|  | if pattern: | 
|  | cmd += ' -p {pattern}'.format(pattern=pattern) | 
|  | cmd += ' -c{0} -W{0} -s{1} {2}'.format(count, size, host) | 
|  | return source.exec_command(cmd) | 
|  |  | 
|  | def ping_remote(): | 
|  | try: | 
|  | result = ping_host(source, dest, count, nic=nic, mtu=mtu, | 
|  | fragmentation=fragmentation, | 
|  | pattern=pattern) | 
|  |  | 
|  | except lib_exc.SSHExecCommandFailed: | 
|  | LOG.warning('Failed to ping IP: %s via a ssh connection ' | 
|  | 'from: %s.', dest, source.host) | 
|  | return not should_succeed | 
|  | LOG.debug('ping result: %s', result) | 
|  |  | 
|  | if forbid_packet_loss and ' 0% packet loss' not in result: | 
|  | LOG.debug('Packet loss detected') | 
|  | return not should_succeed | 
|  |  | 
|  | if (check_response_ip and | 
|  | validators.validate_ip_address(dest) is None): | 
|  | # Assert that the return traffic was from the correct | 
|  | # source address. | 
|  | from_source = 'from %s' % dest | 
|  | self.assertIn(from_source, result) | 
|  | return should_succeed | 
|  |  | 
|  | return test_utils.call_until_true( | 
|  | ping_remote, timeout or CONF.validation.ping_timeout, 1) | 
|  |  | 
|  | def check_remote_connectivity(self, source, dest, should_succeed=True, | 
|  | nic=None, mtu=None, fragmentation=True, | 
|  | servers=None, timeout=None, | 
|  | ping_count=CONF.validation.ping_count, | 
|  | pattern=None, forbid_packet_loss=False, | 
|  | check_response_ip=True): | 
|  | try: | 
|  | self.assertTrue(self._check_remote_connectivity( | 
|  | source, dest, ping_count, should_succeed, nic, mtu, | 
|  | fragmentation, | 
|  | timeout=timeout, pattern=pattern, | 
|  | forbid_packet_loss=forbid_packet_loss, | 
|  | check_response_ip=check_response_ip)) | 
|  | except SSH_EXC_TUPLE as ssh_e: | 
|  | LOG.debug(ssh_e) | 
|  | self._log_console_output(servers) | 
|  | self._log_local_network_status() | 
|  | raise | 
|  | except AssertionError: | 
|  | self._log_console_output(servers) | 
|  | self._log_local_network_status() | 
|  | raise | 
|  |  | 
|  | def ping_ip_address(self, ip_address, should_succeed=True, | 
|  | ping_timeout=None, mtu=None): | 
|  | # the code is taken from tempest/scenario/manager.py in tempest git | 
|  | timeout = ping_timeout or CONF.validation.ping_timeout | 
|  | cmd = ['ping', '-c1', '-w1'] | 
|  |  | 
|  | if mtu: | 
|  | cmd += [ | 
|  | # don't fragment | 
|  | '-M', 'do', | 
|  | # ping receives just the size of ICMP payload | 
|  | '-s', str(net_utils.get_ping_payload_size(mtu, 4)) | 
|  | ] | 
|  | cmd.append(ip_address) | 
|  |  | 
|  | def ping(): | 
|  | proc = subprocess.Popen(cmd, | 
|  | stdout=subprocess.PIPE, | 
|  | stderr=subprocess.PIPE) | 
|  | proc.communicate() | 
|  |  | 
|  | return (proc.returncode == 0) == should_succeed | 
|  |  | 
|  | caller = test_utils.find_test_caller() | 
|  | LOG.debug('%(caller)s begins to ping %(ip)s in %(timeout)s sec and the' | 
|  | ' expected result is %(should_succeed)s', { | 
|  | 'caller': caller, 'ip': ip_address, 'timeout': timeout, | 
|  | 'should_succeed': | 
|  | 'reachable' if should_succeed else 'unreachable' | 
|  | }) | 
|  | result = test_utils.call_until_true(ping, timeout, 1) | 
|  |  | 
|  | # To make sure ping_ip_address called by test works | 
|  | # as expected. | 
|  | self.assertTrue(result) | 
|  |  | 
|  | LOG.debug('%(caller)s finishes ping %(ip)s in %(timeout)s sec and the ' | 
|  | 'ping result is %(result)s', { | 
|  | 'caller': caller, 'ip': ip_address, 'timeout': timeout, | 
|  | 'result': 'expected' if result else 'unexpected' | 
|  | }) | 
|  | return result | 
|  |  | 
|  | def wait_for_server_status(self, server, status, client=None, **kwargs): | 
|  | """Waits for a server to reach a given status. | 
|  |  | 
|  | :param server:  mapping having schema {'id': <server_id>} | 
|  | :param status: string status to wait for (es: 'ACTIVE') | 
|  | :param clien:  servers client (self.os_primary.servers_client as | 
|  | default value) | 
|  | """ | 
|  |  | 
|  | client = client or self.os_primary.servers_client | 
|  | waiters.wait_for_server_status(client, server['id'], status, **kwargs) | 
|  |  | 
|  | def wait_for_server_active(self, server, client=None): | 
|  | """Waits for a server to reach active status. | 
|  |  | 
|  | :param server:  mapping having schema {'id': <server_id>} | 
|  | :param clien:  servers client (self.os_primary.servers_client as | 
|  | default value) | 
|  | """ | 
|  | self.wait_for_server_status( | 
|  | server, constants.SERVER_STATUS_ACTIVE, client) | 
|  |  | 
|  | def wait_for_guest_os_ready(self, server, client=None): | 
|  | if not CONF.compute_feature_enabled.console_output: | 
|  | LOG.debug('Console output not supported, cannot check if server ' | 
|  | '%s is ready.', server['id']) | 
|  | return | 
|  |  | 
|  | client = client or self.os_primary.servers_client | 
|  |  | 
|  | def system_booted(): | 
|  | console_output = client.get_console_output(server['id'])['output'] | 
|  | for line in console_output.split('\n'): | 
|  | if 'login:' in line.lower(): | 
|  | return True | 
|  | return False | 
|  |  | 
|  | try: | 
|  | utils.wait_until_true(system_booted, timeout=90, sleep=5) | 
|  | except utils.WaitTimeout: | 
|  | LOG.debug("No correct output in console of server %s found. " | 
|  | "Guest operating system status can't be checked.", | 
|  | server['id']) | 
|  |  | 
|  | def check_servers_hostnames(self, servers, timeout=None, log_errors=True, | 
|  | external_port=None): | 
|  | """Compare hostnames of given servers with their names.""" | 
|  | try: | 
|  | for server in servers: | 
|  | kwargs = {} | 
|  | if timeout: | 
|  | kwargs['timeout'] = timeout | 
|  | try: | 
|  | kwargs['port'] = external_port or ( | 
|  | server['port_forwarding_tcp']['external_port']) | 
|  | except KeyError: | 
|  | pass | 
|  | ssh_client = ssh.Client( | 
|  | self.fip['floating_ip_address'], | 
|  | CONF.validation.image_ssh_user, | 
|  | pkey=self.keypair['private_key'], | 
|  | **kwargs) | 
|  | self.assertIn(server['name'], | 
|  | ssh_client.get_hostname()) | 
|  | except SSH_EXC_TUPLE as ssh_e: | 
|  | LOG.debug(ssh_e) | 
|  | if log_errors: | 
|  | self._log_console_output(servers) | 
|  | self._log_local_network_status() | 
|  | raise | 
|  | except AssertionError as assert_e: | 
|  | LOG.debug(assert_e) | 
|  | if log_errors: | 
|  | self._log_console_output(servers) | 
|  | self._log_local_network_status() | 
|  | raise | 
|  |  | 
|  | def ensure_nc_listen(self, ssh_client, port, protocol, echo_msg=None, | 
|  | servers=None): | 
|  | """Ensure that nc server listening on the given TCP/UDP port is up. | 
|  |  | 
|  | Listener is created always on remote host. | 
|  | """ | 
|  | def spawn_and_check_process(): | 
|  | self.nc_listen(ssh_client, port, protocol, echo_msg, servers) | 
|  | return utils.process_is_running(ssh_client, "nc") | 
|  |  | 
|  | utils.wait_until_true(spawn_and_check_process) | 
|  |  | 
|  | def nc_listen(self, ssh_client, port, protocol, echo_msg=None, | 
|  | servers=None): | 
|  | """Create nc server listening on the given TCP/UDP port. | 
|  |  | 
|  | Listener is created always on remote host. | 
|  | """ | 
|  | try: | 
|  | return ssh_client.execute_script( | 
|  | get_ncat_server_cmd(port, protocol, echo_msg), | 
|  | become_root=True, combine_stderr=True) | 
|  | except SSH_EXC_TUPLE as ssh_e: | 
|  | LOG.debug(ssh_e) | 
|  | self._log_console_output(servers) | 
|  | self._log_local_network_status() | 
|  | raise | 
|  |  | 
|  | def nc_client(self, ip_address, port, protocol, ssh_client=None): | 
|  | """Check connectivity to TCP/UDP port at host via nc. | 
|  |  | 
|  | If ssh_client is not given, it is executed locally on host where tests | 
|  | are executed. Otherwise ssh_client object is used to execute it. | 
|  | """ | 
|  | cmd = get_ncat_client_cmd(ip_address, port, protocol, | 
|  | ssh_client=ssh_client) | 
|  | result = shell.execute(cmd, ssh_client=ssh_client, check=False) | 
|  | return result.stdout | 
|  |  | 
|  | def _ensure_public_router(self, client=None, tenant_id=None): | 
|  | """Retrieve a router for the given tenant id. | 
|  |  | 
|  | If a public router has been configured, it will be returned. | 
|  |  | 
|  | If a public router has not been configured, but a public | 
|  | network has, a tenant router will be created and returned that | 
|  | routes traffic to the public network. | 
|  | """ | 
|  | if not client: | 
|  | client = self.client | 
|  | if not tenant_id: | 
|  | tenant_id = client.project_id | 
|  | router_id = CONF.network.public_router_id | 
|  | network_id = CONF.network.public_network_id | 
|  | if router_id: | 
|  | body = client.show_router(router_id) | 
|  | return body['router'] | 
|  | elif network_id: | 
|  | router = self.create_router_by_client() | 
|  | self.addCleanup(test_utils.call_and_ignore_notfound_exc, | 
|  | client.delete_router, router['id']) | 
|  | kwargs = {'external_gateway_info': dict(network_id=network_id)} | 
|  | router = client.update_router(router['id'], **kwargs)['router'] | 
|  | return router | 
|  | else: | 
|  | raise Exception("Neither of 'public_router_id' or " | 
|  | "'public_network_id' has been defined.") | 
|  |  | 
|  | def _update_router_admin_state(self, router, admin_state_up): | 
|  | kwargs = dict(admin_state_up=admin_state_up) | 
|  | router = self.client.update_router( | 
|  | router['id'], **kwargs)['router'] | 
|  | self.assertEqual(admin_state_up, router['admin_state_up']) | 
|  |  | 
|  | def _check_cmd_installed_on_server(self, ssh_client, server, cmd): | 
|  | try: | 
|  | ssh_client.execute_script('which %s' % cmd) | 
|  | except SSH_EXC_TUPLE as ssh_e: | 
|  | LOG.debug(ssh_e) | 
|  | self._log_console_output([server]) | 
|  | self._log_local_network_status() | 
|  | raise | 
|  | except exceptions.SSHScriptFailed: | 
|  | raise self.skipException( | 
|  | "%s is not available on server %s" % (cmd, server['id'])) | 
|  |  | 
|  |  | 
|  | class BaseAdminTempestTestCase(base_api.BaseAdminNetworkTest, | 
|  | BaseTempestTestCase): | 
|  | pass |