Merge "Fix the extension list across jobs"
diff --git a/.zuul.yaml b/.zuul.yaml
index 9424684..e226c0a 100644
--- a/.zuul.yaml
+++ b/.zuul.yaml
@@ -98,6 +98,7 @@
 
 - job:
     name: neutron-tempest-plugin-api-queens
+    nodeset: openstack-single-node-xenial
     parent: neutron-tempest-plugin-api
     override-checkout: stable/queens
     vars:
@@ -111,6 +112,7 @@
 
 - job:
     name: neutron-tempest-plugin-api-rocky
+    nodeset: openstack-single-node-xenial
     parent: neutron-tempest-plugin-api
     override-checkout: stable/rocky
     vars:
@@ -172,6 +174,7 @@
 - job:
     name: neutron-tempest-plugin-scenario-linuxbridge-queens
     parent: neutron-tempest-plugin-scenario-linuxbridge
+    nodeset: openstack-single-node-xenial
     override-checkout: stable/queens
     vars:
       branch_override: stable/queens
@@ -190,6 +193,7 @@
 - job:
     name: neutron-tempest-plugin-scenario-linuxbridge-rocky
     parent: neutron-tempest-plugin-scenario-linuxbridge
+    nodeset: openstack-single-node-xenial
     override-checkout: stable/rocky
     vars:
       branch_override: stable/rocky
@@ -334,6 +338,7 @@
 - job:
     name: neutron-tempest-plugin-dvr-multinode-scenario-queens
     parent: neutron-tempest-plugin-dvr-multinode-scenario
+    nodeset: openstack-two-node-xenial
     override-checkout: stable/queens
     vars:
       branch_override: stable/queens
@@ -343,6 +348,7 @@
 - job:
     name: neutron-tempest-plugin-dvr-multinode-scenario-rocky
     parent: neutron-tempest-plugin-dvr-multinode-scenario
+    nodeset: openstack-two-node-xenial
     override-checkout: stable/rocky
     vars:
       branch_override: stable/rocky
@@ -378,6 +384,7 @@
 - job:
     name: neutron-tempest-plugin-designate-scenario-queens
     parent: neutron-tempest-plugin-designate-scenario
+    nodeset: openstack-single-node-xenial
     override-checkout: stable/queens
     vars:
       branch_override: stable/queens
@@ -387,6 +394,7 @@
 - job:
     name: neutron-tempest-plugin-designate-scenario-rocky
     parent: neutron-tempest-plugin-designate-scenario
+    nodeset: openstack-single-node-xenial
     override-checkout: stable/rocky
     vars:
       branch_override: stable/rocky
diff --git a/neutron_tempest_plugin/api/admin/test_shared_network_extension.py b/neutron_tempest_plugin/api/admin/test_shared_network_extension.py
index cef0ffc..eb902b9 100644
--- a/neutron_tempest_plugin/api/admin/test_shared_network_extension.py
+++ b/neutron_tempest_plugin/api/admin/test_shared_network_extension.py
@@ -101,8 +101,7 @@
     @decorators.idempotent_id('9c31fabb-0181-464f-9ace-95144fe9ca77')
     def test_create_port_shared_network_as_non_admin_tenant(self):
         # create a port as non admin
-        body = self.client.create_port(network_id=self.shared_network['id'])
-        port = body['port']
+        port = self.create_port(self.shared_network)
         self.addCleanup(self.admin_client.delete_port, port['id'])
         # verify the tenant id of admin network and non admin port
         self.assertNotEqual(self.shared_network['tenant_id'],
@@ -257,7 +256,7 @@
     def test_port_presence_prevents_network_rbac_policy_deletion(self):
         res = self._make_admin_net_and_subnet_shared_to_tenant_id(
             self.client.tenant_id)
-        port = self.client.create_port(network_id=res['network']['id'])['port']
+        port = self.create_port(res['network'])
         # a port on the network should prevent the deletion of a policy
         # required for it to exist
         with testtools.ExpectedException(lib_exc.Conflict):
@@ -274,7 +273,7 @@
                          object_type='network', object_id=net['id'],
                          action='access_as_shared',
                          target_tenant=net['tenant_id'])['rbac_policy']
-        port = self.client.create_port(network_id=net['id'])['port']
+        port = self.create_port(net)
         self.client.delete_rbac_policy(self_share['id'])
         self.client.delete_port(port['id'])
 
@@ -290,8 +289,7 @@
     @decorators.idempotent_id('f7539232-389a-4e9c-9e37-e42a129eb541')
     def test_tenant_cant_delete_other_tenants_ports(self):
         net = self.create_network()
-        port = self.client.create_port(network_id=net['id'])['port']
-        self.addCleanup(self.client.delete_port, port['id'])
+        port = self.create_port(net)
         with testtools.ExpectedException(lib_exc.NotFound):
             self.client2.delete_port(port['id'])
 
@@ -405,7 +403,7 @@
                          object_type='network', object_id=net['id'],
                          action='access_as_shared',
                          target_tenant=net['tenant_id'])['rbac_policy']
-        port = self.client.create_port(network_id=net['id'])['port']
+        port = self.create_port(net)
         self.client.update_rbac_policy(self_share['id'],
                                        target_tenant=self.client2.tenant_id)
         self.client.delete_port(port['id'])
diff --git a/neutron_tempest_plugin/api/test_allowed_address_pair.py b/neutron_tempest_plugin/api/test_allowed_address_pair.py
index 0137ff2..dd48382 100644
--- a/neutron_tempest_plugin/api/test_allowed_address_pair.py
+++ b/neutron_tempest_plugin/api/test_allowed_address_pair.py
@@ -53,11 +53,10 @@
         # Create port with allowed address pair attribute
         allowed_address_pairs = [{'ip_address': self.ip_address,
                                   'mac_address': self.mac_address}]
-        body = self.client.create_port(
-            network_id=self.network['id'],
+        body = self.create_port(
+            self.network,
             allowed_address_pairs=allowed_address_pairs)
-        port_id = body['port']['id']
-        self.addCleanup(self.client.delete_port, port_id)
+        port_id = body['id']
 
         # Confirm port was created with allowed address pair attribute
         body = self.client.list_ports()
@@ -69,9 +68,8 @@
 
     def _update_port_with_address(self, address, mac_address=None, **kwargs):
         # Create a port without allowed address pair
-        body = self.client.create_port(network_id=self.network['id'])
-        port_id = body['port']['id']
-        self.addCleanup(self.client.delete_port, port_id)
+        body = self.create_port(self.network)
+        port_id = body['id']
         if mac_address is None:
             mac_address = self.mac_address
 
@@ -99,11 +97,9 @@
     @decorators.idempotent_id('b3f20091-6cd5-472b-8487-3516137df933')
     def test_update_port_with_multiple_ip_mac_address_pair(self):
         # Create an ip _address and mac_address through port create
-        resp = self.client.create_port(network_id=self.network['id'])
-        newportid = resp['port']['id']
-        self.addCleanup(self.client.delete_port, newportid)
-        ipaddress = resp['port']['fixed_ips'][0]['ip_address']
-        macaddress = resp['port']['mac_address']
+        resp = self.create_port(self.network)
+        ipaddress = resp['fixed_ips'][0]['ip_address']
+        macaddress = resp['mac_address']
 
         # Update allowed address pair port with multiple ip and  mac
         allowed_address_pairs = {'ip_address': ipaddress,
diff --git a/neutron_tempest_plugin/api/test_extra_dhcp_options.py b/neutron_tempest_plugin/api/test_extra_dhcp_options.py
index cb4dba8..844666a 100644
--- a/neutron_tempest_plugin/api/test_extra_dhcp_options.py
+++ b/neutron_tempest_plugin/api/test_extra_dhcp_options.py
@@ -56,11 +56,10 @@
     @decorators.idempotent_id('d2c17063-3767-4a24-be4f-a23dbfa133c9')
     def test_create_list_port_with_extra_dhcp_options(self):
         # Create a port with Extra DHCP Options
-        body = self.client.create_port(
-            network_id=self.network['id'],
+        body = self.create_port(
+            self.network,
             extra_dhcp_opts=self.extra_dhcp_opts)
-        port_id = body['port']['id']
-        self.addCleanup(self.client.delete_port, port_id)
+        port_id = body['id']
 
         # Confirm port created has Extra DHCP Options
         body = self.client.list_ports()
diff --git a/neutron_tempest_plugin/api/test_networks_negative.py b/neutron_tempest_plugin/api/test_networks_negative.py
index 93f32f7..1cc8b93 100644
--- a/neutron_tempest_plugin/api/test_networks_negative.py
+++ b/neutron_tempest_plugin/api/test_networks_negative.py
@@ -28,8 +28,7 @@
     @decorators.attr(type='negative')
     @decorators.idempotent_id('9f80f25b-5d1b-4f26-9f6b-774b9b270819')
     def test_delete_network_in_use(self):
-        port = self.client.create_port(network_id=self.network['id'])
-        self.addCleanup(self.client.delete_port, port['port']['id'])
+        self.create_port(self.network)
         with testtools.ExpectedException(lib_exc.Conflict):
             self.client.delete_subnet(self.subnet['id'])
         with testtools.ExpectedException(lib_exc.Conflict):
diff --git a/neutron_tempest_plugin/api/test_routers_negative.py b/neutron_tempest_plugin/api/test_routers_negative.py
index 2f4ad44..bbd6c5d 100644
--- a/neutron_tempest_plugin/api/test_routers_negative.py
+++ b/neutron_tempest_plugin/api/test_routers_negative.py
@@ -39,9 +39,9 @@
     @decorators.idempotent_id('e3e751af-15a2-49cc-b214-a7154579e94f')
     def test_delete_router_in_use(self):
         # This port is deleted after a test by remove_router_interface.
-        port = self.client.create_port(network_id=self.network['id'])
+        port = self.create_port(self.network)
         self.client.add_router_interface_with_port_id(
-            self.router['id'], port['port']['id'])
+            self.router['id'], port['id'])
         with testtools.ExpectedException(lib_exc.Conflict):
             self.client.delete_router(self.router['id'])
 
diff --git a/neutron_tempest_plugin/common/ip.py b/neutron_tempest_plugin/common/ip.py
new file mode 100644
index 0000000..1702bd3
--- /dev/null
+++ b/neutron_tempest_plugin/common/ip.py
@@ -0,0 +1,316 @@
+# Copyright (c) 2018 Red Hat, Inc.
+#
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+import collections
+import subprocess
+
+import netaddr
+from neutron_lib import constants
+from oslo_log import log
+from oslo_utils import excutils
+
+from neutron_tempest_plugin.common import shell
+
+
+LOG = log.getLogger(__name__)
+
+
+class IPCommand(object):
+
+    sudo = 'sudo'
+    ip_path = '/sbin/ip'
+
+    def __init__(self, ssh_client=None, timeout=None):
+        self.ssh_client = ssh_client
+        self.timeout = timeout
+
+    def get_command(self, obj, *command):
+        command_line = '{sudo!s} {ip_path!r} {object!s} {command!s}'.format(
+            sudo=self.sudo, ip_path=self.ip_path, object=obj,
+            command=subprocess.list2cmdline([str(c) for c in command]))
+        return command_line
+
+    def execute(self, obj, *command):
+        command_line = self.get_command(obj, *command)
+        return shell.execute(command_line, ssh_client=self.ssh_client,
+                             timeout=self.timeout).stdout
+
+    def configure_vlan_subport(self, port, subport, vlan_tag, subnets):
+        addresses = self.list_addresses()
+        try:
+            subport_device = get_port_device_name(addresses=addresses,
+                                                  port=subport)
+        except ValueError:
+            pass
+        else:
+            LOG.debug('Interface %r already configured.', subport_device)
+            return subport_device
+
+        subport_ips = [
+            "{!s}/{!s}".format(ip, prefix_len)
+            for ip, prefix_len in _get_ip_address_prefix_len_pairs(
+                port=subport, subnets=subnets)]
+        if not subport_ips:
+            raise ValueError(
+                "Unable to get IP address and subnet prefix lengths for "
+                "subport")
+
+        port_device = get_port_device_name(addresses=addresses, port=port)
+        subport_device = '{!s}.{!s}'.format(port_device, vlan_tag)
+        LOG.debug('Configuring VLAN subport interface %r on top of interface '
+                  '%r with IPs: %s', subport_device, port_device,
+                  ', '.join(subport_ips))
+
+        self.add_link(link=port_device, name=subport_device, link_type='vlan',
+                      segmentation_id=vlan_tag)
+        self.set_link(device=subport_device, state='up')
+        for subport_ip in subport_ips:
+            self.add_address(address=subport_ip, device=subport_device)
+        return subport_device
+
+    def list_addresses(self, device=None, ip_addresses=None, port=None,
+                       subnets=None):
+        command = ['list']
+        if device:
+            command += ['dev', device]
+        output = self.execute('address', *command)
+        addresses = list(parse_addresses(output))
+
+        return list_ip_addresses(addresses=addresses,
+                                 ip_addresses=ip_addresses, port=port,
+                                 subnets=subnets)
+
+    def add_link(self, name, link_type, link=None, segmentation_id=None):
+        command = ['add']
+        if link:
+            command += ['link', link]
+        command += ['name', name, 'type', link_type]
+        if id:
+            command += ['id', segmentation_id]
+        return self.execute('link', *command)
+
+    def set_link(self, device, state=None):
+        command = ['set', 'dev', device]
+        if state:
+            command.append(state)
+        return self.execute('link', *command)
+
+    def add_address(self, address, device):
+        # ip addr add 192.168.1.1/24 dev em1
+        return self.execute('address', 'add', address, 'dev', device)
+
+    def list_routes(self, *args):
+        output = self.execute('route', 'show', *args)
+        return list(parse_routes(output))
+
+
+def parse_addresses(command_output):
+    address = device = None
+    addresses = []
+    for i, line in enumerate(command_output.split('\n')):
+        try:
+            line_number = i + 1
+            fields = line.strip().split()
+            if not fields:
+                continue
+            indent = line.index(fields[0] + ' ')
+            if indent == 0:
+                # example of line
+                # 2: enp0s25: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast state UP qlen 1000  # noqa
+                address = None
+                name = fields[1]
+                if name.endswith(':'):
+                    name = name[:-1]
+                if '@' in name:
+                    name, parent = name.split('@', 1)
+                else:
+                    parent = None
+
+                if len(fields) > 2:
+                    # flags example: <LOOPBACK,UP,LOWER_UP>
+                    flags = fields[2]
+                    if flags.startswith('<'):
+                        flags = flags[1:]
+                    if flags.startswith('>'):
+                        flags = flags[:-1]
+                    flags = flags.split(',')
+
+                device = Device(name=name, parent=parent, flags=flags,
+                                properties=dict(parse_properties(fields[3:])))
+                LOG.debug("Device parsed: %r", device)
+
+            elif indent == 4:
+                address = Address.create(
+                    family=fields[0], address=fields[1], device=device,
+                    properties=dict(parse_properties(fields[2:])))
+                addresses.append(address)
+                LOG.debug("Address parsed: %r", address)
+
+            elif indent == 7:
+                address.properties.update(parse_properties(fields))
+                LOG.debug("Address properties parsed: %r", address.properties)
+
+            else:
+                assert False, "Invalid line indentation: {!r}".format(indent)
+
+        except Exception:
+            with excutils.save_and_reraise_exception():
+                LOG.exception("Error parsing ip command output at line %d:\n"
+                              "%r\n",
+                              line_number, line)
+            raise
+
+    return addresses
+
+
+def parse_properties(fields):
+    for i, field in enumerate(fields):
+        if i % 2 == 0:
+            key = field
+        else:
+            yield key, field
+
+
+class HasProperties(object):
+
+    def __getattr__(self, name):
+        try:
+            return self.properties[name]
+        except KeyError:
+            pass
+        # This should raise AttributeError
+        return getattr(super(HasProperties, self), name)
+
+
+class Address(HasProperties,
+              collections.namedtuple('Address',
+                                     ['family', 'address', 'device',
+                                      'properties'])):
+
+    _subclasses = {}
+
+    @classmethod
+    def create(cls, family, address, device, properties):
+        cls = cls._subclasses.get(family, cls)
+        return cls(family=family, address=address, device=device,
+                   properties=properties)
+
+    @classmethod
+    def register_subclass(cls, family, subclass=None):
+        if not issubclass(subclass, cls):
+            msg = "{!r} is not sub-class of {!r}".format(cls, Address)
+            raise TypeError(msg)
+        cls._subclasses[family] = subclass
+
+
+class Device(HasProperties,
+             collections.namedtuple('Device',
+                                    ['name', 'parent', 'flags',
+                                     'properties'])):
+    pass
+
+
+def register_address_subclass(families):
+
+    def decorator(subclass):
+        for family in families:
+            Address.register_subclass(family=family, subclass=subclass)
+        return subclass
+
+    return decorator
+
+
+@register_address_subclass(['inet', 'inet6'])
+class InetAddress(Address):
+
+    @property
+    def ip(self):
+        return self.network.ip
+
+    @property
+    def network(self):
+        return netaddr.IPNetwork(self.address)
+
+
+def parse_routes(command_output):
+    for line in command_output.split('\n'):
+        fields = line.strip().split()
+        if fields:
+            dest = fields[0]
+            properties = dict(parse_properties(fields[1:]))
+            if dest == 'default':
+                dest = constants.IPv4_ANY
+                via = properties.get('via')
+                if via:
+                    dest = constants.IP_ANY[netaddr.IPAddress(via).version]
+            yield Route(dest=dest, properties=properties)
+
+
+def list_ip_addresses(addresses, ip_addresses=None, port=None,
+                      subnets=None):
+    if port:
+        # filter addresses by port IP addresses
+        ip_addresses = set(ip_addresses) if ip_addresses else set()
+        ip_addresses.update(list_port_ip_addresses(port=port,
+                                                   subnets=subnets))
+    if ip_addresses:
+        addresses = [a for a in addresses if (hasattr(a, 'ip') and
+                                              str(a.ip) in ip_addresses)]
+    return addresses
+
+
+def list_port_ip_addresses(port, subnets=None):
+    fixed_ips = port['fixed_ips']
+    if subnets:
+        subnets = {subnet['id']: subnet for subnet in subnets}
+        fixed_ips = [fixed_ip
+                     for fixed_ip in fixed_ips
+                     if fixed_ip['subnet_id'] in subnets]
+    return [ip['ip_address'] for ip in port['fixed_ips']]
+
+
+def get_port_device_name(addresses, port):
+    for address in list_ip_addresses(addresses=addresses, port=port):
+        return address.device.name
+
+    msg = "Port %r fixed IPs not found on server.".format(port['id'])
+    raise ValueError(msg)
+
+
+def _get_ip_address_prefix_len_pairs(port, subnets):
+    subnets = {subnet['id']: subnet for subnet in subnets}
+    for fixed_ip in port['fixed_ips']:
+        subnet = subnets.get(fixed_ip['subnet_id'])
+        if subnet:
+            yield (fixed_ip['ip_address'],
+                   netaddr.IPNetwork(subnet['cidr']).prefixlen)
+
+
+class Route(HasProperties,
+            collections.namedtuple('Route',
+                                   ['dest', 'properties'])):
+
+    @property
+    def dest_ip(self):
+        return netaddr.IPNetwork(self.dest)
+
+    @property
+    def via_ip(self):
+        return netaddr.IPAddress(self.via)
+
+    @property
+    def src_ip(self):
+        return netaddr.IPAddress(self.src)
diff --git a/neutron_tempest_plugin/services/network/json/network_client.py b/neutron_tempest_plugin/services/network/json/network_client.py
index d590c25..25fc8c1 100644
--- a/neutron_tempest_plugin/services/network/json/network_client.py
+++ b/neutron_tempest_plugin/services/network/json/network_client.py
@@ -42,20 +42,17 @@
         # The following list represents resource names that do not require
         # changing underscore to a hyphen
         hyphen_exceptions = ["service_profiles", "availability_zones"]
-        # the following map is used to construct proper URI
-        # for the given neutron resource
+        # The following map is used to construct proper URI
+        # for the given neutron resource.
+        # No need to populate this map if the neutron resource
+        # doesn't have a URI prefix.
         service_resource_prefix_map = {
-            'networks': '',
-            'subnets': '',
-            'subnetpools': '',
-            'ports': '',
             'metering_labels': 'metering',
             'metering_label_rules': 'metering',
             'policies': 'qos',
             'bandwidth_limit_rules': 'qos',
             'minimum_bandwidth_rules': 'qos',
             'rule_types': 'qos',
-            'rbac-policies': '',
             'logs': 'log',
             'loggable_resources': 'log',
         }