Merge "Remove "dhcp_agent_scheduler" API extension from common list"
diff --git a/.gitignore b/.gitignore
index a678dd2..228b0a8 100644
--- a/.gitignore
+++ b/.gitignore
@@ -26,7 +26,6 @@
 .coverage*
 !.coveragerc
 .tox
-nosetests.xml
 .venv
 .stestr/
 
diff --git a/neutron_tempest_plugin/api/test_ndp_proxy.py b/neutron_tempest_plugin/api/test_ndp_proxy.py
index 1b2165b..92d65c2 100644
--- a/neutron_tempest_plugin/api/test_ndp_proxy.py
+++ b/neutron_tempest_plugin/api/test_ndp_proxy.py
@@ -24,23 +24,35 @@
 CONF = config.CONF
 
 
-class NDPProxyTestJSON(base.BaseNetworkTest):
+class NDPProxyTestJSON(base.BaseAdminNetworkTest):
 
     credentials = ['primary', 'admin']
-    required_extensions = ['router', 'l3-ndp-proxy']
+    required_extensions = ['router', 'l3-ndp-proxy', 'address-scope']
 
     @classmethod
     def resource_setup(cls):
         super(NDPProxyTestJSON, cls).resource_setup()
-        cls.ext_net_id = CONF.network.public_network_id
-
+        address_scope = cls.create_address_scope(
+            "test-as", **{'ip_version': constants.IP_VERSION_6})
+        subnetpool = cls.create_subnetpool(
+            "test-subnetpool",
+            **{'address_scope_id': address_scope['id'],
+               'default_prefixlen': 112,
+               'prefixes': ['2001:abc::0/96']})
+        # Create an external network and it's subnet
+        ext_net = cls.create_network('test-ext-net', client=cls.admin_client,
+                                     external=True)
+        cls.create_subnet(
+            ext_net, client=cls.admin_client,
+            ip_version=constants.IP_VERSION_6,
+            **{'subnetpool_id': subnetpool['id'], "cidr": "2001:abc::1:0/112"})
         # Create network, subnet, router and add interface
         cls.network = cls.create_network()
         cls.subnet = cls.create_subnet(
             cls.network, ip_version=constants.IP_VERSION_6,
-            cidr='2002::abcd:0/112')
+            **{'subnetpool_id': subnetpool['id'], "cidr": "2001:abc::2:0/112"})
         cls.router = cls.create_router(data_utils.rand_name('router'),
-                                       external_network_id=cls.ext_net_id,
+                                       external_network_id=ext_net['id'],
                                        enable_ndp_proxy=True)
         cls.create_router_interface(cls.router['id'], cls.subnet['id'])
 
diff --git a/neutron_tempest_plugin/api/test_ndp_proxy_negative.py b/neutron_tempest_plugin/api/test_ndp_proxy_negative.py
index acbbdcb..56188f3 100644
--- a/neutron_tempest_plugin/api/test_ndp_proxy_negative.py
+++ b/neutron_tempest_plugin/api/test_ndp_proxy_negative.py
@@ -24,16 +24,28 @@
 CONF = config.CONF
 
 
-class NDPProxyNegativeTestJSON(base.BaseNetworkTest):
+class NDPProxyNegativeTestJSON(base.BaseAdminNetworkTest):
 
     credentials = ['primary', 'admin']
-    required_extensions = ['router', 'l3-ndp-proxy']
+    required_extensions = ['router', 'l3-ndp-proxy', 'address-scope']
 
     @classmethod
     def resource_setup(cls):
         super(NDPProxyNegativeTestJSON, cls).resource_setup()
-        cls.ext_net_id = CONF.network.public_network_id
-
+        address_scope = cls.create_address_scope(
+            "test-as", **{'ip_version': constants.IP_VERSION_6})
+        subnetpool = cls.create_subnetpool(
+            "test-subnetpool",
+            **{'address_scope_id': address_scope['id'],
+               'default_prefixlen': 112,
+               'prefixes': ['2001:abc::0/96']})
+        # Create an external network and it's subnet
+        ext_net = cls.create_network('test-ext-net', client=cls.admin_client,
+                                     external=True)
+        cls.create_subnet(
+            ext_net, client=cls.admin_client,
+            ip_version=constants.IP_VERSION_6,
+            **{'subnetpool_id': subnetpool['id'], "cidr": "2001:abc::1:0/112"})
         # Create network, subnet, router and add interface
         cls.network = cls.create_network()
         cls.subnet = cls.create_subnet(
@@ -41,7 +53,7 @@
             cidr='2002::abcd:0/112')
         cls.router = cls.create_router(
             data_utils.rand_name('router'),
-            external_network_id=cls.ext_net_id)
+            external_network_id=ext_net['id'])
 
     @decorators.attr(type='negative')
     @decorators.idempotent_id('a0897204-bb85-41cc-a5fd-5d0ab8116a07')
@@ -102,3 +114,22 @@
                           self.router['id'],
                           enable_ndp_proxy=True,
                           external_gateway_info={})
+
+    @decorators.attr(type='negative')
+    @decorators.idempotent_id('194b5ee7-4c59-4643-aabf-80a125c3f688')
+    def test_enable_ndp_proxy_without_address_scope(self):
+        extnet = self.create_network("extnet", client=self.admin_client,
+                                     external=True)
+        self.create_subnet(extnet, client=self.admin_client,
+                           ip_version=constants.IP_VERSION_6,
+                           cidr='2001:abc1::0/112')
+        self.assertRaises(exceptions.Conflict,
+                          self.client.create_router,
+                          name=data_utils.rand_name('router'),
+                          enable_ndp_proxy=True,
+                          external_gateway_info={'network_id': extnet['id']})
+        router = self.create_router(data_utils.rand_name('router'))
+        self.assertRaises(exceptions.Conflict,
+                          self.client.update_router,
+                          router['id'], enable_ndp_proxy=True,
+                          external_gateway_info={'network_id': extnet['id']})
diff --git a/neutron_tempest_plugin/api/test_port_forwardings.py b/neutron_tempest_plugin/api/test_port_forwardings.py
index 79cce39..3abcd17 100644
--- a/neutron_tempest_plugin/api/test_port_forwardings.py
+++ b/neutron_tempest_plugin/api/test_port_forwardings.py
@@ -128,12 +128,17 @@
         # Check that all PFs are visible in Floating IP details
         fip = self.client.show_floatingip(fip_id)['floatingip']
         self.assertEqual(len(created_pfs), len(fip['port_forwardings']))
+        detail_enabled = utils.is_extension_enabled(
+            'floating-ip-port-forwarding-detail', service='network')
         for pf in created_pfs:
             expected_pf = {
                 'external_port': pf['external_port'],
                 'internal_port': pf['internal_port'],
                 'protocol': pf['protocol'],
                 'internal_ip_address': pf['internal_ip_address']}
+            if detail_enabled:
+                expected_pf['id'] = pf['id']
+                expected_pf['internal_port_id'] = pf['internal_port_id']
             self.assertIn(expected_pf, fip['port_forwardings'])
 
         # Test list of port forwardings
diff --git a/neutron_tempest_plugin/api/test_qos.py b/neutron_tempest_plugin/api/test_qos.py
index 2929542..372bf1e 100644
--- a/neutron_tempest_plugin/api/test_qos.py
+++ b/neutron_tempest_plugin/api/test_qos.py
@@ -1154,10 +1154,10 @@
            2-6, 42, 44, and 50-54 (which should be forbidden)
         """
 
-        def _test_update_dscp_mark_values(self, dscp_policy_id, rule_id):
-            for mark in range(n_constants.VALID_DSCP_MARKS[1],
-                              self.VALID_DSCP_MARK1 + 1):
-                if mark in n_constants.VALID_DSCP_MARKS:
+        def _test_update_dscp_mark_values(self, dscp_policy_id, rule_id,
+                                          valid_dscp_marks):
+            for mark in range(valid_dscp_marks[1], self.VALID_DSCP_MARK1 + 1):
+                if mark in valid_dscp_marks:
                     self.admin_client.update_dscp_marking_rule(
                         dscp_policy_id, rule_id, dscp_mark=mark)
 
@@ -1174,6 +1174,20 @@
                                     self.admin_client.create_dscp_marking_rule,
                                     dscp_policy_id,
                                     mark)
+
+        # Retrieve the valid range of DSCP marks from the API.
+        rule_type_details = self.admin_client.show_qos_rule_type(
+            qos_consts.RULE_TYPE_DSCP_MARKING).get('rule_type')
+        # There should be at least one driver supporting the DSCP marking rule.
+        dscp_driver = rule_type_details['drivers'][0]
+        for parameter in dscp_driver['supported_parameters']:
+            if parameter['parameter_name'] == qos_consts.DSCP_MARK:
+                valid_dscp_marks = parameter['parameter_values']
+                break
+        else:
+            self.fail('The DSCP marking rule does not have the %s parameter' %
+                      qos_consts.DSCP_MARK)
+
         # Setup network
         self.network = self.create_network()
 
@@ -1204,7 +1218,8 @@
                          retrieved_rule['dscp_mark']))
 
         # Try to set marks in range 8:56 (invalid marks should raise an error)
-        _test_update_dscp_mark_values(self, dscp_policy_id, rule_id)
+        _test_update_dscp_mark_values(self, dscp_policy_id, rule_id,
+                                      valid_dscp_marks)
 
 
 class QosMinimumBandwidthRuleTestJSON(base.BaseAdminNetworkTest):
diff --git a/neutron_tempest_plugin/common/utils.py b/neutron_tempest_plugin/common/utils.py
index c28ce74..ab99db9 100644
--- a/neutron_tempest_plugin/common/utils.py
+++ b/neutron_tempest_plugin/common/utils.py
@@ -27,6 +27,7 @@
 
 import eventlet
 
+from oslo_log import log
 from tempest.lib import exceptions
 
 from neutron_tempest_plugin import config
@@ -37,6 +38,7 @@
     "https": 443,
 }
 CONF = config.CONF
+LOG = log.getLogger(__name__)
 
 
 class classproperty(object):
@@ -202,20 +204,30 @@
                         timeout=self.test_timeout,
                         sleep=self.test_sleep)
         try:
+            LOG.info("Checking connectivity between server and client -"
+                    " attempt {}".format(self.test_attempt))
             self.server_ssh.exec_command(
                     'grep {} output.txt'.format(self.test_str))
             self.client_ssh.exec_command(
                     'grep {} output.txt'.format(self.test_str))
             if not self.should_pass:
+                LOG.warning("attempt {} succeed while it should fail".format(
+                    self.test_attempt))
                 return False
             else:
                 if not self.connection_started:
                     self.connection_started = True
+                LOG.info("attempt {} succeed as it expected".format(
+                    self.test_attempt))
                 return True
         except exceptions.SSHExecCommandFailed:
             if self.should_pass:
+                LOG.warning("attempt {} failed while it should pass".format(
+                    self.test_attempt))
                 return False
             else:
+                LOG.info("attempt {} failed as it expected".format(
+                    self.test_attempt))
                 return True
         finally:
             self.test_attempt += 1
diff --git a/neutron_tempest_plugin/scenario/test_security_groups.py b/neutron_tempest_plugin/scenario/test_security_groups.py
index 05fbfe8..5af84db 100644
--- a/neutron_tempest_plugin/scenario/test_security_groups.py
+++ b/neutron_tempest_plugin/scenario/test_security_groups.py
@@ -520,12 +520,9 @@
 
         # verify that conections are not working
         for port in range(80, 84):
-            self._verify_http_connection(
-                ssh_clients[0],
-                ssh_clients[2],
-                test_ip, port,
-                servers,
-                should_pass=False)
+            with utils.StatefulConnection(
+                    ssh_clients[0], ssh_clients[2], test_ip, port) as con:
+                con.test_connection(should_pass=False)
 
         # add two remote-group rules with port-ranges
         rule_list = [{'protocol': constants.PROTO_NUM_TCP,
@@ -543,11 +540,9 @@
 
         # verify that conections are working
         for port in range(80, 84):
-            self._verify_http_connection(
-                ssh_clients[0],
-                ssh_clients[2],
-                test_ip, port,
-                servers)
+            with utils.StatefulConnection(
+                    ssh_clients[0], ssh_clients[2], test_ip, port) as con:
+                con.test_connection()
 
         # list the tcp rule id by SG id and port-range
         sg_rule_id = self.os_primary.network_client.list_security_group_rules(
@@ -559,12 +554,9 @@
 
         # verify that conections are not working
         for port in range(80, 82):
-            self._verify_http_connection(
-                ssh_clients[0],
-                ssh_clients[2],
-                test_ip, port,
-                servers,
-                should_pass=False)
+            with utils.StatefulConnection(
+                    ssh_clients[0], ssh_clients[2], test_ip, port) as con:
+                con.test_connection(should_pass=False)
 
     @decorators.idempotent_id('f07d0159-8f9e-4faa-87f5-a869ab0ad490')
     def test_intra_sg_isolation(self):
@@ -675,11 +667,13 @@
         # status can change the datapath. Let's check the rules in two
         # attempts
         for _ in range(2):
-            self._verify_http_connection(client_ssh[0], srv_ssh, srv_ip,
-                                         tcp_port, [])
+            with utils.StatefulConnection(
+                    client_ssh[0], srv_ssh, srv_ip, tcp_port) as con:
+                con.test_connection()
             for port in range(tcp_port, tcp_port + 3):
-                self._verify_http_connection(client_ssh[1], srv_ssh, srv_ip,
-                                             port, [])
+                with utils.StatefulConnection(
+                        client_ssh[1], srv_ssh, srv_ip, port) as con:
+                    con.test_connection()
 
     @decorators.idempotent_id('96dcd5ff-9d45-4e0d-bea0-0b438cbd388f')
     def test_remove_sec_grp_from_active_vm(self):
diff --git a/requirements.txt b/requirements.txt
index f25caec..34531e9 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -15,7 +15,6 @@
 tempest>=29.2.0 # Apache-2.0
 tenacity>=3.2.1 # Apache-2.0
 ddt>=1.0.1 # MIT
-nose>=1.3.7 # LGPL
 testtools>=2.2.0 # MIT
 testscenarios>=0.4 # Apache-2.0/BSD
 eventlet!=0.18.3,!=0.20.1,>=0.18.2 # MIT
diff --git a/zuul.d/master_jobs.yaml b/zuul.d/master_jobs.yaml
index 8e21880..7cd5079 100644
--- a/zuul.d/master_jobs.yaml
+++ b/zuul.d/master_jobs.yaml
@@ -75,6 +75,7 @@
         - fip-port-details
         - flavors
         - floating-ip-port-forwarding
+        - floating-ip-port-forwarding-detail
         - floatingip-pools
         - ip-substring-filtering
         - l3-conntrack-helper
@@ -300,7 +301,6 @@
       - ^neutron/services/trunk/drivers/ovn/.*$
       - ^neutron/cmd/ovn/.*$
       - ^neutron/common/ovn/.*$
-      - ^neutron_tempest_plugin/api/test_.*$
       - ^neutron_tempest_plugin/(bgpvpn|fwaas|neutron_dynamic_routing|sfc|tap_as_a_service|vpnaas).*$
       - ^neutron_tempest_plugin/services/bgp/.*$
       - ^rally-jobs/.*$
@@ -402,7 +402,6 @@
       - ^neutron/services/trunk/drivers/ovn/.*$
       - ^neutron/cmd/ovn/.*$
       - ^neutron/common/ovn/.*$
-      - ^neutron_tempest_plugin/api/test_.*$
       - ^neutron_tempest_plugin/(bgpvpn|fwaas|neutron_dynamic_routing|sfc|tap_as_a_service|vpnaas).*$
       - ^neutron_tempest_plugin/services/bgp/.*$
       - ^rally-jobs/.*$
@@ -558,7 +557,6 @@
       - ^neutron/services/trunk/drivers/ovn/.*$
       - ^neutron/cmd/ovn/.*$
       - ^neutron/common/ovn/.*$
-      - ^neutron_tempest_plugin/api/test_.*$
       - ^neutron_tempest_plugin/(bgpvpn|fwaas|neutron_dynamic_routing|sfc|tap_as_a_service|vpnaas).*$
       - ^neutron_tempest_plugin/services/bgp/.*$
       - ^rally-jobs/.*$
@@ -695,7 +693,6 @@
       - ^neutron/services/trunk/drivers/linuxbridge/.*$
       - ^neutron/services/trunk/drivers/openvswitch/.*$
       - ^neutron/scheduler/.*$
-      - ^neutron_tempest_plugin/api/test_.*$
       - ^neutron_tempest_plugin/(bgpvpn|fwaas|neutron_dynamic_routing|sfc|tap_as_a_service|vpnaas).*$
       - ^neutron_tempest_plugin/services/bgp/.*$
       - ^rally-jobs/.*$
@@ -1042,6 +1039,7 @@
         q-l3: true
         q-meta: true
         q-metering: true
+      tempest_concurrency: 4
       tempest_test_regex: ^neutron_tempest_plugin\.bgpvpn
       network_api_extensions_bgpvpn:
         - bgpvpn
@@ -1214,6 +1212,7 @@
       - openstack/neutron-tempest-plugin
       - openstack/tempest
     vars:
+      tempest_concurrency: 4
       tempest_test_regex: ^neutron_tempest_plugin\.vpnaas
       devstack_plugins:
         neutron-vpnaas: https://opendev.org/openstack/neutron-vpnaas.git
@@ -1279,6 +1278,7 @@
       - openstack/tap-as-a-service
       - openstack/tempest
     vars:
+      tempest_concurrency: 4
       tempest_test_regex: ^neutron_tempest_plugin\.tap_as_a_service
       tox_envlist: all
       network_api_extensions_tempest:
diff --git a/zuul.d/queens_jobs.yaml b/zuul.d/queens_jobs.yaml
index 35fba1b..3a1c37f 100644
--- a/zuul.d/queens_jobs.yaml
+++ b/zuul.d/queens_jobs.yaml
@@ -25,6 +25,7 @@
         q-meta: true
         q-metering: true
       branch_override: stable/queens
+      tempest_concurrency: 4
       tempest_test_regex: ^neutron_tempest_plugin\.api
       # TODO(slaweq): find a way to put this list of extensions in
       # neutron repository and keep it different per branch,
diff --git a/zuul.d/rocky_jobs.yaml b/zuul.d/rocky_jobs.yaml
index 62e57bf..75d7098 100644
--- a/zuul.d/rocky_jobs.yaml
+++ b/zuul.d/rocky_jobs.yaml
@@ -27,6 +27,7 @@
         q-meta: true
         q-metering: true
       branch_override: stable/rocky
+      tempest_concurrency: 4
       tempest_test_regex: ^neutron_tempest_plugin\.api
       # TODO(slaweq): find a way to put this list of extensions in
       # neutron repository and keep it different per branch,
diff --git a/zuul.d/stein_jobs.yaml b/zuul.d/stein_jobs.yaml
index e9e8137..0f18a8a 100644
--- a/zuul.d/stein_jobs.yaml
+++ b/zuul.d/stein_jobs.yaml
@@ -25,6 +25,7 @@
         q-meta: true
         q-metering: true
       branch_override: stable/stein
+      tempest_concurrency: 4
       tempest_test_regex: ^neutron_tempest_plugin\.api
       # TODO(slaweq): find a way to put this list of extensions in
       # neutron repository and keep it different per branch,
@@ -112,6 +113,8 @@
         Q_ML2_TENANT_NETWORK_TYPE: vxlan
         Q_ML2_PLUGIN_MECHANISM_DRIVERS: openvswitch
         ML2_L3_PLUGIN: router
+        ADVANCED_INSTANCE_TYPE: ds512M
+        ADVANCED_INSTANCE_USER: ubuntu
       devstack_local_conf:
         post-config:
           # NOTE(slaweq): We can get rid of this hardcoded absolute path when
@@ -285,6 +288,8 @@
         q-l3: true
         q-meta: true
         q-metering: true
+        # SG logging isn't supported by linuxbridge backend
+        neutron-log: false
       network_api_extensions: *api_extensions
       network_api_extensions_linuxbridge:
         - vlan-transparent
diff --git a/zuul.d/train_jobs.yaml b/zuul.d/train_jobs.yaml
index 961b3b7..c8c2efe 100644
--- a/zuul.d/train_jobs.yaml
+++ b/zuul.d/train_jobs.yaml
@@ -25,6 +25,7 @@
         q-meta: true
         q-metering: true
       branch_override: stable/train
+      tempest_concurrency: 4
       tempest_test_regex: ^neutron_tempest_plugin\.api
       # TODO(slaweq): find a way to put this list of extensions in
       # neutron repository and keep it different per branch,
diff --git a/zuul.d/ussuri_jobs.yaml b/zuul.d/ussuri_jobs.yaml
index 28fccc6..56da160 100644
--- a/zuul.d/ussuri_jobs.yaml
+++ b/zuul.d/ussuri_jobs.yaml
@@ -25,6 +25,7 @@
         q-meta: true
         q-metering: true
       branch_override: stable/ussuri
+      tempest_concurrency: 4
       tempest_test_regex: ^neutron_tempest_plugin\.api
       # TODO(slaweq): find a way to put this list of extensions in
       # neutron repository and keep it different per branch,
diff --git a/zuul.d/victoria_jobs.yaml b/zuul.d/victoria_jobs.yaml
index 67737a7..7a92403 100644
--- a/zuul.d/victoria_jobs.yaml
+++ b/zuul.d/victoria_jobs.yaml
@@ -24,6 +24,7 @@
         q-meta: true
         q-metering: true
       branch_override: stable/victoria
+      tempest_concurrency: 4
       tempest_test_regex: ^neutron_tempest_plugin\.api
       # TODO(slaweq): find a way to put this list of extensions in
       # neutron repository and keep it different per branch,
diff --git a/zuul.d/wallaby_jobs.yaml b/zuul.d/wallaby_jobs.yaml
index 746ee9c..a3c07c0 100644
--- a/zuul.d/wallaby_jobs.yaml
+++ b/zuul.d/wallaby_jobs.yaml
@@ -3,6 +3,7 @@
     parent: neutron-tempest-plugin-base
     override-checkout: stable/wallaby
     vars:
+      tempest_concurrency: 4
       tempest_test_regex: ^neutron_tempest_plugin\.api
       # TODO(slaweq): find a way to put this list of extensions in
       # neutron repository and keep it different per branch,
diff --git a/zuul.d/xena_jobs.yaml b/zuul.d/xena_jobs.yaml
index 1e6eb80..e0a58a7 100644
--- a/zuul.d/xena_jobs.yaml
+++ b/zuul.d/xena_jobs.yaml
@@ -3,6 +3,7 @@
     parent: neutron-tempest-plugin-base
     override-checkout: stable/xena
     vars:
+      tempest_concurrency: 4
       tempest_test_regex: ^neutron_tempest_plugin\.api
       # TODO(slaweq): find a way to put this list of extensions in
       # neutron repository and keep it different per branch,
diff --git a/zuul.d/yoga_jobs.yaml b/zuul.d/yoga_jobs.yaml
index 87db14e..bc88c83 100644
--- a/zuul.d/yoga_jobs.yaml
+++ b/zuul.d/yoga_jobs.yaml
@@ -3,6 +3,7 @@
     parent: neutron-tempest-plugin-base
     override-checkout: stable/yoga
     vars:
+      tempest_concurrency: 4
       tempest_test_regex: ^neutron_tempest_plugin\.api
       # TODO(slaweq): find a way to put this list of extensions in
       # neutron repository and keep it different per branch,