Merge "Mark test_minimum_basic_instance_hard_reboot_after_vol_snap_deletion as slow test"
diff --git a/releasenotes/notes/add-ssh-allow-agent-2dee6448fd250e50.yaml b/releasenotes/notes/add-ssh-allow-agent-2dee6448fd250e50.yaml
new file mode 100644
index 0000000..33f11ce
--- /dev/null
+++ b/releasenotes/notes/add-ssh-allow-agent-2dee6448fd250e50.yaml
@@ -0,0 +1,10 @@
+---
+features:
+  - |
+    Adds a ``ssh_allow_agent`` parameter to the ``RemoteClient`` class
+    wrapper and the direct ssh ``Client`` class to allow a caller to
+    explicitly request that the SSH Agent is not consulted for
+    authentication. This is useful if your attempting explicit password
+    based authentication as ``paramiko``, the underlying library used for
+    SSH, defaults to utilizing an ssh-agent process before attempting
+    password authentication.
diff --git a/roles/run-tempest-26/tasks/main.yaml b/roles/run-tempest-26/tasks/main.yaml
index f846006..7423bfb 100644
--- a/roles/run-tempest-26/tasks/main.yaml
+++ b/roles/run-tempest-26/tasks/main.yaml
@@ -62,7 +62,9 @@
       when: blacklist_stat.stat.exists
 
 - name: Run Tempest
-  command: tox -e {{tox_envlist}} {{tox_extra_args}} -- {{tempest_test_regex|quote}} {{blacklist_option|default('')}} \
+  command: tox -e {{tox_envlist}} {{tox_extra_args}} -- \
+            {{tempest_test_regex|quote if (tempest_test_regex|length>0)|default(None, True)}} \
+            {{blacklist_option|default(None)}} \
             --concurrency={{tempest_concurrency|default(default_concurrency)}} \
             --black-regex={{tempest_black_regex|quote}}
   args:
diff --git a/roles/run-tempest/tasks/main.yaml b/roles/run-tempest/tasks/main.yaml
index e569e53..3fb494f 100644
--- a/roles/run-tempest/tasks/main.yaml
+++ b/roles/run-tempest/tasks/main.yaml
@@ -120,10 +120,11 @@
     - target_branch in ["stable/train", "stable/ussuri", "stable/victoria"]
 
 - name: Run Tempest
-  command: tox -e {{tox_envlist}} {{tox_extra_args}} -- {{tempest_test_regex|quote}} \
-           {{blacklist_option|default('')}}  {{exclude_list_option|default('')}} \
+  command: tox -e {{tox_envlist}} {{tox_extra_args}} -- \
+           {{tempest_test_regex|quote if (tempest_test_regex|length>0)|default(None, True)}} \
+           {{blacklist_option|default(None)}}  {{exclude_list_option|default(None)}} \
             --concurrency={{tempest_concurrency|default(default_concurrency)}} \
-           {{tempest_test_exclude_regex|default('')}}
+           {{tempest_test_exclude_regex|default(None)}}
   args:
     chdir: "{{devstack_base_dir}}/tempest"
   register: tempest_run_result
diff --git a/tempest/api/compute/admin/test_volume.py b/tempest/api/compute/admin/test_volume.py
index 2fcd053..e7c931e 100644
--- a/tempest/api/compute/admin/test_volume.py
+++ b/tempest/api/compute/admin/test_volume.py
@@ -13,8 +13,6 @@
 #    License for the specific language governing permissions and limitations
 #    under the License.
 
-import io
-
 from tempest.api.compute import base
 from tempest.common import waiters
 from tempest import config
@@ -49,9 +47,11 @@
         :param return image_id: The UUID of the newly created image.
         """
         image = self.admin_image_client.show_image(CONF.compute.image_ref)
-        image_data = self.admin_image_client.show_image_file(
-            CONF.compute.image_ref).data
-        image_file = io.BytesIO(image_data)
+        # NOTE(danms): We need to stream this, so chunked=True means we get
+        # back a urllib3.HTTPResponse and have to carefully pass it to
+        # store_image_file() to upload it in pieces.
+        image_data_resp = self.admin_image_client.show_image_file(
+            CONF.compute.image_ref, chunked=True)
         create_dict = {
             'container_format': image['container_format'],
             'disk_format': image['disk_format'],
@@ -60,24 +60,22 @@
             'visibility': 'public',
         }
         create_dict.update(kwargs)
-        new_image = self.admin_image_client.create_image(**create_dict)
-        self.addCleanup(self.admin_image_client.wait_for_resource_deletion,
-                        new_image['id'])
-        self.addCleanup(self.admin_image_client.delete_image, new_image['id'])
-        self.admin_image_client.store_image_file(new_image['id'], image_file)
-
+        try:
+            new_image = self.admin_image_client.create_image(**create_dict)
+            self.addCleanup(self.admin_image_client.wait_for_resource_deletion,
+                            new_image['id'])
+            self.addCleanup(
+                self.admin_image_client.delete_image, new_image['id'])
+            self.admin_image_client.store_image_file(new_image['id'],
+                                                     image_data_resp)
+        finally:
+            image_data_resp.release_conn()
         return new_image['id']
 
 
 class AttachSCSIVolumeTestJSON(BaseAttachSCSIVolumeTest):
     """Test attaching scsi volume to server"""
 
-    # NOTE(gibi): https://bugs.launchpad.net/nova/+bug/2002951/comments/5 shows
-    # that calling _create_image_with_custom_property can cause excessive
-    # memory usage in the test executor as it downloads a glance image in
-    # memory. This is causing gate failures so the test is disabled. One
-    # potential fix is to do a chunked data download / upload loop instead.
-    @decorators.skip_because(bug="2002951", condition=True)
     @decorators.idempotent_id('777e468f-17ca-4da4-b93d-b7dbf56c0494')
     def test_attach_scsi_disk_with_config_drive(self):
         """Test the attach/detach volume with config drive/scsi disk
diff --git a/tempest/api/image/v2/test_images.py b/tempest/api/image/v2/test_images.py
index d590668..e8734e0 100644
--- a/tempest/api/image/v2/test_images.py
+++ b/tempest/api/image/v2/test_images.py
@@ -14,8 +14,10 @@
 #    License for the specific language governing permissions and limitations
 #    under the License.
 
+import contextlib
 import io
 import random
+import time
 
 from oslo_log import log as logging
 from tempest.api.image import base
@@ -29,6 +31,19 @@
 LOG = logging.getLogger(__name__)
 
 
+@contextlib.contextmanager
+def retry_bad_request(fn):
+    retries = 3
+    for i in range(retries):
+        try:
+            yield
+        except lib_exc.BadRequest:
+            if i < retries:
+                time.sleep(1)
+            else:
+                raise
+
+
 class ImportImagesTest(base.BaseV2ImageTest):
     """Here we test the import operations for image"""
 
@@ -817,8 +832,14 @@
         # Add a new location
         new_loc = {'metadata': {'foo': 'bar'},
                    'url': CONF.image.http_image}
-        self.client.update_image(image['id'], [
-            dict(add='/locations/-', value=new_loc)])
+
+        # NOTE(danms): If glance was unable to fetch the remote image via
+        # HTTP, it will return BadRequest. Because this can be transient in
+        # CI, we try this a few times before we agree that it has failed
+        # for a reason worthy of failing the test.
+        with retry_bad_request():
+            self.client.update_image(image['id'], [
+                dict(add='/locations/-', value=new_loc)])
 
         # The image should now be active, with one location that looks
         # like we expect
@@ -848,8 +869,14 @@
 
         new_loc = {'metadata': {'speed': '88mph'},
                    'url': '%s#new' % CONF.image.http_image}
-        self.client.update_image(image['id'], [
-            dict(add='/locations/-', value=new_loc)])
+
+        # NOTE(danms): If glance was unable to fetch the remote image via
+        # HTTP, it will return BadRequest. Because this can be transient in
+        # CI, we try this a few times before we agree that it has failed
+        # for a reason worthy of failing the test.
+        with retry_bad_request():
+            self.client.update_image(image['id'], [
+                dict(add='/locations/-', value=new_loc)])
 
         # The image should now have two locations and the last one
         # (locations are ordered) should have the new URL.
diff --git a/tempest/lib/common/http.py b/tempest/lib/common/http.py
index 33f871b..d163968 100644
--- a/tempest/lib/common/http.py
+++ b/tempest/lib/common/http.py
@@ -60,7 +60,12 @@
             retry = urllib3.util.Retry(redirect=False)
         r = super(ClosingProxyHttp, self).request(method, url, retries=retry,
                                                   *args, **new_kwargs)
-        return Response(r), r.data
+        if not kwargs.get('preload_content', True):
+            # This means we asked urllib3 for streaming content, so we
+            # need to return the raw response and not read any data yet
+            return r, b''
+        else:
+            return Response(r), r.data
 
 
 class ClosingHttp(urllib3.poolmanager.PoolManager):
@@ -109,4 +114,9 @@
             retry = urllib3.util.Retry(redirect=False)
         r = super(ClosingHttp, self).request(method, url, retries=retry,
                                              *args, **new_kwargs)
-        return Response(r), r.data
+        if not kwargs.get('preload_content', True):
+            # This means we asked urllib3 for streaming content, so we
+            # need to return the raw response and not read any data yet
+            return r, b''
+        else:
+            return Response(r), r.data
diff --git a/tempest/lib/common/rest_client.py b/tempest/lib/common/rest_client.py
index a11b7c1..6cf5b73 100644
--- a/tempest/lib/common/rest_client.py
+++ b/tempest/lib/common/rest_client.py
@@ -19,6 +19,7 @@
 import re
 import time
 import urllib
+import urllib3
 
 import jsonschema
 from oslo_log import log as logging
@@ -298,7 +299,7 @@
         """
         return self.request('POST', url, extra_headers, headers, body, chunked)
 
-    def get(self, url, headers=None, extra_headers=False):
+    def get(self, url, headers=None, extra_headers=False, chunked=False):
         """Send a HTTP GET request using keystone service catalog and auth
 
         :param str url: the relative url to send the get request to
@@ -307,11 +308,19 @@
                                    returned by the get_headers() method are to
                                    be used but additional headers are needed in
                                    the request pass them in as a dict.
+        :param bool chunked: Boolean value that indicates if we should stream
+                             the response instead of reading it all at once.
+                             If True, data will be empty and the raw urllib3
+                             response object will be returned.
+                             NB: If you pass True here, you **MUST** call
+                             release_conn() on the response object before
+                             finishing!
         :return: a tuple with the first entry containing the response headers
                  and the second the response body
         :rtype: tuple
         """
-        return self.request('GET', url, extra_headers, headers)
+        return self.request('GET', url, extra_headers, headers,
+                            chunked=chunked)
 
     def delete(self, url, headers=None, body=None, extra_headers=False):
         """Send a HTTP DELETE request using keystone service catalog and auth
@@ -480,7 +489,7 @@
         self.LOG.info(
             'Request (%s): %s %s %s%s',
             caller_name,
-            resp['status'],
+            resp.status,
             method,
             req_url,
             secs,
@@ -617,17 +626,30 @@
         """
         if headers is None:
             headers = self.get_headers()
+        # In urllib3, chunked only affects the upload. However, we may
+        # want to read large responses to GET incrementally. Re-purpose
+        # chunked=True on a GET to also control how we handle the response.
+        preload = not (method.lower() == 'get' and chunked)
+        if not preload:
+            # NOTE(danms): Not specifically necessary, but don't send
+            # chunked=True to urllib3 on a GET, since it is technically
+            # for PUT/POST type operations
+            chunked = False
         # Do the actual request, and time it
         start = time.time()
         self._log_request_start(method, url)
         resp, resp_body = self.http_obj.request(
             url, method, headers=headers,
-            body=body, chunked=chunked)
+            body=body, chunked=chunked, preload_content=preload)
         end = time.time()
         req_body = body if log_req_body is None else log_req_body
-        self._log_request(method, url, resp, secs=(end - start),
-                          req_headers=headers, req_body=req_body,
-                          resp_body=resp_body)
+        if preload:
+            # NOTE(danms): If we are reading the whole response, we can do
+            # this logging. If not, skip the logging because it will result
+            # in us reading the response data prematurely.
+            self._log_request(method, url, resp, secs=(end - start),
+                              req_headers=headers, req_body=req_body,
+                              resp_body=resp_body)
         return resp, resp_body
 
     def request(self, method, url, extra_headers=False, headers=None,
@@ -773,6 +795,10 @@
         # resp this could possibly fail
         if str(type(resp)) == "<type 'instance'>":
             ctype = resp.getheader('content-type')
+        elif isinstance(resp, urllib3.HTTPResponse):
+            # If we requested chunked=True streaming, this will be a raw
+            # urllib3.HTTPResponse
+            ctype = resp.getheaders()['content-type']
         else:
             try:
                 ctype = resp['content-type']
diff --git a/tempest/lib/common/ssh.py b/tempest/lib/common/ssh.py
index cb59a82..aad04b8 100644
--- a/tempest/lib/common/ssh.py
+++ b/tempest/lib/common/ssh.py
@@ -53,7 +53,8 @@
 
     def __init__(self, host, username, password=None, timeout=300, pkey=None,
                  channel_timeout=10, look_for_keys=False, key_filename=None,
-                 port=22, proxy_client=None, ssh_key_type='rsa'):
+                 port=22, proxy_client=None, ssh_key_type='rsa',
+                 ssh_allow_agent=True):
         """SSH client.
 
         Many of parameters are just passed to the underlying implementation
@@ -76,6 +77,9 @@
             for ssh-over-ssh.  The default is None, which means
             not to use ssh-over-ssh.
         :param ssh_key_type: ssh key type (rsa, ecdsa)
+        :param ssh_allow_agent: boolean, default True, if the SSH client is
+            allowed to also utilize the ssh-agent. Explicit use of passwords
+            in some tests may need this set as False.
         :type proxy_client: ``tempest.lib.common.ssh.Client`` object
         """
         self.host = host
@@ -105,6 +109,7 @@
             raise exceptions.SSHClientProxyClientLoop(
                 host=self.host, port=self.port, username=self.username)
         self._proxy_conn = None
+        self.ssh_allow_agent = ssh_allow_agent
 
     def _get_ssh_connection(self, sleep=1.5, backoff=1):
         """Returns an ssh connection to the specified host."""
@@ -133,7 +138,7 @@
                             look_for_keys=self.look_for_keys,
                             key_filename=self.key_filename,
                             timeout=self.channel_timeout, pkey=self.pkey,
-                            sock=proxy_chan)
+                            sock=proxy_chan, allow_agent=self.ssh_allow_agent)
                 LOG.info("ssh connection to %s@%s successfully created",
                          self.username, self.host)
                 return ssh
diff --git a/tempest/lib/common/utils/linux/remote_client.py b/tempest/lib/common/utils/linux/remote_client.py
index d0cdc25..662b452 100644
--- a/tempest/lib/common/utils/linux/remote_client.py
+++ b/tempest/lib/common/utils/linux/remote_client.py
@@ -69,7 +69,8 @@
                  server=None, servers_client=None, ssh_timeout=300,
                  connect_timeout=60, console_output_enabled=True,
                  ssh_shell_prologue="set -eu -o pipefail; PATH=$PATH:/sbin;",
-                 ping_count=1, ping_size=56, ssh_key_type='rsa'):
+                 ping_count=1, ping_size=56, ssh_key_type='rsa',
+                 ssh_allow_agent=True):
         """Executes commands in a VM over ssh
 
         :param ip_address: IP address to ssh to
@@ -85,6 +86,8 @@
         :param ping_count: Number of ping packets
         :param ping_size: Packet size for ping packets
         :param ssh_key_type: ssh key type (rsa, ecdsa)
+        :param ssh_allow_agent: Boolean if ssh agent support is permitted.
+            Defaults to True.
         """
         self.server = server
         self.servers_client = servers_client
@@ -94,11 +97,14 @@
         self.ping_count = ping_count
         self.ping_size = ping_size
         self.ssh_key_type = ssh_key_type
+        self.ssh_allow_agent = ssh_allow_agent
 
         self.ssh_client = ssh.Client(ip_address, username, password,
                                      ssh_timeout, pkey=pkey,
                                      channel_timeout=connect_timeout,
-                                     ssh_key_type=ssh_key_type)
+                                     ssh_key_type=ssh_key_type,
+                                     ssh_allow_agent=ssh_allow_agent,
+                                     )
 
     @debug_ssh
     def exec_command(self, cmd):
diff --git a/tempest/lib/services/image/v2/images_client.py b/tempest/lib/services/image/v2/images_client.py
index ae6ce25..8460b57 100644
--- a/tempest/lib/services/image/v2/images_client.py
+++ b/tempest/lib/services/image/v2/images_client.py
@@ -248,17 +248,26 @@
         self.expected_success(202, resp.status)
         return rest_client.ResponseBody(resp)
 
-    def show_image_file(self, image_id):
+    def show_image_file(self, image_id, chunked=False):
         """Download binary image data.
 
+        :param bool chunked: If True, do not read the body and return only
+                             the raw urllib3 response object for processing.
+                             NB: If you pass True here, you **MUST** call
+                             release_conn() on the response object before
+                             finishing!
+
         For a full list of available parameters, please refer to the official
         API reference:
         https://docs.openstack.org/api-ref/image/v2/#download-binary-image-data
         """
         url = 'images/%s/file' % image_id
-        resp, body = self.get(url)
+        resp, body = self.get(url, chunked=chunked)
         self.expected_success([200, 204, 206], resp.status)
-        return rest_client.ResponseBodyData(resp, body)
+        if chunked:
+            return resp
+        else:
+            return rest_client.ResponseBodyData(resp, body)
 
     def add_image_tag(self, image_id, tag):
         """Add an image tag.
diff --git a/tempest/tests/lib/common/test_http.py b/tempest/tests/lib/common/test_http.py
index a19153f..aae6ba2 100644
--- a/tempest/tests/lib/common/test_http.py
+++ b/tempest/tests/lib/common/test_http.py
@@ -149,6 +149,31 @@
              'xtra key': 'Xtra Value'},
             response)
 
+    def test_request_preload(self):
+        # Given
+        connection = self.closing_http()
+        headers = {'Xtra Key': 'Xtra Value'}
+        http_response = urllib3.HTTPResponse(headers=headers)
+        request = self.patch('urllib3.PoolManager.request',
+                             return_value=http_response)
+        retry = self.patch('urllib3.util.Retry')
+
+        # When
+        response, _ = connection.request(
+            method=REQUEST_METHOD,
+            url=REQUEST_URL,
+            headers=headers,
+            preload_content=False)
+
+        # Then
+        request.assert_called_once_with(
+            REQUEST_METHOD,
+            REQUEST_URL,
+            headers=dict(headers, connection='close'),
+            preload_content=False,
+            retries=retry(raise_on_redirect=False, redirect=5))
+        self.assertIsInstance(response, urllib3.HTTPResponse)
+
 
 class TestClosingProxyHttp(TestClosingHttp):
 
diff --git a/tempest/tests/lib/common/test_rest_client.py b/tempest/tests/lib/common/test_rest_client.py
index 910756f..81a76e0 100644
--- a/tempest/tests/lib/common/test_rest_client.py
+++ b/tempest/tests/lib/common/test_rest_client.py
@@ -55,6 +55,7 @@
     def test_get(self):
         __, return_dict = self.rest_client.get(self.url)
         self.assertEqual('GET', return_dict['method'])
+        self.assertTrue(return_dict['preload_content'])
 
     def test_delete(self):
         __, return_dict = self.rest_client.delete(self.url)
@@ -78,6 +79,17 @@
         __, return_dict = self.rest_client.copy(self.url)
         self.assertEqual('COPY', return_dict['method'])
 
+    def test_get_chunked(self):
+        self.useFixture(fixtures.MockPatchObject(self.rest_client,
+                                                 '_log_request'))
+        __, return_dict = self.rest_client.get(self.url, chunked=True)
+        # Default is preload_content=True, make sure we passed False
+        self.assertFalse(return_dict['preload_content'])
+        # Make sure we did not pass chunked=True to urllib3 for GET
+        self.assertFalse(return_dict['chunked'])
+        # Make sure we did not call _log_request() on the raw response
+        self.rest_client._log_request.assert_not_called()
+
 
 class TestRestClientNotFoundHandling(BaseRestClientTestClass):
     def setUp(self):
diff --git a/tempest/tests/lib/fake_http.py b/tempest/tests/lib/fake_http.py
index cfa4b93..5fa0c43 100644
--- a/tempest/tests/lib/fake_http.py
+++ b/tempest/tests/lib/fake_http.py
@@ -21,14 +21,17 @@
         self.return_type = return_type
 
     def request(self, uri, method="GET", body=None, headers=None,
-                redirections=5, connection_type=None, chunked=False):
+                redirections=5, connection_type=None, chunked=False,
+                preload_content=False):
         if not self.return_type:
             fake_headers = fake_http_response(headers)
             return_obj = {
                 'uri': uri,
                 'method': method,
                 'body': body,
-                'headers': headers
+                'headers': headers,
+                'chunked': chunked,
+                'preload_content': preload_content,
             }
             return (fake_headers, return_obj)
         elif isinstance(self.return_type, int):
diff --git a/tempest/tests/lib/services/image/v2/test_images_client.py b/tempest/tests/lib/services/image/v2/test_images_client.py
index 5b162f8..27a50a9 100644
--- a/tempest/tests/lib/services/image/v2/test_images_client.py
+++ b/tempest/tests/lib/services/image/v2/test_images_client.py
@@ -13,6 +13,9 @@
 #    under the License.
 
 import io
+from unittest import mock
+
+import fixtures
 
 from tempest.lib.common.utils import data_utils
 from tempest.lib.services.image.v2 import images_client
@@ -239,6 +242,21 @@
             headers={'Content-Type': 'application/octet-stream'},
             status=200)
 
+    def test_show_image_file_chunked(self):
+        # Since chunked=True on a GET should pass the response object
+        # basically untouched, we use a mock here so we get some assurances.
+        http_response = mock.MagicMock()
+        http_response.status = 200
+        self.useFixture(fixtures.MockPatch(
+            'tempest.lib.common.rest_client.RestClient.get',
+            return_value=(http_response, b'')))
+        resp = self.client.show_image_file(
+            self.FAKE_CREATE_UPDATE_SHOW_IMAGE['id'],
+            chunked=True)
+        self.assertEqual(http_response, resp)
+        resp.__contains__.assert_not_called()
+        resp.__getitem__.assert_not_called()
+
     def test_add_image_tag(self):
         self.check_service_client_function(
             self.client.add_image_tag,
diff --git a/tempest/tests/lib/test_ssh.py b/tempest/tests/lib/test_ssh.py
index 886d99c..13870ba 100644
--- a/tempest/tests/lib/test_ssh.py
+++ b/tempest/tests/lib/test_ssh.py
@@ -75,7 +75,8 @@
             look_for_keys=False,
             timeout=10.0,
             password=None,
-            sock=None
+            sock=None,
+            allow_agent=True
         )]
         self.assertEqual(expected_connect, client_mock.connect.mock_calls)
         self.assertEqual(0, s_mock.call_count)
@@ -91,7 +92,8 @@
 
         proxy_client = ssh.Client('proxy-host', 'proxy-user', timeout=2)
         client = ssh.Client('localhost', 'root', timeout=2,
-                            proxy_client=proxy_client)
+                            proxy_client=proxy_client,
+                            ssh_allow_agent=False)
         client._get_ssh_connection(sleep=1)
 
         aa_mock.assert_has_calls([mock.call(), mock.call()])
@@ -106,7 +108,8 @@
             look_for_keys=False,
             timeout=10.0,
             password=None,
-            sock=None
+            sock=None,
+            allow_agent=True
         )]
         self.assertEqual(proxy_expected_connect,
                          proxy_client_mock.connect.mock_calls)
@@ -121,7 +124,8 @@
             look_for_keys=False,
             timeout=10.0,
             password=None,
-            sock=proxy_client_mock.get_transport().open_session()
+            sock=proxy_client_mock.get_transport().open_session(),
+            allow_agent=False
         )]
         self.assertEqual(expected_connect, client_mock.connect.mock_calls)
         self.assertEqual(0, s_mock.call_count)
diff --git a/tox.ini b/tox.ini
index 618f9e0..e1c17df 100644
--- a/tox.ini
+++ b/tox.ini
@@ -132,10 +132,11 @@
 basepython = {[tempestenv]basepython}
 setenv = {[tempestenv]setenv}
 deps = {[tempestenv]deps}
+regex = '(^tempest\.scenario.*)|(^tempest\.serial_tests)|(?!.*\[.*\bslow\b.*\])(^tempest\.api)'
 # The regex below is used to select all tempest scenario and including the non slow api tests
 commands =
     find . -type f -name "*.pyc" -delete
-    tempest run --regex '(^tempest\.scenario.*)|(^tempest\.serial_tests)|(?!.*\[.*\bslow\b.*\])(^tempest\.api)' {posargs}
+    tempest run --regex {[testenv:full-parallel]regex} {posargs}
 
 [testenv:api-microversion-tests]
 envdir = .tox/tempest
@@ -143,11 +144,12 @@
 basepython = {[tempestenv]basepython}
 setenv = {[tempestenv]setenv}
 deps = {[tempestenv]deps}
+regex = '(^tempest\.api\.compute)|(^tempest\.api\.volume)'
 # The regex below is used to select all tempest api tests for services having API
 # microversion concept.
 commands =
     find . -type f -name "*.pyc" -delete
-    tempest run --regex '(^tempest\.api\.compute)|(^tempest\.api\.volume)' {posargs}
+    tempest run --regex {[testenv:api-microversion-tests]regex} {posargs}
 
 [testenv:integrated-network]
 envdir = .tox/tempest
@@ -155,12 +157,14 @@
 basepython = {[tempestenv]basepython}
 setenv = {[tempestenv]setenv}
 deps = {[tempestenv]deps}
+regex1 = '(?!.*\[.*\bslow\b.*\])(^tempest\.api)'
+regex2 = '(?!.*\[.*\bslow\b.*\])(^tempest\.scenario)|(^tempest\.serial_tests)'
 # The regex below is used to select which tests to run and exclude the slow tag and
 # tests listed in exclude-list file:
 commands =
     find . -type f -name "*.pyc" -delete
-    tempest run --regex '(?!.*\[.*\bslow\b.*\])(^tempest\.api)' --exclude-list ./tools/tempest-integrated-gate-networking-exclude-list.txt {posargs}
-    tempest run --combine --serial --regex '(?!.*\[.*\bslow\b.*\])(^tempest\.scenario)|(^tempest\.serial_tests)' --exclude-list ./tools/tempest-integrated-gate-networking-exclude-list.txt {posargs}
+    tempest run --regex {[testenv:integrated-network]regex1} --exclude-list ./tools/tempest-integrated-gate-networking-exclude-list.txt {posargs}
+    tempest run --combine --serial --regex {[testenv:integrated-network]regex2} --exclude-list ./tools/tempest-integrated-gate-networking-exclude-list.txt {posargs}
 
 [testenv:integrated-compute]
 envdir = .tox/tempest
@@ -168,12 +172,14 @@
 basepython = {[tempestenv]basepython}
 setenv = {[tempestenv]setenv}
 deps = {[tempestenv]deps}
+regex1 = '(?!.*\[.*\bslow\b.*\])(^tempest\.api)'
+regex2 = '(?!.*\[.*\bslow\b.*\])(^tempest\.scenario)|(^tempest\.serial_tests)'
 # The regex below is used to select which tests to run and exclude the slow tag and
 # tests listed in exclude-list file:
 commands =
     find . -type f -name "*.pyc" -delete
-    tempest run --regex '(?!.*\[.*\bslow\b.*\])(^tempest\.api)' --exclude-list ./tools/tempest-integrated-gate-compute-exclude-list.txt {posargs}
-    tempest run --combine --serial --regex '(?!.*\[.*\bslow\b.*\])(^tempest\.scenario)|(^tempest\.serial_tests)' --exclude-list ./tools/tempest-integrated-gate-compute-exclude-list.txt {posargs}
+    tempest run --regex {[testenv:integrated-compute]regex1} --exclude-list ./tools/tempest-integrated-gate-compute-exclude-list.txt {posargs}
+    tempest run --combine --serial --regex {[testenv:integrated-compute]regex2} --exclude-list ./tools/tempest-integrated-gate-compute-exclude-list.txt {posargs}
 
 [testenv:integrated-placement]
 envdir = .tox/tempest
@@ -181,12 +187,14 @@
 basepython = {[tempestenv]basepython}
 setenv = {[tempestenv]setenv}
 deps = {[tempestenv]deps}
+regex1 = '(?!.*\[.*\bslow\b.*\])(^tempest\.api)'
+regex2 = '(?!.*\[.*\bslow\b.*\])(^tempest\.scenario)|(^tempest\.serial_tests)'
 # The regex below is used to select which tests to run and exclude the slow tag and
 # tests listed in exclude-list file:
 commands =
     find . -type f -name "*.pyc" -delete
-    tempest run --regex '(?!.*\[.*\bslow\b.*\])(^tempest\.api)' --exclude-list ./tools/tempest-integrated-gate-placement-exclude-list.txt {posargs}
-    tempest run --combine --serial --regex '(?!.*\[.*\bslow\b.*\])(^tempest\.scenario)|(^tempest\.serial_tests)' --exclude-list ./tools/tempest-integrated-gate-placement-exclude-list.txt {posargs}
+    tempest run --regex {[testenv:integrated-placement]regex1} --exclude-list ./tools/tempest-integrated-gate-placement-exclude-list.txt {posargs}
+    tempest run --combine --serial --regex {[testenv:integrated-placement]regex2} --exclude-list ./tools/tempest-integrated-gate-placement-exclude-list.txt {posargs}
 
 [testenv:integrated-storage]
 envdir = .tox/tempest
@@ -194,12 +202,14 @@
 basepython = {[tempestenv]basepython}
 setenv = {[tempestenv]setenv}
 deps = {[tempestenv]deps}
+regex1 = '(?!.*\[.*\bslow\b.*\])(^tempest\.api)'
+regex2 = '(?!.*\[.*\bslow\b.*\])(^tempest\.scenario)|(^tempest\.serial_tests)'
 # The regex below is used to select which tests to run and exclude the slow tag and
 # tests listed in exclude-list file:
 commands =
     find . -type f -name "*.pyc" -delete
-    tempest run --regex '(?!.*\[.*\bslow\b.*\])(^tempest\.api)' --exclude-list ./tools/tempest-integrated-gate-storage-exclude-list.txt {posargs}
-    tempest run --combine --serial --regex '(?!.*\[.*\bslow\b.*\])(^tempest\.scenario)|(^tempest\.serial_tests)' --exclude-list ./tools/tempest-integrated-gate-storage-exclude-list.txt {posargs}
+    tempest run --regex {[testenv:integrated-storage]regex1} --exclude-list ./tools/tempest-integrated-gate-storage-exclude-list.txt {posargs}
+    tempest run --combine --serial --regex {[testenv:integrated-storage]regex2} --exclude-list ./tools/tempest-integrated-gate-storage-exclude-list.txt {posargs}
 
 [testenv:integrated-object-storage]
 envdir = .tox/tempest
@@ -207,12 +217,14 @@
 basepython = {[tempestenv]basepython}
 setenv = {[tempestenv]setenv}
 deps = {[tempestenv]deps}
+regex1 = '(?!.*\[.*\bslow\b.*\])(^tempest\.api)'
+regex2 = '(?!.*\[.*\bslow\b.*\])(^tempest\.scenario)|(^tempest\.serial_tests)'
 # The regex below is used to select which tests to run and exclude the slow tag and
 # tests listed in exclude-list file:
 commands =
     find . -type f -name "*.pyc" -delete
-    tempest run --regex '(?!.*\[.*\bslow\b.*\])(^tempest\.api)' --exclude-list ./tools/tempest-integrated-gate-object-storage-exclude-list.txt {posargs}
-    tempest run --combine --serial --regex '(?!.*\[.*\bslow\b.*\])(^tempest\.scenario)|(^tempest\.serial_tests)' --exclude-list ./tools/tempest-integrated-gate-object-storage-exclude-list.txt {posargs}
+    tempest run --regex {[testenv:integrated-object-storage]regex1} --exclude-list ./tools/tempest-integrated-gate-object-storage-exclude-list.txt {posargs}
+    tempest run --combine --serial --regex {[testenv:integrated-object-storage]regex2} --exclude-list ./tools/tempest-integrated-gate-object-storage-exclude-list.txt {posargs}
 
 [testenv:full-serial]
 envdir = .tox/tempest
@@ -220,12 +232,13 @@
 basepython = {[tempestenv]basepython}
 setenv = {[tempestenv]setenv}
 deps = {[tempestenv]deps}
+regex = '(?!.*\[.*\bslow\b.*\])(^tempest\.(api|scenario|serial_tests))'
 # The regex below is used to select which tests to run and exclude the slow tag:
 # See the testrepository bug: https://bugs.launchpad.net/testrepository/+bug/1208610
 # FIXME: We can replace it with the `--exclude-regex` option to exclude tests now.
 commands =
     find . -type f -name "*.pyc" -delete
-    tempest run --serial --regex '(?!.*\[.*\bslow\b.*\])(^tempest\.(api|scenario|serial_tests))' {posargs}
+    tempest run --serial --regex {[testenv:full-serial]regex} {posargs}
 
 [testenv:scenario]
 envdir = .tox/tempest
@@ -233,10 +246,11 @@
 basepython = {[tempestenv]basepython}
 setenv = {[tempestenv]setenv}
 deps = {[tempestenv]deps}
+regex = '(^tempest\.scenario)'
 # The regex below is used to select all scenario tests
 commands =
     find . -type f -name "*.pyc" -delete
-    tempest run --serial --regex '(^tempest\.scenario)' {posargs}
+    tempest run --serial --regex {[testenv:scenario]regex} {posargs}
 
 [testenv:smoke]
 envdir = .tox/tempest
@@ -244,9 +258,10 @@
 basepython = {[tempestenv]basepython}
 setenv = {[tempestenv]setenv}
 deps = {[tempestenv]deps}
+regex = '\[.*\bsmoke\b.*\]'
 commands =
     find . -type f -name "*.pyc" -delete
-    tempest run --regex '\[.*\bsmoke\b.*\]' {posargs}
+    tempest run --regex {[testenv:smoke]regex} {posargs}
 
 [testenv:smoke-serial]
 envdir = .tox/tempest
@@ -254,12 +269,13 @@
 basepython = {[tempestenv]basepython}
 setenv = {[tempestenv]setenv}
 deps = {[tempestenv]deps}
+regex = '\[.*\bsmoke\b.*\]'
 # This is still serial because neutron doesn't work with parallel. See:
 # https://bugs.launchpad.net/tempest/+bug/1216076 so the neutron smoke
 # job would fail if we moved it to parallel.
 commands =
     find . -type f -name "*.pyc" -delete
-    tempest run --serial --regex '\[.*\bsmoke\b.*\]' {posargs}
+    tempest run --serial --regex {[testenv:smoke-serial]regex} {posargs}
 
 [testenv:slow-serial]
 envdir = .tox/tempest
@@ -267,10 +283,11 @@
 basepython = {[tempestenv]basepython}
 setenv = {[tempestenv]setenv}
 deps = {[tempestenv]deps}
+regex = '\[.*\bslow\b.*\]'
 # The regex below is used to select the slow tagged tests to run serially:
 commands =
     find . -type f -name "*.pyc" -delete
-    tempest run --serial --regex '\[.*\bslow\b.*\]' {posargs}
+    tempest run --serial --regex {[testenv:slow-serial]regex} {posargs}
 
 [testenv:ipv6-only]
 envdir = .tox/tempest
@@ -278,12 +295,13 @@
 basepython = {[tempestenv]basepython}
 setenv = {[tempestenv]setenv}
 deps = {[tempestenv]deps}
+regex = '\[.*\bsmoke|ipv6|test_network_v6\b.*\]'
 # Run only smoke and ipv6 tests. This env is used to tests
 # the ipv6 deployments and basic tests run fine so that we can
 # verify that services listen on IPv6 address.
 commands =
     find . -type f -name "*.pyc" -delete
-    tempest run --regex '\[.*\bsmoke|ipv6|test_network_v6\b.*\]' {posargs}
+    tempest run --regex {[testenv:ipv6-only]regex} {posargs}
 
 [testenv:venv]
 deps =
@@ -442,8 +460,9 @@
 basepython = {[tempestenv]basepython}
 setenv = {[tempestenv]setenv}
 deps = {[tempestenv]deps}
+regex = '\[.*\bsmoke\b.*\]'
 # The below command install stestr master version and run smoke tests
 commands =
     find . -type f -name "*.pyc" -delete
     pip install -U git+https://github.com/mtreinish/stestr
-    tempest run --regex '\[.*\bsmoke\b.*\]' {posargs}
+    tempest run --regex {[testenv:stestr-master]regex} {posargs}
diff --git a/zuul.d/project.yaml b/zuul.d/project.yaml
index 966cc9a..68fd14f 100644
--- a/zuul.d/project.yaml
+++ b/zuul.d/project.yaml
@@ -42,7 +42,11 @@
             irrelevant-files: *tempest-irrelevant-files
         - tempest-full-xena:
             irrelevant-files: *tempest-irrelevant-files
+        # Temporarily marked as n-v due to the below bug which blocks
+        # the CI and complicates merging of patches.
+        # https://bugs.launchpad.net/tempest/+bug/1998916
         - tempest-multinode-full-py3:
+            voting: false
             irrelevant-files: *tempest-irrelevant-files
         - tempest-tox-plugin-sanity-check:
             irrelevant-files: &tempest-irrelevant-files-2
@@ -146,8 +150,9 @@
             irrelevant-files: *tempest-irrelevant-files
         - tempest-ipv6-only:
             irrelevant-files: *tempest-irrelevant-files-3
-        - tempest-multinode-full-py3:
-            irrelevant-files: *tempest-irrelevant-files
+        # https://bugs.launchpad.net/tempest/+bug/1998916
+        #- tempest-multinode-full-py3:
+        #    irrelevant-files: *tempest-irrelevant-files
         - tempest-full-enforce-scope-new-defaults:
             irrelevant-files: *tempest-irrelevant-files
         #- devstack-plugin-ceph-tempest-py3: