Merge "Fix potential issue with adding a fixed ip"
diff --git a/.zuul.yaml b/.zuul.yaml
index 9753656..71bbeca 100644
--- a/.zuul.yaml
+++ b/.zuul.yaml
@@ -247,30 +247,10 @@
       devstack_localrc:
         USE_PYTHON3: true
 
-- nodeset:
-    name: openstack-bionic-node
-    nodes:
-      - name: controller
-        label: ubuntu-bionic
-    groups:
-      - name: tempest
-        nodes:
-          - controller
-
-- nodeset:
-    name: openstack-opensuse150-node
-    nodes:
-      - name: controller
-        label: opensuse-150
-    groups:
-      - name: tempest
-        nodes:
-          - controller
-
 - job:
     name: tempest-full-py3-opensuse150
     parent: tempest-full-py3
-    nodeset: openstack-opensuse150-node
+    nodeset: devstack-single-node-opensuse-150
     description: |
       Base integration test with Neutron networking and py36 running
       on openSUSE Leap 15.0
@@ -351,7 +331,7 @@
     parent: tox
     description: |
       Run tempest plugin sanity check script using tox.
-    nodeset: ubuntu-xenial
+    nodeset: ubuntu-bionic
     vars:
       tox_envlist: plugin-sanity-check
     voting: false
diff --git a/playbooks/devstack-tempest.yaml b/playbooks/devstack-tempest.yaml
index b51e701..5f87abd 100644
--- a/playbooks/devstack-tempest.yaml
+++ b/playbooks/devstack-tempest.yaml
@@ -11,7 +11,7 @@
     # This enviroment variable is used by the optional tempest-gabbi
     # job provided by the gabbi-tempest plugin. It can be safely ignored
     # if that plugin is not being used.
-    GABBI_TEMPEST_PATH: "{{ gabbi_tempest_path }}"
+    GABBI_TEMPEST_PATH: "{{ gabbi_tempest_path | default('') }}"
   roles:
     - setup-tempest-run-dir
     - setup-tempest-data-dir
diff --git a/releasenotes/notes/add-profiler-config-options-db7c4ae6d338ee5c.yaml b/releasenotes/notes/add-profiler-config-options-db7c4ae6d338ee5c.yaml
new file mode 100644
index 0000000..2245044
--- /dev/null
+++ b/releasenotes/notes/add-profiler-config-options-db7c4ae6d338ee5c.yaml
@@ -0,0 +1,10 @@
+---
+features:
+  - |
+    Add support of `OSProfiler library`_ for profiling and distributed
+    tracing of OpenStack. A new config option ``key`` in section ``profiler``
+    is added, the option sets the secret key used to enable profiling in
+    OpenStack services. The value needs to correspond to the one specified
+    in [profiler]/hmac_keys option of OpenStack services.
+
+    .. _OSProfiler library: https://docs.openstack.org/osprofiler/
diff --git a/releasenotes/notes/bug-1808473-54ada26ab78e7b02.yaml b/releasenotes/notes/bug-1808473-54ada26ab78e7b02.yaml
new file mode 100644
index 0000000..c280198
--- /dev/null
+++ b/releasenotes/notes/bug-1808473-54ada26ab78e7b02.yaml
@@ -0,0 +1,7 @@
+---
+fixes:
+  - |
+    Fixed bug #1808473. ``tempest run`` CLI will error if a non-exist config file is
+    input to parameter --config-file. Earlier non-exist config value was silently
+    getting ignored and the default config file was used instead which used to give
+    false behavior to the user on using the passed config file.
diff --git a/releasenotes/notes/tempest-stein-release-18bad34136a2e6ef.yaml b/releasenotes/notes/tempest-stein-release-18bad34136a2e6ef.yaml
new file mode 100644
index 0000000..212cf7d
--- /dev/null
+++ b/releasenotes/notes/tempest-stein-release-18bad34136a2e6ef.yaml
@@ -0,0 +1,18 @@
+---
+prelude: >
+    This release is to tag the Tempest for OpenStack Stein release.
+    This release marks the start of Stein release support in Tempest and
+    the end of support for Ocata in Tempest.
+    After this release, Tempest will support below OpenStack Releases:
+
+      * Stein
+      * Rocky
+      * Queens
+      * Pike
+
+    Current development of Tempest is for OpenStack Train development
+    cycle. Every Tempest commit is also tested against master during
+    the Train cycle. However, this does not necessarily mean that using
+    Tempest as of this tag will work against a Train (or future release)
+    cloud.
+    To be on safe side, use this tag to test the OpenStack Stein release.
diff --git a/tempest/api/compute/servers/test_novnc.py b/tempest/api/compute/servers/test_novnc.py
index 5801db1..daf6a06 100644
--- a/tempest/api/compute/servers/test_novnc.py
+++ b/tempest/api/compute/servers/test_novnc.py
@@ -143,7 +143,7 @@
         data_length = len(data) if data is not None else 0
         self.assertFalse(data_length <= 24 or
                          data_length != (struct.unpack(">L",
-                                         data[20:24])[0] + 24),
+                                                       data[20:24])[0] + 24),
                          'Server initialization was not the right format.')
         # Since the rest of the data on the screen is arbitrary, we will
         # close the socket and end our validation of the data at this point
@@ -151,7 +151,7 @@
         # initialization was the right format
         self.assertFalse(data_length <= 24 or
                          data_length != (struct.unpack(">L",
-                                         data[20:24])[0] + 24))
+                                                       data[20:24])[0] + 24))
 
     def _validate_websocket_upgrade(self):
         self.assertTrue(
diff --git a/tempest/api/identity/admin/v3/test_inherits.py b/tempest/api/identity/admin/v3/test_inherits.py
index acc5a8c..2672f71 100644
--- a/tempest/api/identity/admin/v3/test_inherits.py
+++ b/tempest/api/identity/admin/v3/test_inherits.py
@@ -9,14 +9,22 @@
 #    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
 #    License for the specific language governing permissions and limitations
 #    under the License.
+import testtools
 
 from tempest.api.identity import base
 from tempest.common import utils
+from tempest import config
 from tempest.lib.common.utils import data_utils
 from tempest.lib import decorators
 
+CONF = config.CONF
+
 
 class InheritsV3TestJSON(base.BaseIdentityV3AdminTest):
+    # NOTE: force_tenant_isolation is true in the base class by default but
+    # overridden to false here to allow test execution for clouds using the
+    # pre-provisioned credentials provider.
+    force_tenant_isolation = False
 
     @classmethod
     def skip_checks(cls):
@@ -43,18 +51,26 @@
             domain_id=cls.domain['id'])['group']
         cls.addClassResourceCleanup(cls.groups_client.delete_group,
                                     cls.group['id'])
-        cls.user = cls.users_client.create_user(
-            name=u_name, description=u_desc, password=u_password,
-            email=u_email, project_id=cls.project['id'],
-            domain_id=cls.domain['id'])['user']
-        cls.addClassResourceCleanup(cls.users_client.delete_user,
-                                    cls.user['id'])
+        if not CONF.identity_feature_enabled.immutable_user_source:
+            cls.user = cls.users_client.create_user(
+                name=u_name,
+                description=u_desc,
+                password=u_password,
+                email=u_email,
+                project_id=cls.project['id'],
+                domain_id=cls.domain['id']
+            )['user']
+            cls.addClassResourceCleanup(cls.users_client.delete_user,
+                                        cls.user['id'])
 
     def _list_assertions(self, body, fetched_role_ids, role_id):
         self.assertEqual(len(body), 1)
         self.assertIn(role_id, fetched_role_ids)
 
     @decorators.idempotent_id('4e6f0366-97c8-423c-b2be-41eae6ac91c8')
+    @testtools.skipIf(CONF.identity_feature_enabled.immutable_user_source,
+                      'Skipped because environment has an immutable user '
+                      'source and solely provides read-only access to users.')
     def test_inherit_assign_list_check_revoke_roles_on_domains_user(self):
         # Create role
         src_role = self.setup_test_role()
@@ -103,6 +119,9 @@
             self.domain['id'], self.group['id'], src_role['id'])
 
     @decorators.idempotent_id('18b70e45-7687-4b72-8277-b8f1a47d7591')
+    @testtools.skipIf(CONF.identity_feature_enabled.immutable_user_source,
+                      'Skipped because environment has an immutable user '
+                      'source and solely provides read-only access to users.')
     def test_inherit_assign_check_revoke_roles_on_projects_user(self):
         # Create role
         src_role = self.setup_test_role()
@@ -134,6 +153,9 @@
              self.project['id'], self.group['id'], src_role['id']))
 
     @decorators.idempotent_id('3acf666e-5354-42ac-8e17-8b68893bcd36')
+    @testtools.skipIf(CONF.identity_feature_enabled.immutable_user_source,
+                      'Skipped because environment has an immutable user '
+                      'source and solely provides read-only access to users.')
     def test_inherit_assign_list_revoke_user_roles_on_domain(self):
         # Create role
         src_role = self.setup_test_role()
@@ -178,6 +200,9 @@
         self.assertEmpty(assignments)
 
     @decorators.idempotent_id('9f02ccd9-9b57-46b4-8f77-dd5a736f3a06')
+    @testtools.skipIf(CONF.identity_feature_enabled.immutable_user_source,
+                      'Skipped because environment has an immutable user '
+                      'source and solely provides read-only access to users.')
     def test_inherit_assign_list_revoke_user_roles_on_project_tree(self):
         # Create role
         src_role = self.setup_test_role()
diff --git a/tempest/api/identity/admin/v3/test_list_projects.py b/tempest/api/identity/admin/v3/test_list_projects.py
index 50f3186..299a618 100644
--- a/tempest/api/identity/admin/v3/test_list_projects.py
+++ b/tempest/api/identity/admin/v3/test_list_projects.py
@@ -13,11 +13,14 @@
 #    License for the specific language governing permissions and limitations
 #    under the License.
 
+from oslo_log import log as logging
+
 from tempest.api.identity import base
 from tempest import config
 from tempest.lib.common.utils import data_utils
 from tempest.lib import decorators
 
+LOG = logging.getLogger(__name__)
 CONF = config.CONF
 
 
@@ -26,6 +29,9 @@
     def _list_projects_with_params(self, included, excluded, params, key):
         # Validate that projects in ``included`` belongs to the projects
         # returned that match ``params`` but not projects in ``excluded``
+        all_projects = self.projects_client.list_projects()['projects']
+        LOG.debug("Complete list of projects available in keystone: %s",
+                  all_projects)
         body = self.projects_client.list_projects(params)['projects']
         for p in included:
             self.assertIn(p[key], map(lambda x: x[key], body))
@@ -39,13 +45,12 @@
     def resource_setup(cls):
         super(ListProjectsTestJSON, cls).resource_setup()
         cls.project_ids = list()
-        # Create a domain
-        cls.domain = cls.create_domain()
+        cls.domain_id = cls.os_admin.credentials.domain_id
         # Create project with domain
         cls.p1_name = data_utils.rand_name('project')
         cls.p1 = cls.projects_client.create_project(
             cls.p1_name, enabled=False,
-            domain_id=cls.domain['id'])['project']
+            domain_id=cls.domain_id)['project']
         cls.addClassResourceCleanup(cls.projects_client.delete_project,
                                     cls.p1['id'])
         cls.project_ids.append(cls.p1['id'])
diff --git a/tempest/api/identity/admin/v3/test_roles.py b/tempest/api/identity/admin/v3/test_roles.py
index 47f663c..b67de95 100644
--- a/tempest/api/identity/admin/v3/test_roles.py
+++ b/tempest/api/identity/admin/v3/test_roles.py
@@ -12,13 +12,17 @@
 #    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
 #    License for the specific language governing permissions and limitations
 #    under the License.
+import testtools
 
 from tempest.api.identity import base
+from tempest import config
 from tempest.lib.common.utils import data_utils
 from tempest.lib.common.utils import test_utils
 from tempest.lib import decorators
 from tempest.lib import exceptions as lib_exc
 
+CONF = config.CONF
+
 
 class RolesV3TestJSON(base.BaseIdentityV3AdminTest):
 
@@ -48,16 +52,21 @@
             domain_id=cls.domain['id'])['group']
         cls.addClassResourceCleanup(cls.groups_client.delete_group,
                                     cls.group_body['id'])
-        cls.user_body = cls.users_client.create_user(
-            name=u_name, description=u_desc, password=cls.u_password,
-            email=u_email, project_id=cls.project['id'],
-            domain_id=cls.domain['id'])['user']
-        cls.addClassResourceCleanup(cls.users_client.delete_user,
-                                    cls.user_body['id'])
         cls.role = cls.roles_client.create_role(
             name=data_utils.rand_name('Role'))['role']
         cls.addClassResourceCleanup(cls.roles_client.delete_role,
                                     cls.role['id'])
+        if not CONF.identity_feature_enabled.immutable_user_source:
+            cls.user_body = cls.users_client.create_user(
+                name=u_name,
+                description=u_desc,
+                email=u_email,
+                password=cls.u_password,
+                domain_id=cls.domain['id'],
+                project_id=cls.project['id']
+            )['user']
+            cls.addClassResourceCleanup(cls.users_client.delete_user,
+                                        cls.user_body['id'])
 
     @decorators.attr(type='smoke')
     @decorators.idempotent_id('18afc6c0-46cf-4911-824e-9989cc056c3a')
@@ -84,6 +93,9 @@
         self.assertIn(role['id'], [r['id'] for r in roles])
 
     @decorators.idempotent_id('c6b80012-fe4a-498b-9ce8-eb391c05169f')
+    @testtools.skipIf(CONF.identity_feature_enabled.immutable_user_source,
+                      'Skipped because environment has an immutable user '
+                      'source and solely provides read-only access to users.')
     def test_grant_list_revoke_role_to_user_on_project(self):
         self.roles_client.create_user_role_on_project(self.project['id'],
                                                       self.user_body['id'],
@@ -102,6 +114,9 @@
             self.project['id'], self.user_body['id'], self.role['id'])
 
     @decorators.idempotent_id('6c9a2940-3625-43a3-ac02-5dcec62ef3bd')
+    @testtools.skipIf(CONF.identity_feature_enabled.immutable_user_source,
+                      'Skipped because environment has an immutable user '
+                      'source and solely provides read-only access to users.')
     def test_grant_list_revoke_role_to_user_on_domain(self):
         self.roles_client.create_user_role_on_domain(
             self.domain['id'], self.user_body['id'], self.role['id'])
@@ -119,6 +134,9 @@
             self.domain['id'], self.user_body['id'], self.role['id'])
 
     @decorators.idempotent_id('cbf11737-1904-4690-9613-97bcbb3df1c4')
+    @testtools.skipIf(CONF.identity_feature_enabled.immutable_user_source,
+                      'Skipped because environment has an immutable user '
+                      'source and solely provides read-only access to users.')
     def test_grant_list_revoke_role_to_group_on_project(self):
         # Grant role to group on project
         self.roles_client.create_group_role_on_project(
@@ -254,6 +272,9 @@
         self.assertIn(self.roles[2]['id'], implies_ids)
 
     @decorators.idempotent_id('c8828027-df48-4021-95df-b65b92c7429e')
+    @testtools.skipIf(CONF.identity_feature_enabled.immutable_user_source,
+                      'Skipped because environment has an immutable user '
+                      'source and solely provides read-only access to users.')
     def test_assignments_for_implied_roles_create_delete(self):
         # Create a grant using "roles[0]"
         self.roles_client.create_user_role_on_project(
@@ -344,6 +365,9 @@
             domain_role1['id'])
 
     @decorators.idempotent_id('3859df7e-5b78-4e4d-b10e-214c8953842a')
+    @testtools.skipIf(CONF.identity_feature_enabled.immutable_user_source,
+                      'Skipped because environment has an immutable user '
+                      'source and solely provides read-only access to users.')
     def test_assignments_for_domain_roles(self):
         domain_role = self.setup_test_role(domain_id=self.domain['id'])
 
diff --git a/tempest/api/network/test_dhcp_ipv6.py b/tempest/api/network/test_dhcp_ipv6.py
index 3ab2909..eb31ed3 100644
--- a/tempest/api/network/test_dhcp_ipv6.py
+++ b/tempest/api/network/test_dhcp_ipv6.py
@@ -206,7 +206,7 @@
                                  for k in port['fixed_ips']])
                 real_dhcp_ip, real_eui_ip = [real_ips[sub['id']]
                                              for sub in [subnet_dhcp,
-                                             subnet_slaac]]
+                                                         subnet_slaac]]
                 self.ports_client.delete_port(port['id'])
                 self.ports.pop()
                 body = self.ports_client.list_ports()
@@ -257,7 +257,7 @@
                                  for k in port['fixed_ips']])
                 real_dhcp_ip, real_eui_ip = [real_ips[sub['id']]
                                              for sub in [subnet_dhcp,
-                                             subnet_slaac]]
+                                                         subnet_slaac]]
                 self._clean_network()
                 self.assertEqual(real_eui_ip,
                                  eui_ip,
diff --git a/tempest/api/network/test_networks.py b/tempest/api/network/test_networks.py
index 7345fd1..ed8eb52 100644
--- a/tempest/api/network/test_networks.py
+++ b/tempest/api/network/test_networks.py
@@ -317,7 +317,7 @@
 
         subnet = self.create_subnet(
             network, **self.subnet_dict(['gateway', 'host_routes',
-                                        'dns_nameservers',
+                                         'dns_nameservers',
                                          'allocation_pools']))
         subnet_id = subnet['id']
         new_gateway = str(netaddr.IPAddress(
diff --git a/tempest/api/network/test_ports.py b/tempest/api/network/test_ports.py
index 2c9159c..25976ce 100644
--- a/tempest/api/network/test_ports.py
+++ b/tempest/api/network/test_ports.py
@@ -107,7 +107,7 @@
         address = self.cidr
         address.prefixlen = self.mask_bits
         if ((address.version == 4 and address.prefixlen >= 30) or
-           (address.version == 6 and address.prefixlen >= 126)):
+            (address.version == 6 and address.prefixlen >= 126)):
             msg = ("Subnet %s isn't large enough for the test" % address.cidr)
             raise exceptions.InvalidConfiguration(msg)
         allocation_pools = {'allocation_pools': [{'start': str(address[2]),
diff --git a/tempest/cmd/cleanup.py b/tempest/cmd/cleanup.py
index f6af0ba..e6db2e9 100644
--- a/tempest/cmd/cleanup.py
+++ b/tempest/cmd/cleanup.py
@@ -310,8 +310,8 @@
             svc.run()
 
         with open(SAVED_STATE_JSON, 'w+') as f:
-            f.write(json.dumps(data,
-                    sort_keys=True, indent=2, separators=(',', ': ')))
+            f.write(json.dumps(data, sort_keys=True,
+                               indent=2, separators=(',', ': ')))
 
     def _load_json(self, saved_state_json=SAVED_STATE_JSON):
         try:
diff --git a/tempest/cmd/run.py b/tempest/cmd/run.py
index 823ed11..77d4496 100644
--- a/tempest/cmd/run.py
+++ b/tempest/cmd/run.py
@@ -103,6 +103,9 @@
 from tempest.common import credentials_factory as credentials
 from tempest import config
 
+if six.PY2:
+    # Python 2 has not FileNotFoundError exception
+    FileNotFoundError = IOError
 
 CONF = config.CONF
 SAVED_STATE_JSON = "saved_state.json"
@@ -112,7 +115,12 @@
 
     def _set_env(self, config_file=None):
         if config_file:
-            CONF.set_config_path(os.path.abspath(config_file))
+            if os.path.exists(os.path.abspath(config_file)):
+                CONF.set_config_path(os.path.abspath(config_file))
+            else:
+                raise FileNotFoundError(
+                    "Config file: %s doesn't exist" % config_file)
+
         # NOTE(mtreinish): This is needed so that stestr doesn't gobble up any
         # stacktraces on failure.
         if 'TESTR_PDB' in os.environ:
@@ -202,8 +210,8 @@
             svc.run()
 
         with open(SAVED_STATE_JSON, 'w+') as f:
-            f.write(json.dumps(data,
-                    sort_keys=True, indent=2, separators=(',', ': ')))
+            f.write(json.dumps(data, sort_keys=True,
+                               indent=2, separators=(',', ': ')))
 
     def get_parser(self, prog_name):
         parser = super(TempestRun, self).get_parser(prog_name)
diff --git a/tempest/common/identity.py b/tempest/common/identity.py
index 525110b..cd6d058 100644
--- a/tempest/common/identity.py
+++ b/tempest/common/identity.py
@@ -26,7 +26,7 @@
         if project['name'] == project_name:
             return project
     raise lib_exc.NotFound('No such project(%s) in %s' % (project_name,
-                           projects))
+                                                          projects))
 
 
 def get_tenant_by_name(client, tenant_name):
diff --git a/tempest/common/utils/net_utils.py b/tempest/common/utils/net_utils.py
index 867b3dd..b697ef1 100644
--- a/tempest/common/utils/net_utils.py
+++ b/tempest/common/utils/net_utils.py
@@ -19,7 +19,6 @@
 
 def get_unused_ip_addresses(ports_client, subnets_client,
                             network_id, subnet_id, count):
-
     """Return a list with the specified number of unused IP addresses
 
     This method uses the given ports_client to find the specified number of
diff --git a/tempest/config.py b/tempest/config.py
index fbe18a3..24ae3ae 100644
--- a/tempest/config.py
+++ b/tempest/config.py
@@ -1100,6 +1100,18 @@
 """)
 ]
 
+
+profiler_group = cfg.OptGroup(name="profiler",
+                              title="OpenStack Profiler")
+
+ProfilerGroup = [
+    cfg.StrOpt('key',
+               help="The secret key to enable OpenStack Profiler. The value "
+                    "should match the one configured in OpenStack services "
+                    "under `[profiler]/hmac_keys` property. The default empty "
+                    "value keeps profiling disabled"),
+]
+
 DefaultGroup = [
     cfg.BoolOpt('pause_teardown',
                 default=False,
@@ -1132,6 +1144,7 @@
     (service_available_group, ServiceAvailableGroup),
     (debug_group, DebugGroup),
     (placement_group, PlacementGroup),
+    (profiler_group, ProfilerGroup),
     (None, DefaultGroup)
 ]
 
@@ -1249,7 +1262,7 @@
 
         logging_cfg_path = "%s/logging.conf" % os.path.dirname(path)
         if ((not hasattr(_CONF, 'log_config_append') or
-            _CONF.log_config_append is None) and
+             _CONF.log_config_append is None) and
             os.path.isfile(logging_cfg_path)):
             # if logging conf is in place we need to set log_config_append
             _CONF.log_config_append = logging_cfg_path
diff --git a/tempest/lib/cmd/check_uuid.py b/tempest/lib/cmd/check_uuid.py
index ac40eef..71ecb32 100755
--- a/tempest/lib/cmd/check_uuid.py
+++ b/tempest/lib/cmd/check_uuid.py
@@ -110,7 +110,7 @@
             for item in files:
                 if item.endswith('.py'):
                     module_name = '.'.join((root_package,
-                                           os.path.splitext(item)[0]))
+                                            os.path.splitext(item)[0]))
                     if not module_name.startswith(UNIT_TESTS_EXCLUDE):
                         modules.append(module_name)
         return modules
diff --git a/tempest/lib/common/api_version_utils.py b/tempest/lib/common/api_version_utils.py
index bcb076b..709c319 100644
--- a/tempest/lib/common/api_version_utils.py
+++ b/tempest/lib/common/api_version_utils.py
@@ -54,7 +54,7 @@
     config_min_version = api_version_request.APIVersionRequest(cfg_min_version)
     config_max_version = api_version_request.APIVersionRequest(cfg_max_version)
     if ((min_version > max_version) or
-       (config_min_version > config_max_version)):
+        (config_min_version > config_max_version)):
         msg = ("Test Class versions [%s - %s]. "
                "Configuration versions [%s - %s]."
                % (min_version.get_string(),
diff --git a/tempest/lib/common/preprov_creds.py b/tempest/lib/common/preprov_creds.py
index fcdeb17..1011504 100644
--- a/tempest/lib/common/preprov_creds.py
+++ b/tempest/lib/common/preprov_creds.py
@@ -273,7 +273,7 @@
             # NOTE(andreaf) Not all fields may be available on all credentials
             # so defaulting to None for that case.
             if all([getattr(creds, k, None) == hash_attributes.get(k, None) for
-                   k in init_attributes]):
+                    k in init_attributes]):
                 return _hash
         raise AttributeError('Invalid credentials %s' % creds)
 
diff --git a/tempest/lib/common/profiler.py b/tempest/lib/common/profiler.py
new file mode 100644
index 0000000..1544337
--- /dev/null
+++ b/tempest/lib/common/profiler.py
@@ -0,0 +1,64 @@
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+import base64
+import hashlib
+import hmac
+import json
+
+from oslo_utils import encodeutils
+from oslo_utils import uuidutils
+
+_profiler = {}
+
+
+def enable(profiler_key, trace_id=None):
+    """Enable global profiler instance
+
+    :param profiler_key: the secret key used to enable profiling in services
+    :param trace_id: unique id of the trace, if empty the id is generated
+        automatically
+    """
+    _profiler['key'] = profiler_key
+    _profiler['uuid'] = trace_id or uuidutils.generate_uuid()
+
+
+def disable():
+    """Disable global profiler instance"""
+    _profiler.clear()
+
+
+def serialize_as_http_headers():
+    """Serialize profiler state as HTTP headers
+
+    This function corresponds to the one from osprofiler library.
+    :return: dictionary with 2 keys `X-Trace-Info` and `X-Trace-HMAC`.
+    """
+    p = _profiler
+    if not p:  # profiler is not enabled
+        return {}
+
+    info = {'base_id': p['uuid'], 'parent_id': p['uuid']}
+    trace_info = base64.urlsafe_b64encode(
+        encodeutils.to_utf8(json.dumps(info)))
+    trace_hmac = _sign(trace_info, p['key'])
+
+    return {
+        'X-Trace-Info': trace_info,
+        'X-Trace-HMAC': trace_hmac,
+    }
+
+
+def _sign(trace_info, key):
+    h = hmac.new(encodeutils.to_utf8(key), digestmod=hashlib.sha1)
+    h.update(trace_info)
+    return h.hexdigest()
diff --git a/tempest/lib/common/rest_client.py b/tempest/lib/common/rest_client.py
index 3be441e..f076727 100644
--- a/tempest/lib/common/rest_client.py
+++ b/tempest/lib/common/rest_client.py
@@ -27,6 +27,7 @@
 
 from tempest.lib.common import http
 from tempest.lib.common import jsonschema_validator
+from tempest.lib.common import profiler
 from tempest.lib.common.utils import test_utils
 from tempest.lib import exceptions
 
@@ -131,8 +132,10 @@
             accept_type = 'json'
         if send_type is None:
             send_type = 'json'
-        return {'Content-Type': 'application/%s' % send_type,
-                'Accept': 'application/%s' % accept_type}
+        headers = {'Content-Type': 'application/%s' % send_type,
+                   'Accept': 'application/%s' % accept_type}
+        headers.update(profiler.serialize_as_http_headers())
+        return headers
 
     def __str__(self):
         STRING_LIMIT = 80
diff --git a/tempest/lib/common/utils/data_utils.py b/tempest/lib/common/utils/data_utils.py
index 3483c51..7f94612 100644
--- a/tempest/lib/common/utils/data_utils.py
+++ b/tempest/lib/common/utils/data_utils.py
@@ -170,7 +170,7 @@
     :rtype: string
     """
     return b''.join([six.int2byte(random.randint(0, 255))
-                    for i in range(size)])
+                     for i in range(size)])
 
 
 # Courtesy of http://stackoverflow.com/a/312464
diff --git a/tempest/lib/services/compute/flavors_client.py b/tempest/lib/services/compute/flavors_client.py
index 2fad0a4..5d2dd46 100644
--- a/tempest/lib/services/compute/flavors_client.py
+++ b/tempest/lib/services/compute/flavors_client.py
@@ -172,7 +172,7 @@
         https://developer.openstack.org/api-ref/compute/#show-an-extra-spec-for-a-flavor
         """
         resp, body = self.get('flavors/%s/os-extra_specs/%s' % (flavor_id,
-                              key))
+                                                                key))
         body = json.loads(body)
         self.validate_response(
             schema_extra_specs.set_get_flavor_extra_specs_key,
diff --git a/tempest/lib/services/compute/servers_client.py b/tempest/lib/services/compute/servers_client.py
index 9eed4b3..18e08cc 100644
--- a/tempest/lib/services/compute/servers_client.py
+++ b/tempest/lib/services/compute/servers_client.py
@@ -636,7 +636,7 @@
     def list_virtual_interfaces(self, server_id):
         """List the virtual interfaces used in an instance."""
         resp, body = self.get('/'.join(['servers', server_id,
-                              'os-virtual-interfaces']))
+                                        'os-virtual-interfaces']))
         body = json.loads(body)
         self.validate_response(schema.list_virtual_interfaces, resp, body)
         return rest_client.ResponseBody(resp, body)
diff --git a/tempest/scenario/manager.py b/tempest/scenario/manager.py
index dbb9acc..d2fd021 100644
--- a/tempest/scenario/manager.py
+++ b/tempest/scenario/manager.py
@@ -125,6 +125,27 @@
         returns a test server. The purpose of this wrapper is to minimize
         the impact on the code of the tests already using this
         function.
+
+        :param **kwargs:
+            See extra parameters below
+
+        :Keyword Arguments:
+            * *vnic_type* (``string``) --
+              used when launching instances with pre-configured ports.
+              Examples:
+                normal: a traditional virtual port that is either attached
+                        to a linux bridge or an openvswitch bridge on a
+                        compute node.
+                direct: an SR-IOV port that is directly attached to a VM
+                macvtap: an SR-IOV port that is attached to a VM via a macvtap
+                         device.
+              Defaults to ``CONF.network.port_vnic_type``.
+            * *port_profile* (``dict``) --
+              This attribute is a dictionary that can be used (with admin
+              credentials) to supply information influencing the binding of
+              the port.
+              example: port_profile = "capabilities:[switchdev]"
+              Defaults to ``CONF.network.port_profile``.
         """
 
         # NOTE(jlanoux): As a first step, ssh checks in the scenario
@@ -143,8 +164,8 @@
         if name is None:
             name = data_utils.rand_name(self.__class__.__name__ + "-server")
 
-        vnic_type = CONF.network.port_vnic_type
-        profile = CONF.network.port_profile
+        vnic_type = kwargs.pop('vnic_type', CONF.network.port_vnic_type)
+        profile = kwargs.pop('port_profile', CONF.network.port_profile)
 
         # If vnic_type or profile are configured create port for
         # every network
@@ -166,7 +187,7 @@
                         clients.security_groups_client.list_security_groups(
                         ).get('security_groups')
                     sec_dict = dict([(s['name'], s['id'])
-                                    for s in security_groups])
+                                     for s in security_groups])
 
                     sec_groups_names = [s['name'] for s in kwargs.pop(
                         'security_groups')]
diff --git a/tempest/scenario/test_minimum_basic.py b/tempest/scenario/test_minimum_basic.py
index 2b35e45..cee543b 100644
--- a/tempest/scenario/test_minimum_basic.py
+++ b/tempest/scenario/test_minimum_basic.py
@@ -48,6 +48,7 @@
     10. Check SSH connection to instance after reboot
 
     """
+
     def nova_show(self, server):
         got_server = (self.servers_client.show_server(server['id'])
                       ['server'])
diff --git a/tempest/scenario/test_volume_boot_pattern.py b/tempest/scenario/test_volume_boot_pattern.py
index e5730e7c..6ed7e30 100644
--- a/tempest/scenario/test_volume_boot_pattern.py
+++ b/tempest/scenario/test_volume_boot_pattern.py
@@ -86,7 +86,6 @@
                           'Cinder volume snapshots are disabled')
     @utils.services('compute', 'volume', 'image')
     def test_volume_boot_pattern(self):
-
         """This test case attempts to reproduce the following steps:
 
         * Create in Cinder some bootable volume importing a Glance image
diff --git a/tempest/test.py b/tempest/test.py
index c3c58dc..85000b6 100644
--- a/tempest/test.py
+++ b/tempest/test.py
@@ -28,6 +28,7 @@
 from tempest.common import utils
 from tempest import config
 from tempest.lib.common import fixed_network
+from tempest.lib.common import profiler
 from tempest.lib.common import validation_resources as vr
 from tempest.lib import decorators
 from tempest.lib import exceptions as lib_exc
@@ -231,6 +232,9 @@
         if CONF.pause_teardown:
             BaseTestCase.insert_pdb_breakpoint()
 
+        if CONF.profiler.key:
+            profiler.disable()
+
     @classmethod
     def insert_pdb_breakpoint(cls):
         """Add pdb breakpoint.
@@ -608,6 +612,8 @@
             self.useFixture(fixtures.LoggerFixture(nuke_handlers=False,
                                                    format=self.log_format,
                                                    level=None))
+        if CONF.profiler.key:
+            profiler.enable(CONF.profiler.key)
 
     @property
     def credentials_provider(self):
diff --git a/tempest/test_discover/plugins.py b/tempest/test_discover/plugins.py
index 9c18052..7a037eb 100644
--- a/tempest/test_discover/plugins.py
+++ b/tempest/test_discover/plugins.py
@@ -179,6 +179,7 @@
     This class is used to manage the lifecycle of external tempest test
     plugins. It provides functions for getting set
     """
+
     def __init__(self):
         self.ext_plugins = stevedore.ExtensionManager(
             'tempest.test_plugins', invoke_on_load=True,
diff --git a/tempest/test_discover/test_discover.py b/tempest/test_discover/test_discover.py
index 330f370..143c6e1 100644
--- a/tempest/test_discover/test_discover.py
+++ b/tempest/test_discover/test_discover.py
@@ -37,7 +37,7 @@
                                            top_level_dir=base_path))
         else:
             suite.addTests(loader.discover(full_test_dir, pattern=pattern,
-                           top_level_dir=base_path))
+                                           top_level_dir=base_path))
 
     plugin_load_tests = ext_plugins.get_plugin_load_tests_tuple()
     if not plugin_load_tests:
diff --git a/tempest/tests/cmd/test_run.py b/tempest/tests/cmd/test_run.py
index 00f8bc5..0e00d94 100644
--- a/tempest/tests/cmd/test_run.py
+++ b/tempest/tests/cmd/test_run.py
@@ -29,6 +29,10 @@
 from tempest.lib.common.utils import data_utils
 from tempest.tests import base
 
+if six.PY2:
+    # Python 2 has not FileNotFoundError exception
+    FileNotFoundError = IOError
+
 DEVNULL = open(os.devnull, 'wb')
 atexit.register(DEVNULL.close)
 
@@ -244,14 +248,22 @@
         # getting set in os environment when some data has passed to
         # set the environment.
 
-        self.run_cmd._set_env("/fakedir/fakefile")
-        self.assertEqual("/fakedir/fakefile", CONF._path)
-        self.assertIn('TEMPEST_CONFIG_DIR', os.environ)
-        self.assertEqual("/fakedir/fakefile",
-                         os.path.join(os.environ['TEMPEST_CONFIG_DIR'],
-                                      os.environ['TEMPEST_CONFIG']))
+        _, path = tempfile.mkstemp()
+        self.addCleanup(os.remove, path)
 
-    def test_tempest_run_set_config_no_path(self):
+        self.run_cmd._set_env(path)
+        self.assertEqual(path, CONF._path)
+        self.assertIn('TEMPEST_CONFIG_DIR', os.environ)
+        self.assertEqual(path, os.path.join(os.environ['TEMPEST_CONFIG_DIR'],
+                                            os.environ['TEMPEST_CONFIG']))
+
+    def test_tempest_run_set_config_no_exist_path(self):
+        path = "fake/path"
+        self.assertRaisesRegex(FileNotFoundError,
+                               'Config file: .* doesn\'t exist',
+                               self.run_cmd._set_env, path)
+
+    def test_tempest_run_no_config_path(self):
         # Note: (mbindlish) This test is created for the bug id: 1783751
         # Checking TEMPEST_CONFIG_DIR and TEMPEST_CONFIG should have no value
         # in os environment when no data has passed to set the environment.
@@ -313,13 +325,15 @@
 
     def test_config_file_specified(self):
         self._setup_test_dirs()
+        _, path = tempfile.mkstemp()
+        self.addCleanup(os.remove, path)
         tempest_run = run.TempestRun(app=mock.Mock(), app_args=mock.Mock())
         parsed_args = mock.Mock()
 
         parsed_args.workspace = None
         parsed_args.state = None
         parsed_args.list_tests = False
-        parsed_args.config_file = '.stestr.conf'
+        parsed_args.config_file = path
 
         with mock.patch('stestr.commands.run_command') as m:
             m.return_value = 0
@@ -341,13 +355,15 @@
 
     def test_config_file_workspace_registered(self):
         self._setup_test_dirs()
+        _, path = tempfile.mkstemp()
+        self.addCleanup(os.remove, path)
         tempest_run = run.TempestRun(app=mock.Mock(), app_args=mock.Mock())
         parsed_args = mock.Mock()
         parsed_args.workspace = self.name
         parsed_args.workspace_path = self.store_file
         parsed_args.state = None
         parsed_args.list_tests = False
-        parsed_args.config_file = '.stestr.conf'
+        parsed_args.config_file = path
 
         with mock.patch('stestr.commands.run_command') as m:
             m.return_value = 0
@@ -406,13 +422,15 @@
     @mock.patch('tempest.cmd.run.TempestRun._init_state')
     def test_no_workspace_config_file_state_true(self, mock_init_state):
         self._setup_test_dirs()
+        _, path = tempfile.mkstemp()
+        self.addCleanup(os.remove, path)
         tempest_run = run.TempestRun(app=mock.Mock(), app_args=mock.Mock())
         parsed_args = mock.Mock()
         parsed_args.workspace = None
         parsed_args.workspace_path = self.store_file
         parsed_args.state = True
         parsed_args.list_tests = False
-        parsed_args.config_file = '.stestr.conf'
+        parsed_args.config_file = path
 
         with mock.patch('stestr.commands.run_command') as m:
             m.return_value = 0
diff --git a/tempest/tests/cmd/test_workspace.py b/tempest/tests/cmd/test_workspace.py
index b4f6c5f..7a6b576 100644
--- a/tempest/tests/cmd/test_workspace.py
+++ b/tempest/tests/cmd/test_workspace.py
@@ -48,7 +48,7 @@
         stdout, stderr = process.communicate()
         return_code = process.returncode
         msg = ("%s failed with:\nstdout: %s\nstderr: %s" % (' '.join(cmd),
-               stdout, stderr))
+                                                            stdout, stderr))
         self.assertEqual(return_code, expected, msg)
 
     def test_run_workspace_list(self):
diff --git a/tempest/tests/lib/common/test_dynamic_creds.py b/tempest/tests/lib/common/test_dynamic_creds.py
index ebcf5d1..4723458 100644
--- a/tempest/tests/lib/common/test_dynamic_creds.py
+++ b/tempest/tests/lib/common/test_dynamic_creds.py
@@ -109,8 +109,8 @@
             return_value=(rest_client.ResponseBody
                           (200,
                            {'roles': [{'id': id, 'name': name},
-                            {'id': '1', 'name': 'FakeRole'},
-                            {'id': '2', 'name': 'Member'}]}))))
+                                      {'id': '1', 'name': 'FakeRole'},
+                                      {'id': '2', 'name': 'Member'}]}))))
         return roles_fix
 
     def _mock_list_2_roles(self):
@@ -120,8 +120,8 @@
             return_value=(rest_client.ResponseBody
                           (200,
                            {'roles': [{'id': '1234', 'name': 'role1'},
-                            {'id': '1', 'name': 'FakeRole'},
-                            {'id': '12345', 'name': 'role2'}]}))))
+                                      {'id': '1', 'name': 'FakeRole'},
+                                      {'id': '12345', 'name': 'role2'}]}))))
         return roles_fix
 
     def _mock_assign_user_role(self):
diff --git a/tempest/tests/lib/common/test_profiler.py b/tempest/tests/lib/common/test_profiler.py
new file mode 100644
index 0000000..59fa0364
--- /dev/null
+++ b/tempest/tests/lib/common/test_profiler.py
@@ -0,0 +1,63 @@
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+import mock
+import testtools
+
+from tempest.lib.common import profiler
+
+
+class TestProfiler(testtools.TestCase):
+
+    def test_serialize(self):
+        key = 'SECRET_KEY'
+        pm = {'key': key, 'uuid': 'ID'}
+
+        with mock.patch('tempest.lib.common.profiler._profiler', pm):
+            with mock.patch('json.dumps') as jdm:
+                jdm.return_value = '{"base_id": "ID", "parent_id": "ID"}'
+
+                expected = {
+                    'X-Trace-HMAC':
+                        '887292df9f13b8b5ecd6bbbd2e16bfaaa4d914b0',
+                    'X-Trace-Info':
+                        b'eyJiYXNlX2lkIjogIklEIiwgInBhcmVudF9pZCI6ICJJRCJ9'
+                }
+
+                self.assertEqual(expected,
+                                 profiler.serialize_as_http_headers())
+
+    def test_profiler_lifecycle(self):
+        key = 'SECRET_KEY'
+        uuid = 'ID'
+
+        self.assertEqual({}, profiler._profiler)
+
+        profiler.enable(key, uuid)
+        self.assertEqual({'key': key, 'uuid': uuid}, profiler._profiler)
+
+        profiler.disable()
+        self.assertEqual({}, profiler._profiler)
+
+    @mock.patch('oslo_utils.uuidutils.generate_uuid')
+    def test_profiler_lifecycle_generate_trace_id(self, generate_uuid_mock):
+        key = 'SECRET_KEY'
+        uuid = 'ID'
+        generate_uuid_mock.return_value = uuid
+
+        self.assertEqual({}, profiler._profiler)
+
+        profiler.enable(key)
+        self.assertEqual({'key': key, 'uuid': uuid}, profiler._profiler)
+
+        profiler.disable()
+        self.assertEqual({}, profiler._profiler)
diff --git a/tempest/tests/lib/services/compute/test_images_client.py b/tempest/tests/lib/services/compute/test_images_client.py
index c2c3b76..d1500e5 100644
--- a/tempest/tests/lib/services/compute/test_images_client.py
+++ b/tempest/tests/lib/services/compute/test_images_client.py
@@ -186,15 +186,19 @@
     def _test_resource_deleted(self, bytes_body=False):
         params = {"id": self.FAKE_IMAGE_ID}
         expected_op = self.FAKE_IMAGE_DATA['show']
-        self.useFixture(fixtures.MockPatch('tempest.lib.services.compute'
-                        '.images_client.ImagesClient.show_image',
-                                           side_effect=lib_exc.NotFound))
+        self.useFixture(
+            fixtures.MockPatch(
+                'tempest.lib.services.compute'
+                '.images_client.ImagesClient.show_image',
+                side_effect=lib_exc.NotFound))
         self.assertEqual(True, self.client.is_resource_deleted(**params))
         tempdata = copy.deepcopy(self.FAKE_IMAGE_DATA['show'])
         tempdata['image']['id'] = None
-        self.useFixture(fixtures.MockPatch('tempest.lib.services.compute'
-                        '.images_client.ImagesClient.show_image',
-                                           return_value=expected_op))
+        self.useFixture(
+            fixtures.MockPatch(
+                'tempest.lib.services.compute'
+                '.images_client.ImagesClient.show_image',
+                return_value=expected_op))
         self.assertEqual(False, self.client.is_resource_deleted(**params))
 
     def test_list_images_with_str_body(self):
diff --git a/tempest/tests/lib/services/volume/v3/test_scheduler_stats_client.py b/tempest/tests/lib/services/volume/v3/test_scheduler_stats_client.py
index e0f5566..84c7589 100644
--- a/tempest/tests/lib/services/volume/v3/test_scheduler_stats_client.py
+++ b/tempest/tests/lib/services/volume/v3/test_scheduler_stats_client.py
@@ -62,7 +62,7 @@
             resp_body = self.FAKE_POOLS_LIST
         else:
             resp_body = {'pools': [{'name': pool['name']}
-                         for pool in self.FAKE_POOLS_LIST['pools']]}
+                                   for pool in self.FAKE_POOLS_LIST['pools']]}
         self.check_service_client_function(
             self.client.list_pools,
             'tempest.lib.common.rest_client.RestClient.get',
diff --git a/tempest/tests/test_hacking.py b/tempest/tests/test_hacking.py
index 9534ce8..83c1abb 100644
--- a/tempest/tests/test_hacking.py
+++ b/tempest/tests/test_hacking.py
@@ -48,6 +48,7 @@
     just assertTrue if the check is expected to fail and assertFalse if it
     should pass.
     """
+
     def test_no_setup_teardown_class_for_tests(self):
         self.assertTrue(checks.no_setup_teardown_class_for_tests(
             "  def setUpClass(cls):", './tempest/tests/fake_test.py'))
diff --git a/tools/format.sh b/tools/format.sh
new file mode 100755
index 0000000..adffb8c
--- /dev/null
+++ b/tools/format.sh
@@ -0,0 +1,5 @@
+#!/bin/bash
+cd $(dirname "$(readlink -f "$0")")
+
+autopep8 --exit-code --max-line-length=79 --experimental --in-place -r ../tempest ../setup.py && echo Formatting was not needed. >&2
+
diff --git a/tools/generate-tempest-plugins-list.sh b/tools/generate-tempest-plugins-list.sh
index 111c9ce..17a4059 100755
--- a/tools/generate-tempest-plugins-list.sh
+++ b/tools/generate-tempest-plugins-list.sh
@@ -69,7 +69,7 @@
 i=0
 for plugin in ${sorted_plugins}; do
     i=$((i+1))
-    giturl="git://git.openstack.org/openstack/${plugin}"
+    giturl="https://git.openstack.org/openstack/${plugin}"
     gitlink="https://git.openstack.org/cgit/openstack/${plugin}"
     printf "%-3s %-${name_col_len}s %s\n" "$i" "${plugin}" "\`${giturl} <${gitlink}>\`__"
 done
diff --git a/tools/tempest-plugin-sanity.sh b/tools/tempest-plugin-sanity.sh
index 16e7b8c..703dce2 100644
--- a/tools/tempest-plugin-sanity.sh
+++ b/tools/tempest-plugin-sanity.sh
@@ -81,11 +81,11 @@
 function clone_project() {
     if [ -e /usr/zuul-env/bin/zuul-cloner ]; then
         /usr/zuul-env/bin/zuul-cloner --cache-dir /opt/git \
-        git://git.openstack.org \
+        https://git.openstack.org \
         openstack/"$1"
 
     elif [ -e /usr/bin/git ]; then
-        /usr/bin/git clone git://git.openstack.org/openstack/"$1" \
+        /usr/bin/git clone https://git.openstack.org/openstack/"$1" \
         openstack/"$1"
 
     fi
diff --git a/tox.ini b/tox.ini
index b565507..433f168 100644
--- a/tox.ini
+++ b/tox.ini
@@ -197,11 +197,21 @@
 whitelist_externals = rm
 
 [testenv:pep8]
+deps =
+    -r test-requirements.txt
+    autopep8
 basepython = python3
 commands =
+    autopep8 --exit-code --max-line-length=79 --experimental --diff -r tempest setup.py
     flake8 {posargs}
     check-uuid
 
+[testenv:autopep8]
+deps = autopep8
+basepython = python3
+commands =
+    autopep8 --max-line-length=79  --experimental --in-place -r tempest setup.py
+
 [testenv:uuidgen]
 commands =
     check-uuid --fix