diff --git a/tempest/lib/common/preprov_creds.py b/tempest/lib/common/preprov_creds.py
index ebc435c..5f8d68f 100644
--- a/tempest/lib/common/preprov_creds.py
+++ b/tempest/lib/common/preprov_creds.py
@@ -94,6 +94,8 @@
             object_storage_reseller_admin_role)
         self.accounts_dir = accounts_lock_dir
         self._creds = {}
+        self._used_projects_file = os.path.join(self.accounts_dir,
+                                                'used_projects')
 
     @classmethod
     def _append_role(cls, role, account_hash, hash_dict):
@@ -180,41 +182,68 @@
         return self.is_multi_user()
 
     def _create_hash_file(self, hash_string):
-        path = os.path.join(os.path.join(self.accounts_dir, hash_string))
+        path = os.path.join(self.accounts_dir, hash_string)
         if not os.path.isfile(path):
             with open(path, 'w') as fd:
                 fd.write(self.name)
             return True
         return False
 
+    def _process_project(self, hash_, used_projects=None, use=True):
+        if used_projects is None:
+            used_projects = self._get_used_projects()
+        if 'tenant_name' in self.hash_dict['creds'][hash_]:
+            project = self.hash_dict['creds'][hash_]['tenant_name']
+        else:
+            project = self.hash_dict['creds'][hash_]['project_name']
+        method = 'append' if use else 'remove'
+        getattr(used_projects, method)(project)
+        with open(self._used_projects_file, 'w') as file:
+            file.write('\n'.join(used_projects) + '\n')
+
+    def _get_used_projects(self):
+        used_projects = []
+        try:
+            with open(self._used_projects_file) as file:
+                for line in file:
+                    line = line.strip()
+                    if line:
+                        used_projects.append(line)
+        except FileNotFoundError:
+            pass
+        return used_projects
+
     @lockutils.synchronized('test_accounts_io', external=True)
     def _get_free_hash(self, hashes):
+        used_projects = self._get_used_projects()
+        hashes = self._exclude_used_projects(hashes, used_projects)
         # Cast as a list because in some edge cases a set will be passed in
         hashes = list(hashes)
         if not os.path.isdir(self.accounts_dir):
             os.mkdir(self.accounts_dir)
             # Create File from first hash (since none are in use)
             self._create_hash_file(hashes[0])
+            self._process_project(hashes[0], used_projects)
             return hashes[0]
         names = []
         for _hash in hashes:
             res = self._create_hash_file(_hash)
             if res:
+                self._process_project(_hash, used_projects)
                 return _hash
             else:
-                path = os.path.join(os.path.join(self.accounts_dir,
-                                                 _hash))
+                path = os.path.join(self.accounts_dir, _hash)
                 with open(path, 'r') as fd:
                     names.append(fd.read())
         msg = ('Insufficient number of users provided. %s have allocated all '
                'the credentials for this allocation request' % ','.join(names))
         raise lib_exc.InvalidCredentials(msg)
 
-    def _exclude_used_projects(self, hashes):
+    def _exclude_used_projects(self, hashes, used_projects):
         excluded_accounts = []
-        for project in [cred.tenant_name for cred in self._creds.values()]:
+        for project in used_projects:
             excluded_accounts.extend(self.hash_dict['projects'][project])
-        return hashes - set(excluded_accounts)
+        return set(hashes) - set(excluded_accounts)
 
     def _get_match_hash_list(self, roles=None):
         hashes = []
@@ -236,7 +265,6 @@
             hashes = temp_list
         else:
             hashes = self.hash_dict['creds'].keys()
-        hashes = self._exclude_used_projects(hashes)
         # NOTE(mtreinish): admin is a special case because of the increased
         # privilege set which could potentially cause issues on tests where
         # that is not expected. So unless the admin role isn't specified do
@@ -268,6 +296,7 @@
 
     @lockutils.synchronized('test_accounts_io', external=True)
     def remove_hash(self, hash_string):
+        self._process_project(hash_string, use=False)
         hash_path = os.path.join(self.accounts_dir, hash_string)
         if not os.path.isfile(hash_path):
             LOG.warning('Expected an account lock file %s to remove, but '
@@ -312,7 +341,7 @@
         self._creds['alt'] = net_creds
         return net_creds
 
-    def get_creds_by_roles(self, roles, force_new=False):
+    def get_creds_by_roles(self, roles, force_new=True):
         roles = list(set(roles))
         exist_creds = self._creds.get(six.text_type(roles).encode(
             'utf-8'), None)
diff --git a/tempest/tests/lib/common/test_preprov_creds.py b/tempest/tests/lib/common/test_preprov_creds.py
index 579363e..cc5b4ce 100644
--- a/tempest/tests/lib/common/test_preprov_creds.py
+++ b/tempest/tests/lib/common/test_preprov_creds.py
@@ -114,6 +114,17 @@
             hash_list.append(temp_hash)
         return hash_list
 
+    def _remove_hash(self, hash_list, hash_index):
+        test_account_class = preprov_creds.PreProvisionedCredentialProvider(
+            **self.fixed_params)
+        remove_mock = self.useFixture(fixtures.MockPatch('os.remove'))
+        rmdir_mock = self.useFixture(fixtures.MockPatch('os.rmdir'))
+        test_account_class.remove_hash(hash_list[hash_index])
+        hash_path = os.path.join(self.fixed_params['accounts_lock_dir'],
+                                 hash_list[hash_index])
+        return {'remove_mock': remove_mock, 'hash_path': hash_path,
+                'rmdir_mock': rmdir_mock}
+
     def test_get_hash(self):
         # Test with all accounts to make sure we try all combinations
         # and hide no race conditions
@@ -182,8 +193,12 @@
                         create=True) as open_mock:
             test_account_class._get_free_hash(hash_list)
             lock_path = os.path.join(self.fixed_params['accounts_lock_dir'],
-                                     hash_list[0])
-            open_mock.assert_called_once_with(lock_path, 'w')
+                                     list(set(hash_list))[0])
+            accounts_lock_dir_calls = \
+                [mock.call(test_account_class._used_projects_file),
+                 mock.call(lock_path, 'w'),
+                 mock.call(test_account_class._used_projects_file, 'w')]
+            open_mock.assert_has_calls(accounts_lock_dir_calls, any_order=True)
         mkdir_path = os.path.join(self.fixed_params['accounts_lock_dir'])
         mkdir_mock.mock.assert_called_once_with(mkdir_path)
 
@@ -225,43 +240,33 @@
                                      hash_list[3])
             open_mock.assert_has_calls([mock.call(lock_path, 'w')])
 
-    @mock.patch('oslo_concurrency.lockutils.lock')
-    def test_remove_hash_last_account(self, lock_mock):
+    @mock.patch('tempest.lib.common.preprov_creds.'
+                'PreProvisionedCredentialProvider._process_project')
+    def test_remove_hash_last_account(self, _process_project_mock):
         hash_list = self._get_hash_list(self.test_accounts)
         # Pretend the pseudo-lock is there
         self.useFixture(
             fixtures.MockPatch('os.path.isfile', return_value=True))
         # Pretend the lock dir is empty
         self.useFixture(fixtures.MockPatch('os.listdir', return_value=[]))
-        test_account_class = preprov_creds.PreProvisionedCredentialProvider(
-            **self.fixed_params)
-        remove_mock = self.useFixture(fixtures.MockPatch('os.remove'))
-        rmdir_mock = self.useFixture(fixtures.MockPatch('os.rmdir'))
-        test_account_class.remove_hash(hash_list[2])
-        hash_path = os.path.join(self.fixed_params['accounts_lock_dir'],
-                                 hash_list[2])
+        result = self._remove_hash(hash_list, 2)
         lock_path = self.fixed_params['accounts_lock_dir']
-        remove_mock.mock.assert_called_once_with(hash_path)
-        rmdir_mock.mock.assert_called_once_with(lock_path)
+        result['remove_mock'].mock.assert_called_once_with(result['hash_path'])
+        result['rmdir_mock'].mock.assert_called_once_with(lock_path)
 
-    @mock.patch('oslo_concurrency.lockutils.lock')
-    def test_remove_hash_not_last_account(self, lock_mock):
+    @mock.patch('tempest.lib.common.preprov_creds.'
+                'PreProvisionedCredentialProvider._process_project')
+    def test_remove_hash_not_last_account(self, _process_project_mock):
         hash_list = self._get_hash_list(self.test_accounts)
         # Pretend the pseudo-lock is there
         self.useFixture(fixtures.MockPatch(
             'os.path.isfile', return_value=True))
-        # Pretend the lock dir is empty
+        # Pretend the lock dir is not empty
         self.useFixture(fixtures.MockPatch('os.listdir', return_value=[
             hash_list[1], hash_list[4]]))
-        test_account_class = preprov_creds.PreProvisionedCredentialProvider(
-            **self.fixed_params)
-        remove_mock = self.useFixture(fixtures.MockPatch('os.remove'))
-        rmdir_mock = self.useFixture(fixtures.MockPatch('os.rmdir'))
-        test_account_class.remove_hash(hash_list[2])
-        hash_path = os.path.join(self.fixed_params['accounts_lock_dir'],
-                                 hash_list[2])
-        remove_mock.mock.assert_called_once_with(hash_path)
-        rmdir_mock.mock.assert_not_called()
+        result = self._remove_hash(hash_list, 2)
+        result['remove_mock'].mock.assert_called_once_with(result['hash_path'])
+        result['rmdir_mock'].mock.assert_not_called()
 
     def test_is_multi_user(self):
         test_accounts_class = preprov_creds.PreProvisionedCredentialProvider(