diff --git a/.zuul.yaml b/.zuul.yaml
index 2619ed7..1eab464 100644
--- a/.zuul.yaml
+++ b/.zuul.yaml
@@ -1,7 +1,7 @@
 - job:
     name: patrole-base
     parent: devstack-tempest
-    description: Patrole base job for admin and Member roles.
+    description: Patrole base job for admin and member roles.
     required-projects:
       - name: openstack/tempest
       - name: openstack/patrole
@@ -54,7 +54,7 @@
 - job:
     name: patrole-member
     parent: patrole-base
-    description: Patrole job for Member role.
+    description: Patrole job for member role.
     # This currently works from stable/pike onward.
     branches:
       - master
@@ -62,7 +62,7 @@
       - stable/pike
     vars:
       devstack_localrc:
-        RBAC_TEST_ROLE: Member
+        RBAC_TEST_ROLE: member
 
 - job:
     name: patrole-member-queens
@@ -93,12 +93,12 @@
 - job:
     name: patrole-py35-member
     parent: patrole-base
-    description: Patrole py3 job for Member role.
+    description: Patrole py35 job for member role.
     vars:
       devstack_localrc:
-        # Use Member for py3 because arguably negative testing is more
+        # Use member for py35 because arguably negative testing is more
         # important than admin, which is already covered by patrole-admin job.
-        RBAC_TEST_ROLE: Member
+        RBAC_TEST_ROLE: member
         USE_PYTHON3: true
       devstack_services:
         s-account: false
diff --git a/devstack/plugin.sh b/devstack/plugin.sh
index d56c963..bd0068b 100644
--- a/devstack/plugin.sh
+++ b/devstack/plugin.sh
@@ -13,16 +13,13 @@
 function install_patrole_tempest_plugin {
     setup_package $PATROLE_DIR -e
 
-    if [[ "$RBAC_TEST_ROLE" == "member" ]]; then
-        RBAC_TEST_ROLE="Member"
-    fi
-
-    iniset $TEMPEST_CONFIG patrole enable_rbac True
-    iniset $TEMPEST_CONFIG patrole rbac_test_role $RBAC_TEST_ROLE
-
     if [[ ${DEVSTACK_SERIES} == 'pike' ]]; then
+        if [[ "$RBAC_TEST_ROLE" == "member" ]]; then
+            RBAC_TEST_ROLE="Member"
+        fi
+
         # Policies used by Patrole testing that were changed in a backwards-incompatible way.
-        # TODO(fmontei): Remove these once stable/pike becomes EOL.
+        # TODO(felipemonteiro): Remove these once stable/pike becomes EOL.
         iniset $TEMPEST_CONFIG policy-feature-enabled create_port_fixed_ips_ip_address_policy False
         iniset $TEMPEST_CONFIG policy-feature-enabled update_port_fixed_ips_ip_address_policy False
         iniset $TEMPEST_CONFIG policy-feature-enabled limits_extension_used_limits_policy False
@@ -30,6 +27,15 @@
         iniset $TEMPEST_CONFIG policy-feature-enabled volume_extension_volume_actions_reserve_policy False
         iniset $TEMPEST_CONFIG policy-feature-enabled volume_extension_volume_actions_unreserve_policy False
     fi
+
+    if [[ ${DEVSTACK_SERIES} == 'queens' ]]; then
+        if [[ "$RBAC_TEST_ROLE" == "member" ]]; then
+            RBAC_TEST_ROLE="Member"
+        fi
+    fi
+
+    iniset $TEMPEST_CONFIG patrole enable_rbac True
+    iniset $TEMPEST_CONFIG patrole rbac_test_role $RBAC_TEST_ROLE
 }
 
 if is_service_enabled tempest; then
diff --git a/patrole_tempest_plugin/rbac_utils.py b/patrole_tempest_plugin/rbac_utils.py
index 2ef88ca..6c40aa1 100644
--- a/patrole_tempest_plugin/rbac_utils.py
+++ b/patrole_tempest_plugin/rbac_utils.py
@@ -147,18 +147,25 @@
             test_obj.os_primary.auth_provider.set_auth()
 
     def _get_roles_by_name(self):
-        available_roles = self.admin_roles_client.list_roles()
-        admin_role_id = rbac_role_id = None
+        available_roles = self.admin_roles_client.list_roles()['roles']
+        role_map = {r['name']: r['id'] for r in available_roles}
+        LOG.debug('Available roles: %s', list(role_map.keys()))
 
-        for role in available_roles['roles']:
-            if role['name'] == CONF.patrole.rbac_test_role:
-                rbac_role_id = role['id']
-            if role['name'] == CONF.identity.admin_role:
-                admin_role_id = role['id']
+        admin_role_id = role_map.get(CONF.identity.admin_role)
+        rbac_role_id = role_map.get(CONF.patrole.rbac_test_role)
 
         if not all([admin_role_id, rbac_role_id]):
-            msg = ("Roles defined by `[patrole] rbac_test_role` and "
-                   "`[identity] admin_role` must be defined in the system.")
+            missing_roles = []
+            msg = ("Could not find `[patrole] rbac_test_role` or "
+                   "`[identity] admin_role`, both of which are required for "
+                   "RBAC testing.")
+            if not admin_role_id:
+                missing_roles.append(CONF.identity.admin_role)
+            if not rbac_role_id:
+                missing_roles.append(CONF.patrole.rbac_test_role)
+            msg += " Following roles were not found: %s." % (
+                ", ".join(missing_roles))
+            msg += " Available roles: %s." % ", ".join(list(role_map.keys()))
             raise rbac_exceptions.RbacResourceSetupFailed(msg)
 
         self.admin_role_id = admin_role_id
@@ -226,4 +233,6 @@
 
     :returns: True if ``rbac_test_role`` is the admin role.
     """
+    # TODO(felipemonteiro): Make this more robust via a context is admin
+    # lookup.
     return CONF.patrole.rbac_test_role == CONF.identity.admin_role
diff --git a/patrole_tempest_plugin/tests/unit/test_rbac_utils.py b/patrole_tempest_plugin/tests/unit/test_rbac_utils.py
index 5e730d3..4937318 100644
--- a/patrole_tempest_plugin/tests/unit/test_rbac_utils.py
+++ b/patrole_tempest_plugin/tests/unit/test_rbac_utils.py
@@ -36,17 +36,15 @@
 
     def test_override_role_with_missing_admin_role(self):
         self.rbac_utils.set_roles('member')
-        error_re = (
-            'Roles defined by `\[patrole\] rbac_test_role` and `\[identity\] '
-            'admin_role` must be defined in the system.')
+        error_re = (".*Following roles were not found: admin. Available "
+                    "roles: member.")
         self.assertRaisesRegex(rbac_exceptions.RbacResourceSetupFailed,
                                error_re, self.rbac_utils.override_role)
 
     def test_override_role_with_missing_rbac_role(self):
         self.rbac_utils.set_roles('admin')
-        error_re = (
-            'Roles defined by `\[patrole\] rbac_test_role` and `\[identity\] '
-            'admin_role` must be defined in the system.')
+        error_re = (".*Following roles were not found: member. Available "
+                    "roles: admin.")
         self.assertRaisesRegex(rbac_exceptions.RbacResourceSetupFailed,
                                error_re, self.rbac_utils.override_role)