Make use of Gerrit CLI to retrieve group UUID
Added the following features:
* ability to retrieve groups by using gerrit CLI added.
Old mechanism of retrieving new group UUIDs by using SQL-query
to database is saved for backwards compatibility.
* ability to run Jeepyb on the same host where Gerrit installed
or on a remote host. This mode can be controlled by using
'jeepyb-run-local' option in the config. Defaults to 'True'.
Change-Id: Ic2aef553a0bd5dc67dc482e6b3640ec11900c5ca
Depends-On: I851ab29999c0d059c9cb71bd38a821c035ae98f7
diff --git a/jeepyb/cmd/manage_projects.py b/jeepyb/cmd/manage_projects.py
index 2c19c3a..c6b708e 100644
--- a/jeepyb/cmd/manage_projects.py
+++ b/jeepyb/cmd/manage_projects.py
@@ -51,6 +51,7 @@
# project: OTHER_PROJECT_NAME
import argparse
+import csv
import ConfigParser
import glob
import hashlib
@@ -68,6 +69,8 @@
import jeepyb.log as l
import jeepyb.utils as u
+from distutils.version import LooseVersion
+
registry = u.ProjectsRegistry()
log = logging.getLogger("manage_projects")
@@ -83,6 +86,8 @@
'Change Owner': 'global:Change-Owner',
}
+_gerrit_groups = None
+
class FetchConfigException(Exception):
pass
@@ -176,34 +181,70 @@
return True
-def _get_group_uuid(group, retries=10):
+def _get_group_uuid(group, gerrit, version, run_local=True, retries=10):
"""
+ Retrieve group either directly from the DB or through gerritlib
+ which uses the ssh command ls-groups with `--verbose` argument.
+
+ Use of the DB directly is enabled for Gerrit < 2.6 or if run_local
+ is enabled.
+
Gerrit keeps internal user groups in the DB while it keeps systems
- groups in All-Projects groups file (in refs/meta/config). This
- will only get the UUIDs for internal user groups.
+ groups in All-Projects groups file (in refs/meta/config). If using
+ the DB directly this will only get the UUIDs for internal user groups,
+ while use of `gerritlib.listGroups(verbose=True)` will return UUIDs
+ for system groups as well.
Note: 'Administrators', 'Non-Interactive Users' and all other custom
groups in Gerrit are defined as internal user groups.
- Wait for up to 10 seconds for the group to be created in the DB.
+ Direct DB access will wait for up to 10 seconds for the group to be
+ created, while use of gerritlib.listGroups() will include any updates
+ contained within the cache cache and so can return immediately.
"""
- query = "SELECT group_uuid FROM account_groups WHERE name = %s"
- con = jeepyb.gerritdb.connect()
- for x in range(retries):
- cursor = con.cursor()
- cursor.execute(query, (group,))
- data = cursor.fetchone()
- cursor.close()
- con.commit()
- if data:
- return data[0]
- if retries > 1:
- time.sleep(1)
+
+ if LooseVersion(version) <= LooseVersion('2.6') or run_local:
+ query = "SELECT group_uuid FROM account_groups WHERE name = %s"
+ con = jeepyb.gerritdb.connect()
+ for x in range(retries):
+ cursor = con.cursor()
+ cursor.execute(query, (group,))
+ data = cursor.fetchone()
+ cursor.close()
+ con.commit()
+ if data:
+ return data[0]
+ if retries > 1:
+ time.sleep(1)
+ else:
+ global _gerrit_groups
+ if _gerrit_groups is None:
+ _gerrit_groups = {
+ group[0]: group[1]
+ for group in csv.reader(gerrit.listGroups(verbose=True),
+ delimiter='\t')
+ }
+ else:
+ if group not in _gerrit_groups:
+ try:
+ # Relying on the outer Exception handler to deal with
+ # the StopIteration when listGroup returns nothing
+ # (for a system group).
+ gdata = next(csv.reader(
+ gerrit.listGroup(group, verbose=True),
+ delimiter='\t'))
+ _gerrit_groups[group] = gdata[1]
+ except Exception:
+ # group not found will also trigger exception
+ pass
+ return _gerrit_groups.get(group, None)
return None
-def get_group_uuid(gerrit, group):
- uuid = _get_group_uuid(group, retries=1)
+def get_group_uuid(gerrit, group, run_local=True):
+ version = ''.join(gerrit.getVersion().split('.')[:2])
+
+ uuid = _get_group_uuid(group, gerrit, version, run_local, retries=1)
if uuid:
return uuid
if group in GERRIT_SYSTEM_GROUPS:
@@ -214,13 +255,13 @@
# Gerrit now adds creating user to groups. We don't want that.
gerrit.removeMember(group, gerrit.username)
break
- uuid = _get_group_uuid(group)
+ uuid = _get_group_uuid(group, gerrit, version, run_local)
if uuid:
return uuid
return None
-def create_groups_file(project, gerrit, repo_path):
+def create_groups_file(project, gerrit, repo_path, run_local=True):
acl_config = os.path.join(repo_path, "project.config")
group_file = os.path.join(repo_path, "groups")
uuids = {}
@@ -230,7 +271,7 @@
group = r.group(1)
if group in uuids.keys():
continue
- uuid = get_group_uuid(gerrit, group)
+ uuid = get_group_uuid(gerrit, group, run_local)
if uuid:
uuids[group] = uuid
else:
@@ -361,7 +402,8 @@
def process_acls(acl_config, project, ACL_DIR, section,
- remote_url, repo_path, ssh_env, gerrit, GERRIT_GITID):
+ remote_url, repo_path, ssh_env, gerrit, GERRIT_GITID,
+ run_local=True):
if not os.path.isfile(acl_config):
return
try:
@@ -369,7 +411,7 @@
if not copy_acl_config(project, repo_path, acl_config):
# nothing was copied, so we're done
return
- create_groups_file(project, gerrit, repo_path)
+ create_groups_file(project, gerrit, repo_path, run_local)
push_acl_config(project, remote_url, repo_path,
GERRIT_GITID, ssh_env)
except Exception:
@@ -423,6 +465,8 @@
LOCAL_GIT_DIR = registry.get_defaults('local-git-dir', '/var/lib/git')
JEEPYB_CACHE_DIR = registry.get_defaults('jeepyb-cache-dir',
'/var/lib/jeepyb')
+ JEEPYB_RUN_LOCAL = registry.get_defaults('jeepyb-run-local',
+ True)
ACL_DIR = registry.get_defaults('acl-dir')
GERRIT_HOST = registry.get_defaults('gerrit-host')
GITREVIEW_GERRIT_HOST = registry.get_defaults(
@@ -552,7 +596,7 @@
process_acls(
acl_config, project, ACL_DIR, section,
remote_url, repo_path, ssh_env, gerrit,
- GERRIT_GITID)
+ GERRIT_GITID, JEEPYB_RUN_LOCAL)
project_cache[project]['acl-sha'] = acl_sha
else:
log.info("%s has matching sha, skipping ACLs",