Merge "Fix incompatibility with gerrit 2.8"
diff --git a/jeepyb/cmd/close_pull_requests.py b/jeepyb/cmd/close_pull_requests.py
index f3c4f4f..19d2d25 100644
--- a/jeepyb/cmd/close_pull_requests.py
+++ b/jeepyb/cmd/close_pull_requests.py
@@ -59,13 +59,18 @@
PROJECTS_YAML = os.environ.get('PROJECTS_YAML',
'/home/gerrit2/projects.yaml')
+ PROJECTS_INI = os.environ.get('PROJECTS_INI',
+ '/home/gerrit2/projects.ini')
GITHUB_SECURE_CONFIG = os.environ.get('GITHUB_SECURE_CONFIG',
'/etc/github/github.secure.config')
secure_config = ConfigParser.ConfigParser()
secure_config.read(GITHUB_SECURE_CONFIG)
- (defaults, config) = [config for config in
- yaml.load_all(open(PROJECTS_YAML))]
+ yaml_docs = [config for config in yaml.load_all(open(PROJECTS_YAML))]
+ if os.path.exists(PROJECTS_INI):
+ config = yaml_docs[0]
+ else:
+ config = yaml_docs[1]
if secure_config.has_option("github", "oauth_token"):
ghub = github.Github(secure_config.get("github", "oauth_token"))
diff --git a/jeepyb/cmd/create_cgitrepos.py b/jeepyb/cmd/create_cgitrepos.py
index bda452f..1572a44 100644
--- a/jeepyb/cmd/create_cgitrepos.py
+++ b/jeepyb/cmd/create_cgitrepos.py
@@ -39,14 +39,16 @@
def main():
- (defaults, config) = tuple(yaml.safe_load_all(open(PROJECTS_YAML)))
+ yaml_docs = [config for config in yaml.safe_load_all(open(PROJECTS_YAML))]
+ config = yaml_docs[-1]
gitorgs = {}
names = set()
for entry in config:
- (org, name) = entry['project'].split('/')
+ project = entry['project']
+ (org, name) = project.split('/')
description = entry.get('description', name)
- assert name not in names
- names.add(name)
+ assert project not in names
+ names.add(project)
gitorgs.setdefault(org, []).append((name, description))
if SCRATCH_SUBPATH:
assert SCRATCH_SUBPATH not in gitorgs
diff --git a/jeepyb/cmd/manage_projects.py b/jeepyb/cmd/manage_projects.py
index 634ce42..372e2ed 100644
--- a/jeepyb/cmd/manage_projects.py
+++ b/jeepyb/cmd/manage_projects.py
@@ -14,21 +14,24 @@
# License for the specific language governing permissions and limitations
# under the License.
-# manage_projects.py reads a project config file called projects.yaml
+# manage_projects.py reads a config file called projects.ini
# It should look like:
-# - homepage: http://openstack.org
-# gerrit-host: review.openstack.org
-# local-git-dir: /var/lib/git
-# gerrit-key: /home/gerrit2/review_site/etc/ssh_host_rsa_key
-# gerrit-committer: Project Creator <openstack-infra@lists.openstack.org>
-# has-github: True
-# has-wiki: False
-# has-issues: False
-# has-downloads: False
-# acl-dir: /home/gerrit2/acls
-# acl-base: /home/gerrit2/acls/project.config
-# ---
+# [projects]
+# homepage=http://openstack.org
+# gerrit-host=review.openstack.org
+# local-git-dir=/var/lib/git
+# gerrit-key=/home/gerrit2/review_site/etc/ssh_host_rsa_key
+# gerrit-committer=Project Creator <openstack-infra@lists.openstack.org>
+# has-github=True
+# has-wiki=False
+# has-issues=False
+# has-downloads=False
+# acl-dir=/home/gerrit2/acls
+# acl-base=/home/gerrit2/acls/project.config
+#
+# manage_projects.py reads a project listing file called projects.yaml
+# It should look like:
# - project: PROJECT_NAME
# options:
# - has-wiki
@@ -176,7 +179,7 @@
con = jeepyb.gerritdb.connect()
for x in range(10):
cursor = con.cursor()
- cursor.execute(query, group)
+ cursor.execute(query, (group,))
data = cursor.fetchone()
cursor.close()
con.commit()
@@ -235,21 +238,16 @@
return dict(GIT_SSH=name)
-def create_github_project(defaults, options, project, description, homepage):
+def create_github_project(
+ default_has_issues, default_has_downloads, default_has_wiki,
+ github_secure_config, options, project, description, homepage):
created = False
- default_has_issues = defaults.get('has-issues', False)
- default_has_downloads = defaults.get('has-downloads', False)
- default_has_wiki = defaults.get('has-wiki', False)
has_issues = 'has-issues' in options or default_has_issues
has_downloads = 'has-downloads' in options or default_has_downloads
has_wiki = 'has-wiki' in options or default_has_wiki
- GITHUB_SECURE_CONFIG = defaults.get(
- 'github-config',
- '/etc/github/github-projects.secure.config')
-
secure_config = ConfigParser.ConfigParser()
- secure_config.read(GITHUB_SECURE_CONFIG)
+ secure_config.read(github_secure_config)
# Project creation doesn't work via oauth
ghub = github.Github(secure_config.get("github", "username"),
@@ -501,6 +499,15 @@
git_mirror_path))
+def get_option(options, section, key, default):
+ if options.has_option(section, key):
+ if type(default) is bool:
+ return options.getboolean(section, key)
+ else:
+ return options.get(section, key)
+ return default
+
+
def main():
parser = argparse.ArgumentParser(description='Manage projects')
parser.add_argument('-v', dest='verbose', action='store_true',
@@ -518,20 +525,66 @@
PROJECTS_YAML = os.environ.get('PROJECTS_YAML',
'/home/gerrit2/projects.yaml')
- configs = [config for config in yaml.load_all(open(PROJECTS_YAML))]
- defaults = configs[0][0]
- default_has_github = defaults.get('has-github', True)
+ yaml_docs = [config for config in yaml.safe_load_all(open(PROJECTS_YAML))]
- LOCAL_GIT_DIR = defaults.get('local-git-dir', '/var/lib/git')
- JEEPYB_CACHE_DIR = defaults.get('jeepyb-cache-dir', '/var/lib/jeepyb')
- ACL_DIR = defaults.get('acl-dir')
- GERRIT_HOST = defaults.get('gerrit-host')
- GERRIT_PORT = int(defaults.get('gerrit-port', '29418'))
- GERRIT_USER = defaults.get('gerrit-user')
- GERRIT_KEY = defaults.get('gerrit-key')
- GERRIT_GITID = defaults.get('gerrit-committer')
- GERRIT_SYSTEM_USER = defaults.get('gerrit-system-user', 'gerrit2')
- GERRIT_SYSTEM_GROUP = defaults.get('gerrit-system-group', 'gerrit2')
+ PROJECTS_INI = os.environ.get('PROJECTS_INI',
+ '/home/gerrit2/projects.ini')
+ if os.path.exists(PROJECTS_INI):
+ # New-style - supports projects.ini
+ projects_yaml_list = yaml_docs[0]
+ defaults = ConfigParser.ConfigParser()
+ defaults.read(PROJECTS_INI)
+
+ default_has_github = get_option(
+ defaults, 'projects', 'has-github', True)
+
+ LOCAL_GIT_DIR = get_option(
+ defaults, 'projects', 'local-git-dir', '/var/lib/git')
+ JEEPYB_CACHE_DIR = get_option(
+ defaults, 'projects', 'jeepyb-cache-dir', '/var/lib/jeepyb')
+ ACL_DIR = defaults.get('projects', 'acl-dir')
+ GERRIT_HOST = defaults.get('projects', 'gerrit-host')
+ GERRIT_PORT = int(get_option(
+ defaults, 'projects', 'gerrit-port', '29418'))
+ GERRIT_USER = defaults.get('projects', 'gerrit-user')
+ GERRIT_KEY = defaults.get('projects', 'gerrit-key')
+ GERRIT_GITID = defaults.get('projects', 'gerrit-committer')
+ GERRIT_SYSTEM_USER = get_option(
+ defaults, 'projects', 'gerrit-system-user', 'gerrit2')
+ GERRIT_SYSTEM_GROUP = get_option(
+ defaults, 'projects', 'gerrit-system-group', 'gerrit2')
+ DEFAULT_HOMEPAGE = get_option(defaults, 'projects', 'homepage', None)
+ DEFAULT_HAS_ISSUES = get_option(
+ defaults, 'projects', 'has-issues', False)
+ DEFAULT_HAS_DOWNLOADS = get_option(
+ defaults, 'projects', 'has-downloads', False)
+ DEFAULT_HAS_WIKI = get_option(defaults, 'projects', 'has-wiki', False)
+ GITHUB_SECURE_CONFIG = get_option(
+ defaults, 'projects', 'github-config',
+ '/etc/github/github-projects.secure.config')
+ else:
+ # Old-style - embedded
+ projects_yaml_list = yaml_docs[1]
+ defaults = yaml_docs[0][0]
+ default_has_github = defaults.get('has-github', True)
+
+ LOCAL_GIT_DIR = defaults.get('local-git-dir', '/var/lib/git')
+ JEEPYB_CACHE_DIR = defaults.get('jeepyb-cache-dir', '/var/lib/jeepyb')
+ ACL_DIR = defaults.get('acl-dir')
+ GERRIT_HOST = defaults.get('gerrit-host')
+ GERRIT_PORT = int(defaults.get('gerrit-port', '29418'))
+ GERRIT_USER = defaults.get('gerrit-user')
+ GERRIT_KEY = defaults.get('gerrit-key')
+ GERRIT_GITID = defaults.get('gerrit-committer')
+ GERRIT_SYSTEM_USER = defaults.get('gerrit-system-user', 'gerrit2')
+ GERRIT_SYSTEM_GROUP = defaults.get('gerrit-system-group', 'gerrit2')
+ DEFAULT_HOMEPAGE = defaults.get('homepage', None)
+ DEFAULT_HAS_ISSUES = defaults.get('has-issues', False)
+ DEFAULT_HAS_DOWNLOADS = defaults.get('has-downloads', False)
+ DEFAULT_HAS_WIKI = defaults.get('has-wiki', False)
+ GITHUB_SECURE_CONFIG = defaults.get(
+ 'github-config',
+ '/etc/github/github-projects.secure.config')
gerrit = gerritlib.gerrit.Gerrit('localhost',
GERRIT_USER,
@@ -541,7 +594,7 @@
ssh_env = make_ssh_wrapper(GERRIT_USER, GERRIT_KEY)
try:
- for section in configs[1]:
+ for section in projects_yaml_list:
project = section['project']
if args.projects and project not in args.projects:
continue
@@ -550,8 +603,7 @@
# Figure out all of the options
options = section.get('options', dict())
description = section.get('description', None)
- homepage = section.get(
- 'homepage', defaults.get('homepage', None))
+ homepage = section.get('homepage', DEFAULT_HOMEPAGE)
upstream = section.get('upstream', None)
upstream_prefix = section.get('upstream-prefix', None)
track_upstream = 'track-upstream' in options
@@ -612,7 +664,9 @@
if 'has-github' in options or default_has_github:
created = create_github_project(
- defaults, options, project, description, homepage)
+ DEFAULT_HAS_ISSUES, DEFAULT_HAS_DOWNLOADS,
+ DEFAULT_HAS_WIKI, GITHUB_SECURE_CONFIG,
+ options, project, description, homepage)
if created:
gerrit.replicate(project)
diff --git a/jeepyb/cmd/notify_impact.py b/jeepyb/cmd/notify_impact.py
index a689f67..c3c3303 100644
--- a/jeepyb/cmd/notify_impact.py
+++ b/jeepyb/cmd/notify_impact.py
@@ -22,7 +22,7 @@
# --change-url https://review.openstack.org/55607 --project nova/ \
# --branch master --commit c262de4417d48be599c3a7496ef94de5c84b188c \
# --impact DocImpact --dest-address none@localhost --dryrun \
-# --ignore-duplicates \
+# --ignore-duplicates --config foo.yaml \
# change-merged
#
# But you'll need a git repository at /home/gerrit2/review_site/git/nova.git
@@ -41,6 +41,8 @@
from launchpadlib import uris
import yaml
+from jeepyb import projects
+
BASE_DIR = '/home/gerrit2/review_site'
EMAIL_TEMPLATE = """
Hi, I'd like you to take a look at this patch for potential
@@ -83,7 +85,10 @@
self.lpconn = lpconn
def create(self, project, bug_title, bug_descr, args):
- print('I would have created a bug, but I am in dry run mode')
+ print('I would have created a bug in %s, but I am in dry run mode.\n\n'
+ 'Title: %s\n'
+ 'Description:\n'
+ '%s' % (project, bug_title, bug_descr))
return None, None
def subscribe(self, buginfo, subscriber):
@@ -91,14 +96,29 @@
'but I am in dry run mode' % subscriber)
-def create_bug(git_log, args, lp_project, config):
+def create_bug(git_log, args, config):
"""Create a bug for a change.
- Create a launchpad bug in lp_project, titled with the first line of
+ Create a launchpad bug in a LP project, titled with the first line of
the git commit message, with the content of the git_log prepended
with the Gerrit review URL. Tag the bug with the name of the repository
it came from. Don't create a duplicate bug. Returns link to the bug.
"""
+
+ # Determine what LP project to use
+ prelude = ''
+ project_name = args.project.rstrip('/')
+ lp_project = projects.docimpact_target(project_name)
+ if lp_project == 'unknown':
+ prelude = ('\n\nDear documentation bug triager. This bug was created '
+ 'here because we did not know how to map the project name '
+ '"%s" to a launchpad project name. This indicates that the '
+ 'notify_impact config needs tweaks. You can ask the '
+ 'OpenStack infra team (#openstack-infra on freenode) for '
+ 'help if you need to.\n'
+ % args.project)
+ lp_project = 'openstack-manuals'
+
lpconn = launchpad.Launchpad.login_with(
'Gerrit User Sync',
uris.LPNET_SERVICE_ROOT,
@@ -111,9 +131,9 @@
else:
actions = BugActionsReal(lpconn)
- lines_in_log = git_log.split("\n")
+ lines_in_log = git_log.split('\n')
bug_title = lines_in_log[4]
- bug_descr = args.change_url + '\n' + git_log
+ bug_descr = args.change_url + prelude + '\n' + git_log
project = lpconn.projects[lp_project]
# check for existing bugs by searching for the title, to avoid
@@ -153,7 +173,7 @@
"""
if args.impact.lower() == 'docimpact':
if args.hook == "change-merged":
- create_bug(git_log, args, 'openstack-manuals', config)
+ create_bug(git_log, args, config)
return
email_content = EMAIL_TEMPLATE % (args.impact,
@@ -208,7 +228,8 @@
parser.add_argument('--impact', default=None)
parser.add_argument('--dest-address', default=None)
- # Automatic config
+ # Automatic config: config contains a mapping of email addresses to
+ # subscribers.
parser.add_argument('--config', type=argparse.FileType('r'),
default=None)
diff --git a/jeepyb/projects.py b/jeepyb/projects.py
index 083da4d..da898b9 100644
--- a/jeepyb/projects.py
+++ b/jeepyb/projects.py
@@ -64,6 +64,10 @@
return direct or _hardcoded_is_direct_release(project_full_name)
+def docimpact_target(project_full_name):
+ return registry.get('docimpact-group', 'unknown')
+
+
# The following functions should be deleted when projects.yaml will be updated
def _hardcoded_is_direct_release(project_full_name):
diff --git a/jeepyb/utils.py b/jeepyb/utils.py
index 5ff3830..d010e3b 100644
--- a/jeepyb/utils.py
+++ b/jeepyb/utils.py
@@ -27,15 +27,20 @@
It could be used as dict 'project name' -> 'project properties'.
"""
- def __init__(self, file_path, env_name=None):
+ def __init__(self, file_path, env_name=None, single_doc=True):
self.file_path = file_path
self.env_name = env_name
+ self.single_doc = single_doc
self._parse_file()
def _parse_file(self):
file_path = os.environ.get(self.env_name, self.file_path)
- configs_list = [config for config in yaml.load_all(open(file_path))][1]
+ yaml_docs = [config for config in yaml.safe_load_all(open(file_path))]
+ if self.single_doc:
+ configs_list = yaml_docs[0]
+ else:
+ configs_list = yaml_docs[1]
configs = {}
for section in configs_list:
@@ -45,3 +50,6 @@
def __getitem__(self, item):
return self.configs[item]
+
+ def get(self, item, default=None):
+ return self.configs.get(item, default)
diff --git a/requirements.txt b/requirements.txt
index f2b536e..e6f46b6 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,3 +1,5 @@
+pbr>=0.5.21,<1.0
+
argparse
gerritlib>=0.3.0
MySQL-python
diff --git a/test-requirements.txt b/test-requirements.txt
index 974f05f..a19a9e3 100644
--- a/test-requirements.txt
+++ b/test-requirements.txt
@@ -1 +1 @@
-hacking>=0.5.6,<0.7
+hacking>=0.8.0,<0.9