Extract projects.yaml loader into a module.
* jeepyb/cmd/close_pull_requests.py, jeepyb/cmd/fetch_remotes.py,
jeepyb/cmd/manage_projects.py: Consume the new jeepyb.projects
module and stop loading the projects.yaml file directly.
* jeepyb/projects.py: New module, extracted from manage_projects.py
and refactored to load a somewhat simplified projects.yaml format.
This is a step toward further streamlining of the format.
Change-Id: I6a05536f016d1327844a652444f89bfde08ac6fa
Reviewed-on: https://review.openstack.org/20104
Reviewed-by: James E. Blair <corvus@inaugust.com>
Approved: Monty Taylor <mordred@inaugust.com>
Reviewed-by: Monty Taylor <mordred@inaugust.com>
Tested-by: Jenkins
diff --git a/jeepyb/cmd/close_pull_requests.py b/jeepyb/cmd/close_pull_requests.py
index e7742f1..b641134 100644
--- a/jeepyb/cmd/close_pull_requests.py
+++ b/jeepyb/cmd/close_pull_requests.py
@@ -1,5 +1,6 @@
#! /usr/bin/env python
# Copyright (C) 2011 OpenStack, LLC.
+# Copyright (c) 2013 Hewlett-Packard Development Company, L.P.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@@ -13,19 +14,6 @@
# License for the specific language governing permissions and limitations
# under the License.
-# Github pull requests closer reads a project config file called projects.yaml
-# It should look like:
-
-# - homepage: http://openstack.org
-# team-id: 153703
-# has-wiki: False
-# has-issues: False
-# has-downloads: False
-# ---
-# - project: PROJECT_NAME
-# options:
-# - has-pull-requests
-
# Github authentication information is read from github.secure.config,
# which should look like:
@@ -41,9 +29,10 @@
import ConfigParser
import github
import os
-import yaml
import logging
+import jeepyb.projects
+
MESSAGE = """Thank you for contributing to OpenStack!
%(project)s uses Gerrit for code review.
@@ -64,8 +53,7 @@
secure_config = ConfigParser.ConfigParser()
secure_config.read(GITHUB_SECURE_CONFIG)
- (defaults, config) = [config for config in
- yaml.load_all(open(PROJECTS_YAML))]
+ projects = jeepyb.projects.get_config(PROJECTS_YAML)[1]
if secure_config.has_option("github", "oauth_token"):
ghub = github.Github(secure_config.get("github", "oauth_token"))
@@ -75,11 +63,10 @@
orgs = ghub.get_user().get_orgs()
orgs_dict = dict(zip([o.login.lower() for o in orgs], orgs))
- for section in config:
- project = section['project']
+ for project, parameters in projects.items:
# Make sure we're supposed to close pull requests for this project:
- if 'options' in section and 'has-pull-requests' in section['options']:
+ if 'has-pull-requests' in parameters['options']:
continue
# Find the project's repo
diff --git a/jeepyb/cmd/fetch_remotes.py b/jeepyb/cmd/fetch_remotes.py
index f34e518..d02adbf 100644
--- a/jeepyb/cmd/fetch_remotes.py
+++ b/jeepyb/cmd/fetch_remotes.py
@@ -1,5 +1,6 @@
#! /usr/bin/env python
# Copyright (C) 2011 OpenStack, LLC.
+# Copyright (C) 2013 Hewlett-Packard Development Company, L.P.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@@ -13,25 +14,13 @@
# License for the specific language governing permissions and limitations
# under the License.
-# Fetch remotes reads a project config file called projects.yaml
-# It should look like:
-
-# - homepage: http://openstack.org
-# team-id: 153703
-# has-wiki: False
-# has-issues: False
-# has-downloads: False
-# ---
-# - project: PROJECT_NAME
-# options:
-# - remote: https://gerrit.googlesource.com/gerrit
-
import logging
import os
import subprocess
import shlex
-import yaml
+
+import jeepyb.projects
def run_command(cmd, status=False, env={}):
@@ -58,20 +47,17 @@
PROJECTS_YAML = os.environ.get('PROJECTS_YAML',
'/home/gerrit2/projects.yaml')
- (defaults, config) = [config for config in
- yaml.load_all(open(PROJECTS_YAML))]
+ projects = jeepyb.projects.get_projects(PROJECTS_YAML)[1]
- for section in config:
- project = section['project']
-
- if 'remote' not in section:
+ for project, parameters in projects.items():
+ if 'remote' not in parameters:
continue
project_git = "%s.git" % project
os.chdir(os.path.join(REPO_ROOT, project_git))
# Make sure that the specified remote exists
- remote_url = section['remote']
+ remote_url = parameters['remote']
# We could check if it exists first, but we're ignoring output anyway
# So just try to make it, and it'll either make a new one or do nothing
run_command("git remote add -f upstream %s" % remote_url)
diff --git a/jeepyb/cmd/manage_projects.py b/jeepyb/cmd/manage_projects.py
index 2137ea0..967abc6 100644
--- a/jeepyb/cmd/manage_projects.py
+++ b/jeepyb/cmd/manage_projects.py
@@ -1,6 +1,6 @@
#! /usr/bin/env python
# Copyright (C) 2011 OpenStack, LLC.
-# Copyright (c) 2012 Hewlett-Packard Development Company, L.P.
+# Copyright (C) 2012, 2013 Hewlett-Packard Development Company, L.P.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@@ -14,36 +14,6 @@
# License for the specific language governing permissions and limitations
# under the License.
-# manage_projects.py reads a project config file called projects.yaml
-# It should look like:
-
-# - homepage: http://openstack.org
-# gerrit-host: review.openstack.org
-# local-git-dir: /var/lib/git
-# gerrit-key: /home/gerrit2/review_site/etc/ssh_host_rsa_key
-# has-wiki: False
-# has-issues: False
-# has-downloads: False
-# acl-dir: /home/gerrit2/acls
-# acl-base: /home/gerrit2/acls/project.config
-# ---
-# - project: PROJECT_NAME
-# options:
-# - has-wiki
-# - has-issues
-# - has-downloads
-# - has-pull-requests
-# homepage: Some homepage that isn't http://openstack.org
-# description: This is a great project
-# remote: https://gerrit.googlesource.com/gerrit
-# upstream: git://github.com/bushy/beards.git
-# acl-config: /path/to/gerrit/project.config
-# acl-append:
-# - /path/to/gerrit/project.config
-# acl-parameters:
-# project: OTHER_PROJECT_NAME
-
-
import ConfigParser
import logging
import os
@@ -51,12 +21,12 @@
import shlex
import subprocess
import tempfile
-import yaml
import github
import gerritlib.gerrit
import jeepyb.gerritdb
+import jeepyb.projects
logging.basicConfig(level=logging.ERROR)
log = logging.getLogger("manage_projects")
@@ -107,8 +77,6 @@
config_file = os.path.join(repo_base, "%s.config" % project)
else:
config_file = os.path.join(acl_dir, "%s.config" % project)
- if 'project' not in parameters:
- parameters['project'] = project
with open(config_file, 'w') as config:
if acl_base and os.path.exists(acl_base):
config.write(open(acl_base, 'r').read())
@@ -228,25 +196,17 @@
def main():
-
PROJECTS_YAML = os.environ.get('PROJECTS_YAML',
'/home/gerrit2/projects.yaml')
- configs = [config for config in yaml.load_all(open(PROJECTS_YAML))]
- defaults = configs[0][0]
- default_has_issues = defaults.get('has-issues', False)
- default_has_downloads = defaults.get('has-downloads', False)
- default_has_wiki = defaults.get('has-wiki', False)
+ defaults, registry = jeepyb.projects.get_projects(PROJECTS_YAML)
- LOCAL_GIT_DIR = defaults.get('local-git-dir', '/var/lib/git')
- ACL_DIR = defaults.get('acl-dir')
- GERRIT_HOST = defaults.get('gerrit-host')
- GERRIT_USER = defaults.get('gerrit-user')
- GERRIT_KEY = defaults.get('gerrit-key')
+ LOCAL_GIT_DIR = defaults['gerrit-defaults']['local-git-dir']
+ ACL_DIR = defaults['gerrit-defaults']['acl-dir']
+ GERRIT_HOST = defaults['gerrit-defaults']['host']
+ GERRIT_USER = defaults['gerrit-defaults']['user']
+ GERRIT_KEY = defaults['gerrit-defaults']['key']
- GITHUB_SECURE_CONFIG = defaults.get(
- 'github-config',
- '/etc/github/github-projects.secure.config')
-
+ GITHUB_SECURE_CONFIG = defaults['github-defaults']['config']
secure_config = ConfigParser.ConfigParser()
secure_config.read(GITHUB_SECURE_CONFIG)
@@ -264,13 +224,8 @@
ssh_env = make_ssh_wrapper(GERRIT_USER, GERRIT_KEY)
try:
- for section in configs[1]:
- project = section['project']
- options = section.get('options', dict())
- description = section.get('description', None)
- homepage = section.get('homepage', defaults.get('homepage', None))
- upstream = section.get('upstream', None)
-
+ for project, parameters in registry.items():
+ options = parameters['options']
project_git = "%s.git" % project
project_dir = os.path.join(LOCAL_GIT_DIR, project_git)
@@ -280,9 +235,6 @@
repo_name = project_split[1]
else:
repo_name = project
- has_issues = 'has-issues' in options or default_has_issues
- has_downloads = 'has-downloads' in options or default_has_downloads
- has_wiki = 'has-wiki' in options or default_has_wiki
try:
org = orgs_dict[project_split[0].lower()]
except KeyError:
@@ -291,19 +243,21 @@
try:
repo = org.get_repo(repo_name)
except github.GithubException:
- repo = org.create_repo(repo_name,
- homepage=homepage,
- has_issues=has_issues,
- has_downloads=has_downloads,
- has_wiki=has_wiki)
- if description:
- repo.edit(repo_name, description=description)
- if homepage:
- repo.edit(repo_name, homepage=homepage)
+ repo = org.create_repo(
+ repo_name,
+ homepage=parameters.get('homepage'),
+ has_issues='has-issues' in options,
+ has_downloads='has-downloads' in options,
+ has_wiki='has-wiki' in options)
+ if 'description' in parameters:
+ repo.edit(repo_name, description=parameters['description'])
+ if 'homepage' in parameters:
+ repo.edit(repo_name, homepage=parameters['homepage'])
- repo.edit(repo_name, has_issues=has_issues,
- has_downloads=has_downloads,
- has_wiki=has_wiki)
+ repo.edit(repo_name,
+ has_issues='has-issues' in options,
+ has_downloads='has-downloads' in options,
+ has_wiki='has-wiki' in options)
if 'gerrit' not in [team.name for team in repo.get_teams()]:
teams = org.get_teams()
@@ -315,9 +269,9 @@
tmpdir = tempfile.mkdtemp()
try:
repo_path = os.path.join(tmpdir, 'repo')
- if upstream:
+ if 'upstream' in parameters:
run_command("git clone %(upstream)s %(repo_path)s" %
- dict(upstream=upstream,
+ dict(upstream=parameters['upstream'],
repo_path=repo_path))
git_command(repo_path,
"fetch origin "
@@ -355,38 +309,32 @@
finally:
run_command("rm -fr %s" % tmpdir)
- try:
- acl_config = section.get('acl-config',
- '%s.config' % os.path.join(ACL_DIR,
- project))
- except AttributeError:
- acl_config = None
+ acl_config = parameters['acl-config']
- if acl_config:
- if not os.path.isfile(acl_config):
- write_acl_config(project,
- ACL_DIR,
- section.get('acl-base', None),
- section.get('acl-append', []),
- section.get('acl-parameters', {}))
- tmpdir = tempfile.mkdtemp()
- try:
- repo_path = os.path.join(tmpdir, 'repo')
- ret, _ = run_command_status("git init %s" % repo_path)
- if ret != 0:
- continue
- if (fetch_config(project,
- remote_url,
- repo_path,
- ssh_env) and
- copy_acl_config(project, repo_path,
- acl_config) and
- create_groups_file(project, gerrit, repo_path)):
- push_acl_config(project,
- remote_url,
- repo_path,
- ssh_env)
- finally:
- run_command("rm -fr %s" % tmpdir)
+ if not os.path.isfile(acl_config):
+ write_acl_config(project,
+ ACL_DIR,
+ parameters['acl-base'],
+ parameters['acl-append'],
+ parameters['acl-parameters'])
+ tmpdir = tempfile.mkdtemp()
+ try:
+ repo_path = os.path.join(tmpdir, 'repo')
+ ret, _ = run_command_status("git init %s" % repo_path)
+ if ret != 0:
+ continue
+ if (fetch_config(project,
+ remote_url,
+ repo_path,
+ ssh_env) and
+ copy_acl_config(project, repo_path,
+ parameters['acl-config']) and
+ create_groups_file(project, gerrit, repo_path)):
+ push_acl_config(project,
+ remote_url,
+ repo_path,
+ ssh_env)
+ finally:
+ run_command("rm -fr %s" % tmpdir)
finally:
os.unlink(ssh_env['GIT_SSH'])
diff --git a/jeepyb/projects.py b/jeepyb/projects.py
new file mode 100644
index 0000000..8e1fbb4
--- /dev/null
+++ b/jeepyb/projects.py
@@ -0,0 +1,109 @@
+#! /usr/bin/env python
+# Copyright (C) 2011 OpenStack, LLC.
+# Copyright (C) 2012, 2013 Hewlett-Packard Development Company, L.P.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+
+import yaml
+
+
+def get_projects(registry_file):
+ """
+ This reads a project registry file which should look like:
+
+ gerrit-defaults:
+ acl-dir: /home/gerrit2/acls
+ host: review.openstack.org
+ key: /home/gerrit2/review_site/etc/ssh_host_rsa_key
+ local-git-dir: /var/lib/git
+ user: openstack-project-creator
+ github-defaults:
+ config: /etc/github/github-projects.secure.config
+ project-defaults:
+ homepage: http://www.openstack.org/
+ options: []
+ ---
+ ONE_PROJECT_NAME:
+ acl-append:
+ - /path/to/gerrit/project.config
+ acl-base: /home/gerrit2/acls/project.config
+ description: This is a great project.
+ homepage: Some homepage that isn't http://www.openstack.org/
+ launchpad: someproject
+ options:
+ - has-downloads
+ - has-issues
+ - has-pull-requests
+ - has-wiki
+ - no-lp-bugs
+ - release-on-merge
+ remote: https://gerrit.googlesource.com/gerrit
+ upstream: git://github.com/bushy/beards.git
+ ANOTHER_PROJECT_NAME:
+ acl-parameters:
+ project: SOME_PROJECT_NAME
+ description: This is an even greater project.
+ """
+
+ configs = [config for config in yaml.load_all(open(registry_file))]
+ if len(configs) == 2:
+ # two sections means the first one contains defaults
+ configured_defaults = configs[0]
+ projects = configs[1]
+ else:
+ # only one means there are no configured defaults
+ configured_defaults = {}
+ projects = configs[0]
+
+ # start with some builtin defaults for safety
+ builtin_defaults = {
+ 'gerrit-defaults': {
+ 'acl-dir': '/home/gerrit2/acls',
+ 'host': 'review.openstack.org',
+ 'key': '/home/gerrit2/review_site/etc/ssh_host_rsa_key',
+ 'local-git-dir': '/var/lib/git',
+ 'user': 'openstack-project-creator',
+ },
+ 'github-defaults': {
+ 'config': '/etc/github/github-projects.secure.config',
+ },
+ 'project-defaults': {
+ 'acl-append': [],
+ 'acl-base': None,
+ 'acl-parameters': {},
+ 'homepage': 'http://www.openstack.org/',
+ 'options': [],
+ }
+ }
+
+ # override the builtin defaults with any provided in the registry file
+ defaults = {}
+ for section in builtin_defaults:
+ defaults[section] = dict(
+ list(builtin_defaults[section].items())
+ + list(configured_defaults.get(section, {}).items())
+ )
+
+ # build the project registry
+ registry = {}
+ for project in projects:
+ registry[project] = dict(
+ list(defaults['project-defaults'].items())
+ + list(projects[project].items())
+ )
+ if 'acl-config' not in registry[project]:
+ registry[project]['acl-config'] = '%s.config' % os.path.join(
+ defaults['gerrit-defaults']['acl-dir'], project)
+
+ return (defaults, registry)