Revert "Extract projects.yaml loader into a module."
This was not ready to be merged.
This reverts commit 8b9f93f67ad43e0ac8dd95e02f8269f6d7cae912
Change-Id: Icef953ba61959b6b2e7aa18731eb7b953379195b
Reviewed-on: https://review.openstack.org/26104
Approved: Jeremy Stanley <fungi@yuggoth.org>
Reviewed-by: Jeremy Stanley <fungi@yuggoth.org>
Tested-by: Jenkins
diff --git a/jeepyb/cmd/close_pull_requests.py b/jeepyb/cmd/close_pull_requests.py
index b641134..e7742f1 100644
--- a/jeepyb/cmd/close_pull_requests.py
+++ b/jeepyb/cmd/close_pull_requests.py
@@ -1,6 +1,5 @@
#! /usr/bin/env python
# Copyright (C) 2011 OpenStack, LLC.
-# Copyright (c) 2013 Hewlett-Packard Development Company, L.P.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@@ -14,6 +13,19 @@
# License for the specific language governing permissions and limitations
# under the License.
+# Github pull requests closer reads a project config file called projects.yaml
+# It should look like:
+
+# - homepage: http://openstack.org
+# team-id: 153703
+# has-wiki: False
+# has-issues: False
+# has-downloads: False
+# ---
+# - project: PROJECT_NAME
+# options:
+# - has-pull-requests
+
# Github authentication information is read from github.secure.config,
# which should look like:
@@ -29,10 +41,9 @@
import ConfigParser
import github
import os
+import yaml
import logging
-import jeepyb.projects
-
MESSAGE = """Thank you for contributing to OpenStack!
%(project)s uses Gerrit for code review.
@@ -53,7 +64,8 @@
secure_config = ConfigParser.ConfigParser()
secure_config.read(GITHUB_SECURE_CONFIG)
- projects = jeepyb.projects.get_config(PROJECTS_YAML)[1]
+ (defaults, config) = [config for config in
+ yaml.load_all(open(PROJECTS_YAML))]
if secure_config.has_option("github", "oauth_token"):
ghub = github.Github(secure_config.get("github", "oauth_token"))
@@ -63,10 +75,11 @@
orgs = ghub.get_user().get_orgs()
orgs_dict = dict(zip([o.login.lower() for o in orgs], orgs))
- for project, parameters in projects.items:
+ for section in config:
+ project = section['project']
# Make sure we're supposed to close pull requests for this project:
- if 'has-pull-requests' in parameters['options']:
+ if 'options' in section and 'has-pull-requests' in section['options']:
continue
# Find the project's repo
diff --git a/jeepyb/cmd/fetch_remotes.py b/jeepyb/cmd/fetch_remotes.py
index d02adbf..f34e518 100644
--- a/jeepyb/cmd/fetch_remotes.py
+++ b/jeepyb/cmd/fetch_remotes.py
@@ -1,6 +1,5 @@
#! /usr/bin/env python
# Copyright (C) 2011 OpenStack, LLC.
-# Copyright (C) 2013 Hewlett-Packard Development Company, L.P.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@@ -14,13 +13,25 @@
# License for the specific language governing permissions and limitations
# under the License.
+# Fetch remotes reads a project config file called projects.yaml
+# It should look like:
+
+# - homepage: http://openstack.org
+# team-id: 153703
+# has-wiki: False
+# has-issues: False
+# has-downloads: False
+# ---
+# - project: PROJECT_NAME
+# options:
+# - remote: https://gerrit.googlesource.com/gerrit
+
import logging
import os
import subprocess
import shlex
-
-import jeepyb.projects
+import yaml
def run_command(cmd, status=False, env={}):
@@ -47,17 +58,20 @@
PROJECTS_YAML = os.environ.get('PROJECTS_YAML',
'/home/gerrit2/projects.yaml')
- projects = jeepyb.projects.get_projects(PROJECTS_YAML)[1]
+ (defaults, config) = [config for config in
+ yaml.load_all(open(PROJECTS_YAML))]
- for project, parameters in projects.items():
- if 'remote' not in parameters:
+ for section in config:
+ project = section['project']
+
+ if 'remote' not in section:
continue
project_git = "%s.git" % project
os.chdir(os.path.join(REPO_ROOT, project_git))
# Make sure that the specified remote exists
- remote_url = parameters['remote']
+ remote_url = section['remote']
# We could check if it exists first, but we're ignoring output anyway
# So just try to make it, and it'll either make a new one or do nothing
run_command("git remote add -f upstream %s" % remote_url)
diff --git a/jeepyb/cmd/manage_projects.py b/jeepyb/cmd/manage_projects.py
index 967abc6..2137ea0 100644
--- a/jeepyb/cmd/manage_projects.py
+++ b/jeepyb/cmd/manage_projects.py
@@ -1,6 +1,6 @@
#! /usr/bin/env python
# Copyright (C) 2011 OpenStack, LLC.
-# Copyright (C) 2012, 2013 Hewlett-Packard Development Company, L.P.
+# Copyright (c) 2012 Hewlett-Packard Development Company, L.P.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@@ -14,6 +14,36 @@
# License for the specific language governing permissions and limitations
# under the License.
+# manage_projects.py reads a project config file called projects.yaml
+# It should look like:
+
+# - homepage: http://openstack.org
+# gerrit-host: review.openstack.org
+# local-git-dir: /var/lib/git
+# gerrit-key: /home/gerrit2/review_site/etc/ssh_host_rsa_key
+# has-wiki: False
+# has-issues: False
+# has-downloads: False
+# acl-dir: /home/gerrit2/acls
+# acl-base: /home/gerrit2/acls/project.config
+# ---
+# - project: PROJECT_NAME
+# options:
+# - has-wiki
+# - has-issues
+# - has-downloads
+# - has-pull-requests
+# homepage: Some homepage that isn't http://openstack.org
+# description: This is a great project
+# remote: https://gerrit.googlesource.com/gerrit
+# upstream: git://github.com/bushy/beards.git
+# acl-config: /path/to/gerrit/project.config
+# acl-append:
+# - /path/to/gerrit/project.config
+# acl-parameters:
+# project: OTHER_PROJECT_NAME
+
+
import ConfigParser
import logging
import os
@@ -21,12 +51,12 @@
import shlex
import subprocess
import tempfile
+import yaml
import github
import gerritlib.gerrit
import jeepyb.gerritdb
-import jeepyb.projects
logging.basicConfig(level=logging.ERROR)
log = logging.getLogger("manage_projects")
@@ -77,6 +107,8 @@
config_file = os.path.join(repo_base, "%s.config" % project)
else:
config_file = os.path.join(acl_dir, "%s.config" % project)
+ if 'project' not in parameters:
+ parameters['project'] = project
with open(config_file, 'w') as config:
if acl_base and os.path.exists(acl_base):
config.write(open(acl_base, 'r').read())
@@ -196,17 +228,25 @@
def main():
+
PROJECTS_YAML = os.environ.get('PROJECTS_YAML',
'/home/gerrit2/projects.yaml')
- defaults, registry = jeepyb.projects.get_projects(PROJECTS_YAML)
+ configs = [config for config in yaml.load_all(open(PROJECTS_YAML))]
+ defaults = configs[0][0]
+ default_has_issues = defaults.get('has-issues', False)
+ default_has_downloads = defaults.get('has-downloads', False)
+ default_has_wiki = defaults.get('has-wiki', False)
- LOCAL_GIT_DIR = defaults['gerrit-defaults']['local-git-dir']
- ACL_DIR = defaults['gerrit-defaults']['acl-dir']
- GERRIT_HOST = defaults['gerrit-defaults']['host']
- GERRIT_USER = defaults['gerrit-defaults']['user']
- GERRIT_KEY = defaults['gerrit-defaults']['key']
+ LOCAL_GIT_DIR = defaults.get('local-git-dir', '/var/lib/git')
+ ACL_DIR = defaults.get('acl-dir')
+ GERRIT_HOST = defaults.get('gerrit-host')
+ GERRIT_USER = defaults.get('gerrit-user')
+ GERRIT_KEY = defaults.get('gerrit-key')
- GITHUB_SECURE_CONFIG = defaults['github-defaults']['config']
+ GITHUB_SECURE_CONFIG = defaults.get(
+ 'github-config',
+ '/etc/github/github-projects.secure.config')
+
secure_config = ConfigParser.ConfigParser()
secure_config.read(GITHUB_SECURE_CONFIG)
@@ -224,8 +264,13 @@
ssh_env = make_ssh_wrapper(GERRIT_USER, GERRIT_KEY)
try:
- for project, parameters in registry.items():
- options = parameters['options']
+ for section in configs[1]:
+ project = section['project']
+ options = section.get('options', dict())
+ description = section.get('description', None)
+ homepage = section.get('homepage', defaults.get('homepage', None))
+ upstream = section.get('upstream', None)
+
project_git = "%s.git" % project
project_dir = os.path.join(LOCAL_GIT_DIR, project_git)
@@ -235,6 +280,9 @@
repo_name = project_split[1]
else:
repo_name = project
+ has_issues = 'has-issues' in options or default_has_issues
+ has_downloads = 'has-downloads' in options or default_has_downloads
+ has_wiki = 'has-wiki' in options or default_has_wiki
try:
org = orgs_dict[project_split[0].lower()]
except KeyError:
@@ -243,21 +291,19 @@
try:
repo = org.get_repo(repo_name)
except github.GithubException:
- repo = org.create_repo(
- repo_name,
- homepage=parameters.get('homepage'),
- has_issues='has-issues' in options,
- has_downloads='has-downloads' in options,
- has_wiki='has-wiki' in options)
- if 'description' in parameters:
- repo.edit(repo_name, description=parameters['description'])
- if 'homepage' in parameters:
- repo.edit(repo_name, homepage=parameters['homepage'])
+ repo = org.create_repo(repo_name,
+ homepage=homepage,
+ has_issues=has_issues,
+ has_downloads=has_downloads,
+ has_wiki=has_wiki)
+ if description:
+ repo.edit(repo_name, description=description)
+ if homepage:
+ repo.edit(repo_name, homepage=homepage)
- repo.edit(repo_name,
- has_issues='has-issues' in options,
- has_downloads='has-downloads' in options,
- has_wiki='has-wiki' in options)
+ repo.edit(repo_name, has_issues=has_issues,
+ has_downloads=has_downloads,
+ has_wiki=has_wiki)
if 'gerrit' not in [team.name for team in repo.get_teams()]:
teams = org.get_teams()
@@ -269,9 +315,9 @@
tmpdir = tempfile.mkdtemp()
try:
repo_path = os.path.join(tmpdir, 'repo')
- if 'upstream' in parameters:
+ if upstream:
run_command("git clone %(upstream)s %(repo_path)s" %
- dict(upstream=parameters['upstream'],
+ dict(upstream=upstream,
repo_path=repo_path))
git_command(repo_path,
"fetch origin "
@@ -309,32 +355,38 @@
finally:
run_command("rm -fr %s" % tmpdir)
- acl_config = parameters['acl-config']
-
- if not os.path.isfile(acl_config):
- write_acl_config(project,
- ACL_DIR,
- parameters['acl-base'],
- parameters['acl-append'],
- parameters['acl-parameters'])
- tmpdir = tempfile.mkdtemp()
try:
- repo_path = os.path.join(tmpdir, 'repo')
- ret, _ = run_command_status("git init %s" % repo_path)
- if ret != 0:
- continue
- if (fetch_config(project,
- remote_url,
- repo_path,
- ssh_env) and
- copy_acl_config(project, repo_path,
- parameters['acl-config']) and
- create_groups_file(project, gerrit, repo_path)):
- push_acl_config(project,
- remote_url,
- repo_path,
- ssh_env)
- finally:
- run_command("rm -fr %s" % tmpdir)
+ acl_config = section.get('acl-config',
+ '%s.config' % os.path.join(ACL_DIR,
+ project))
+ except AttributeError:
+ acl_config = None
+
+ if acl_config:
+ if not os.path.isfile(acl_config):
+ write_acl_config(project,
+ ACL_DIR,
+ section.get('acl-base', None),
+ section.get('acl-append', []),
+ section.get('acl-parameters', {}))
+ tmpdir = tempfile.mkdtemp()
+ try:
+ repo_path = os.path.join(tmpdir, 'repo')
+ ret, _ = run_command_status("git init %s" % repo_path)
+ if ret != 0:
+ continue
+ if (fetch_config(project,
+ remote_url,
+ repo_path,
+ ssh_env) and
+ copy_acl_config(project, repo_path,
+ acl_config) and
+ create_groups_file(project, gerrit, repo_path)):
+ push_acl_config(project,
+ remote_url,
+ repo_path,
+ ssh_env)
+ finally:
+ run_command("rm -fr %s" % tmpdir)
finally:
os.unlink(ssh_env['GIT_SSH'])
diff --git a/jeepyb/projects.py b/jeepyb/projects.py
deleted file mode 100644
index 8e1fbb4..0000000
--- a/jeepyb/projects.py
+++ /dev/null
@@ -1,109 +0,0 @@
-#! /usr/bin/env python
-# Copyright (C) 2011 OpenStack, LLC.
-# Copyright (C) 2012, 2013 Hewlett-Packard Development Company, L.P.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
-# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
-# License for the specific language governing permissions and limitations
-# under the License.
-
-
-import yaml
-
-
-def get_projects(registry_file):
- """
- This reads a project registry file which should look like:
-
- gerrit-defaults:
- acl-dir: /home/gerrit2/acls
- host: review.openstack.org
- key: /home/gerrit2/review_site/etc/ssh_host_rsa_key
- local-git-dir: /var/lib/git
- user: openstack-project-creator
- github-defaults:
- config: /etc/github/github-projects.secure.config
- project-defaults:
- homepage: http://www.openstack.org/
- options: []
- ---
- ONE_PROJECT_NAME:
- acl-append:
- - /path/to/gerrit/project.config
- acl-base: /home/gerrit2/acls/project.config
- description: This is a great project.
- homepage: Some homepage that isn't http://www.openstack.org/
- launchpad: someproject
- options:
- - has-downloads
- - has-issues
- - has-pull-requests
- - has-wiki
- - no-lp-bugs
- - release-on-merge
- remote: https://gerrit.googlesource.com/gerrit
- upstream: git://github.com/bushy/beards.git
- ANOTHER_PROJECT_NAME:
- acl-parameters:
- project: SOME_PROJECT_NAME
- description: This is an even greater project.
- """
-
- configs = [config for config in yaml.load_all(open(registry_file))]
- if len(configs) == 2:
- # two sections means the first one contains defaults
- configured_defaults = configs[0]
- projects = configs[1]
- else:
- # only one means there are no configured defaults
- configured_defaults = {}
- projects = configs[0]
-
- # start with some builtin defaults for safety
- builtin_defaults = {
- 'gerrit-defaults': {
- 'acl-dir': '/home/gerrit2/acls',
- 'host': 'review.openstack.org',
- 'key': '/home/gerrit2/review_site/etc/ssh_host_rsa_key',
- 'local-git-dir': '/var/lib/git',
- 'user': 'openstack-project-creator',
- },
- 'github-defaults': {
- 'config': '/etc/github/github-projects.secure.config',
- },
- 'project-defaults': {
- 'acl-append': [],
- 'acl-base': None,
- 'acl-parameters': {},
- 'homepage': 'http://www.openstack.org/',
- 'options': [],
- }
- }
-
- # override the builtin defaults with any provided in the registry file
- defaults = {}
- for section in builtin_defaults:
- defaults[section] = dict(
- list(builtin_defaults[section].items())
- + list(configured_defaults.get(section, {}).items())
- )
-
- # build the project registry
- registry = {}
- for project in projects:
- registry[project] = dict(
- list(defaults['project-defaults'].items())
- + list(projects[project].items())
- )
- if 'acl-config' not in registry[project]:
- registry[project]['acl-config'] = '%s.config' % os.path.join(
- defaults['gerrit-defaults']['acl-dir'], project)
-
- return (defaults, registry)