Attempt to more fully manage project creation.
Manage project creation via yaml files. Also,
Modify the manage_projects scripts to configure Gerrit project ACLs.
This change expects the project yaml to exist. The change will clone the
project for the localhost Gerrit install. It will then checkout the
meta/config ref, copy the ACL config file into the repo, commit, and
push to the origin. The ACL config location should be specified in the
projects.yaml file with the acl_config key.
For this to work the ACLs will need to be copied by Puppet from Puppet
to the Gerrit host. Add the file resource to do this as well.
Change-Id: I15a1ec13b381dce3c115c01c21f404ab79e72cc4
Reviewed-on: https://review.openstack.org/15352
Reviewed-by: Jeremy Stanley <fungi@yuggoth.org>
Approved: Monty Taylor <mordred@inaugust.com>
Reviewed-by: Monty Taylor <mordred@inaugust.com>
Tested-by: Jenkins
diff --git a/make_local_repos.py b/make_local_repos.py
deleted file mode 100755
index bb8ab9d..0000000
--- a/make_local_repos.py
+++ /dev/null
@@ -1,65 +0,0 @@
-#! /usr/bin/env python
-# Copyright (C) 2011 OpenStack, LLC.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
-# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
-# License for the specific language governing permissions and limitations
-# under the License.
-
-# Make local repos reads a project config file called projects.yaml
-# It should look like:
-
-# - project: PROJECT_NAME
-# remote: https://gerrit.googlesource.com/gerrit
-
-# TODO: add support for
-# ssh -p 29418 localhost gerrit -name create-project PROJECT
-
-import logging
-import os
-import subprocess
-import sys
-import shlex
-import yaml
-
-def run_command(cmd, status=False, env={}):
- cmd_list = shlex.split(str(cmd))
- newenv = os.environ
- newenv.update(env)
- p = subprocess.Popen(cmd_list, stdout=subprocess.PIPE,
- stderr=subprocess.STDOUT, env=newenv)
- (out, nothing) = p.communicate()
- if status:
- return (p.returncode, out.strip())
- return out.strip()
-
-
-def run_command_status(cmd, env={}):
- return run_command(cmd, True, env)
-
-
-logging.basicConfig(level=logging.ERROR)
-
-REPO_ROOT = sys.argv[1]
-PROJECTS_YAML = os.environ.get('PROJECTS_YAML',
- '/home/gerrit2/projects.yaml')
-
-config = yaml.load(open(PROJECTS_YAML))
-
-for section in config:
- project = section['project']
-
- project_git = "%s.git" % project
- project_dir = os.path.join(REPO_ROOT, project_git)
-
- if os.path.exists(project_dir):
- continue
-
- run_command("git --bare init --shared=group %s" % project_dir)
diff --git a/manage_projects.py b/manage_projects.py
new file mode 100755
index 0000000..fb50b26
--- /dev/null
+++ b/manage_projects.py
@@ -0,0 +1,312 @@
+#! /usr/bin/env python
+# Copyright (C) 2011 OpenStack, LLC.
+# Copyright (c) 2012 Hewlett-Packard Development Company, L.P.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+# manage_projects.py reads a project config file called projects.yaml
+# It should look like:
+
+# - homepage: http://openstack.org
+# gerrit-host: review.openstack.org
+# local-git-dir: /var/lib/git
+# gerrit-key: /home/gerrit2/review_site/etc/ssh_host_rsa_key
+# has-wiki: False
+# has-issues: False
+# has-downloads: False
+# ---
+# - project: PROJECT_NAME
+# options:
+# - has-wiki
+# - has-issues
+# - has-downloads
+# - has-pull-requests
+# homepage: Some homepage that isn't http://openstack.org
+# description: This is a great project
+# remote: https://gerrit.googlesource.com/gerrit
+# upstream: git://github.com/bushy/beards.git
+# acl_config: /path/to/gerrit/project.config
+
+
+import ConfigParser
+import logging
+import os
+import re
+import shlex
+import subprocess
+import tempfile
+import yaml
+
+import github
+import gerritlib.gerrit
+
+
+logging.basicConfig(level=logging.ERROR)
+log = logging.getLogger("manage_projects")
+
+
+def run_command(cmd, status=False, env={}):
+ cmd_list = shlex.split(str(cmd))
+ newenv = os.environ
+ newenv.update(env)
+ log.debug("Executing command: %s" % " ".join(cmd_list))
+ p = subprocess.Popen(cmd_list, stdout=subprocess.PIPE,
+ stderr=subprocess.STDOUT, env=newenv)
+ (out, nothing) = p.communicate()
+ log.debug("Return code: %s" % p.returncode)
+ log.debug("Command said: %s" % out.strip())
+ if status:
+ return (p.returncode, out.strip())
+ return out.strip()
+
+
+def run_command_status(cmd, env={}):
+ return run_command(cmd, True, env)
+
+
+def git_command(repo_dir, sub_cmd, env={}):
+ git_dir = os.path.join(repo_dir, '.git')
+ cmd = "git --git-dir=%s --work-tree=%s %s" % (git_dir, repo_dir, sub_cmd)
+ status, _ = run_command(cmd, True, env)
+ return status
+
+
+def fetch_config(project, remote_url, repo_path, env={}):
+ status = git_command(repo_path, "fetch %s +refs/meta/config:"
+ "refs/remotes/gerrit-meta/config" % remote_url, env)
+ if status != 0:
+ print "Failed to fetch refs/meta/config for project: %s" % project
+ return False
+ # Because the following fails if executed more than once you should only
+ # run fetch_config once in each repo.
+ status = git_command(repo_path, "checkout -b config "
+ "remotes/gerrit-meta/config")
+ if status != 0:
+ print "Failed to checkout config for project: %s" % project
+ return False
+
+ return True
+
+
+def copy_acl_config(project, repo_path, acl_config):
+ if not os.path.exists(acl_config):
+ return False
+
+ acl_dest = os.path.join(repo_path, "project.config")
+ status, _ = run_command("cp %s %s" %
+ (acl_config, acl_dest), status=True)
+ if status == 0:
+ status = git_command(repo_path, "diff-index --quiet HEAD --")
+ if status != 0:
+ return True
+ return False
+
+
+def push_acl_config(project, remote_url, repo_path, env={}):
+ cmd = "commit -a -m'Update project config.' --author='Openstack Project " \
+ "Creator <openstack-infra@lists.openstack.org>'"
+ status = git_command(repo_path, cmd)
+ if status != 0:
+ print "Failed to commit config for project: %s" % project
+ return False
+ status = git_command(repo_path,
+ "push %s HEAD:refs/meta/config" %
+ remote_url, env)
+ if status != 0:
+ print "Failed to push config for project: %s" % project
+ return False
+ return True
+
+
+def _get_group_uuid(gerrit, group):
+ query = "select group_uuid from account_groups where name = '%s'" % group
+ data = gerrit.dbQuery(query)
+ if data:
+ for row in data:
+ if row["type"] == "row":
+ return row["columns"]["group_uuid"]
+ return None
+
+
+def get_group_uuid(gerrit, group):
+ uuid = _get_group_uuid(gerrit, group)
+ if uuid:
+ return uuid
+ gerrit.createGroup(group)
+ uuid = _get_group_uuid(gerrit, group)
+ if uuid:
+ return uuid
+ return None
+
+
+def create_groups_file(project, gerrit, repo_path):
+ acl_config = os.path.join(repo_path, "project.config")
+ group_file = os.path.join(repo_path, "groups")
+ uuids = {}
+ for line in open(acl_config, 'r'):
+ r = re.match(r'^\t.*group\s(.*)$', line)
+ if r:
+ group = r.group(1)
+ if group in uuids.keys():
+ continue
+ uuid = get_group_uuid(gerrit, group)
+ if uuid:
+ uuids[group] = uuid
+ else:
+ return False
+ if uuids:
+ with open(group_file, 'w') as fp:
+ for group, uuid in uuids.items():
+ fp.write("%s\t%s\n" % (uuid, group))
+ status = git_command(repo_path, "add groups")
+ if status != 0:
+ print "Failed to add groups file for project: %s" % project
+ return False
+ return True
+
+
+def make_ssh_wrapper(gerrit_user, gerrit_key):
+ (fd, name) = tempfile.mkstemp(text=True)
+ os.write(fd, '#!/bin/bash\n')
+ os.write(fd,
+ 'ssh -i %s -l %s -o "StrictHostKeyChecking no" $@\n' %
+ (gerrit_key, gerrit_user))
+ os.close(fd)
+ os.chmod(name, 755)
+ return dict(GIT_SSH=name)
+
+
+PROJECTS_YAML = os.environ.get('PROJECTS_YAML',
+ '/home/gerrit2/projects.yaml')
+configs = [config for config in yaml.load_all(open(PROJECTS_YAML))]
+defaults = configs[0][0]
+default_has_issues = defaults.get('has-issues', False)
+default_has_downloads = defaults.get('has-downloads', False)
+default_has_wiki = defaults.get('has-wiki', False)
+
+LOCAL_GIT_DIR = defaults.get('local-git-dir', '/var/lib/git')
+GERRIT_HOST = defaults.get('gerrit-host')
+GERRIT_USER = defaults.get('gerrit-user')
+GERRIT_KEY = defaults.get('gerrit-key')
+GITHUB_SECURE_CONFIG = defaults.get('github-config',
+ '/etc/github/github-projects.secure.config')
+
+secure_config = ConfigParser.ConfigParser()
+secure_config.read(GITHUB_SECURE_CONFIG)
+
+# Project creation doesn't work via oauth
+ghub = github.Github(secure_config.get("github", "username"),
+ secure_config.get("github", "password"))
+orgs = ghub.get_user().get_orgs()
+orgs_dict = dict(zip([o.login.lower() for o in orgs], orgs))
+
+gerrit = gerritlib.gerrit.Gerrit('localhost',
+ GERRIT_USER,
+ 29418,
+ GERRIT_KEY)
+project_list = gerrit.listProjects()
+ssh_env = make_ssh_wrapper(GERRIT_USER, GERRIT_KEY)
+try:
+
+ for section in configs[1]:
+ project = section['project']
+ options = section.get('options', dict())
+ description = section.get('description', None)
+ homepage = section.get('homepage', defaults.get('homepage', None))
+ upstream = section.get('upstream', None)
+
+ project_git = "%s.git" % project
+ project_dir = os.path.join(LOCAL_GIT_DIR, project_git)
+
+ # Find the project's repo
+ project_split = project.split('/', 1)
+ if len(project_split) > 1:
+ repo_name = project_split[1]
+ else:
+ repo_name = project
+ has_issues = 'has-issues' in options or default_has_issues
+ has_downloads = 'has-downloads' in options or default_has_downloads
+ has_wiki = 'has-wiki' in options or default_has_wiki
+ org = orgs_dict[project_split[0].lower()]
+ try:
+ repo = org.get_repo(repo_name)
+ except github.GithubException:
+ repo = org.create_repo(repo_name,
+ homepage=homepage,
+ has_issues=has_issues,
+ has_downloads=has_downloads,
+ has_wiki=has_wiki)
+ if description:
+ repo.edit(repo_name, description=description)
+ if homepage:
+ repo.edit(repo_name, homepage=homepage)
+
+ repo.edit(repo_name, has_issues=has_issues,
+ has_downloads=has_downloads,
+ has_wiki=has_wiki)
+
+ if 'gerrit' not in [team.name for team in repo.get_teams()]:
+ teams = org.get_teams()
+ teams_dict = dict(zip([t.name.lower() for t in teams], teams))
+ teams_dict['gerrit'].add_to_repos(repo)
+
+ remote_url = "ssh://localhost:29418/%s" % project
+ if project not in project_list:
+ tmpdir = tempfile.mkdtemp()
+ try:
+ repo_path = os.path.join(tmpdir, 'repo')
+ if upstream:
+ run_command("git clone %(upstream)s %(repo_path)" %
+ dict(upstream=upstream, repo_path=repo_path))
+ else:
+ run_command("git init %s" % repo_path)
+ with open(os.path.join(repo_path,
+ ".gitreview"), 'w') as gitreview:
+ gitreview.write("""
+[gerrit]
+host=%s
+port=29418
+project=%s
+""" % (GERRIT_HOST, project_git))
+ git_command(repo_path, "add .gitreview")
+ git_command(repo_path, "commit -a -m'Added .gitreview'")
+ gerrit.createProject(project)
+
+ if not os.path.exists(project_dir):
+ run_command("git --bare init %s" % project_dir)
+ run_command("chown -R gerrit2:gerrit2 %s" % project_dir)
+
+ git_command(repo_path,
+ "push %s HEAD:refs/heads/master" % remote_url,
+ env=ssh_env)
+ git_command(repo_path,
+ "push --tags %s" % remote_url, env=ssh_env)
+ finally:
+ run_command("rm -fr %s" % tmpdir)
+
+ if 'acl_config' in section:
+ tmpdir = tempfile.mkdtemp()
+ try:
+ repo_path = os.path.join(tmpdir, 'repo')
+ ret, _ = run_command_status("git init %s" % repo_path)
+ if ret != 0:
+ continue
+ if (fetch_config(project, remote_url, repo_path, ssh_env) and
+ copy_acl_config(project, repo_path,
+ section['acl_config']) and
+ create_groups_file(project, gerrit, repo_path)):
+ push_acl_config(project, remote_url, repo_path, ssh_env)
+ finally:
+ run_command("rm -fr %s" % tmpdir)
+finally:
+ os.unlink(ssh_env['GIT_SSH'])