manage_projects.py

#! /usr/bin/env python
# Copyright (C) 2011 OpenStack, LLC.
# Copyright (c) 2012 Hewlett-Packard Development Company, L.P.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.

# manage_projects.py reads a project config file called projects.yaml
# It should look like:

# - homepage: http://openstack.org
#   gerrit-host: review.openstack.org
#   local-git-dir: /var/lib/git
#   gerrit-key: /home/gerrit2/review_site/etc/ssh_host_rsa_key
#   has-wiki: False
#   has-issues: False
#   has-downloads: False
# ---
# - project: PROJECT_NAME
#   options:
#    - has-wiki
#    - has-issues
#    - has-downloads
#    - has-pull-requests
#   homepage: Some homepage that isn't http://openstack.org
#   description: This is a great project
#   remote: https://gerrit.googlesource.com/gerrit
#   upstream: git://github.com/bushy/beards.git
#   acl_config: /path/to/gerrit/project.config


import ConfigParser
import logging
import os
import re
import shlex
import subprocess
import tempfile
import yaml

import github
import gerritlib.gerrit


logging.basicConfig(level=logging.ERROR)
log = logging.getLogger("manage_projects")


def run_command(cmd, status=False, env={}):
    cmd_list = shlex.split(str(cmd))
    newenv = os.environ
    newenv.update(env)
    log.debug("Executing command: %s" % " ".join(cmd_list))
    p = subprocess.Popen(cmd_list, stdout=subprocess.PIPE,
                         stderr=subprocess.STDOUT, env=newenv)
    (out, nothing) = p.communicate()
    log.debug("Return code: %s" % p.returncode)
    log.debug("Command said: %s" % out.strip())
    if status:
        return (p.returncode, out.strip())
    return out.strip()


def run_command_status(cmd, env={}):
    return run_command(cmd, True, env)


def git_command(repo_dir, sub_cmd, env={}):
    git_dir = os.path.join(repo_dir, '.git')
    cmd = "git --git-dir=%s --work-tree=%s %s" % (git_dir, repo_dir, sub_cmd)
    status, _ = run_command(cmd, True, env)
    return status


def git_command_output(repo_dir, sub_cmd, env={}):
    git_dir = os.path.join(repo_dir, '.git')
    cmd = "git --git-dir=%s --work-tree=%s %s" % (git_dir, repo_dir, sub_cmd)
    status, out = run_command(cmd, True, env)
    return (status, out)


def fetch_config(project, remote_url, repo_path, env={}):
    status = git_command(repo_path, "fetch %s +refs/meta/config:"
                         "refs/remotes/gerrit-meta/config" % remote_url, env)
    if status != 0:
        print "Failed to fetch refs/meta/config for project: %s" % project
        return False
    # Because the following fails if executed more than once you should only
    # run fetch_config once in each repo.
    status = git_command(repo_path, "checkout -b config "
                         "remotes/gerrit-meta/config")
    if status != 0:
        print "Failed to checkout config for project: %s" % project
        return False

    return True


def copy_acl_config(project, repo_path, acl_config):
    if not os.path.exists(acl_config):
        return False

    acl_dest = os.path.join(repo_path, "project.config")
    status, _ = run_command("cp %s %s" %
                            (acl_config, acl_dest), status=True)
    if status == 0:
        status = git_command(repo_path, "diff-index --quiet HEAD --")
        if status != 0:
            return True
    return False


def push_acl_config(project, remote_url, repo_path, env={}):
    cmd = "commit -a -m'Update project config.' --author='Openstack Project " \
            "Creator <openstack-infra@lists.openstack.org>'"
    status = git_command(repo_path, cmd)
    if status != 0:
        print "Failed to commit config for project: %s" % project
        return False
    status, out = git_command_output(repo_path,
                         "push %s HEAD:refs/meta/config" %
                         remote_url, env)
    if status != 0:
        print "Failed to push config for project: %s" % project
        print out
        return False
    return True


def _get_group_uuid(gerrit, group):
    query = "select group_uuid from account_groups where name = '%s'" % group
    data = gerrit.dbQuery(query)
    if data:
        for row in data:
            if row["type"] == "row":
                return row["columns"]["group_uuid"]
    return None


def get_group_uuid(gerrit, group):
    uuid = _get_group_uuid(gerrit, group)
    if uuid:
        return uuid
    gerrit.createGroup(group)
    uuid = _get_group_uuid(gerrit, group)
    if uuid:
        return uuid
    return None


def create_groups_file(project, gerrit, repo_path):
    acl_config = os.path.join(repo_path, "project.config")
    group_file = os.path.join(repo_path, "groups")
    uuids = {}
    for line in open(acl_config, 'r'):
        r = re.match(r'^\s+.*group\s+(.*)$', line)
        if r:
            group = r.group(1)
            if group in uuids.keys():
                continue
            uuid = get_group_uuid(gerrit, group)
            if uuid:
                uuids[group] = uuid
            else:
                return False
    if uuids:
        with open(group_file, 'w') as fp:
            for group, uuid in uuids.items():
                fp.write("%s\t%s\n" % (uuid, group))
    status = git_command(repo_path, "add groups")
    if status != 0:
        print "Failed to add groups file for project: %s" % project
        return False
    return True


def make_ssh_wrapper(gerrit_user, gerrit_key):
    (fd, name) = tempfile.mkstemp(text=True)
    os.write(fd, '#!/bin/bash\n')
    os.write(fd,
             'ssh -i %s -l %s -o "StrictHostKeyChecking no" $@\n' %
             (gerrit_key, gerrit_user))
    os.close(fd)
    os.chmod(name, 755)
    return dict(GIT_SSH=name)


PROJECTS_YAML = os.environ.get('PROJECTS_YAML',
                               '/home/gerrit2/projects.yaml')
configs = [config for config in yaml.load_all(open(PROJECTS_YAML))]
defaults = configs[0][0]
default_has_issues = defaults.get('has-issues', False)
default_has_downloads = defaults.get('has-downloads', False)
default_has_wiki = defaults.get('has-wiki', False)

LOCAL_GIT_DIR = defaults.get('local-git-dir', '/var/lib/git')
GERRIT_HOST = defaults.get('gerrit-host')
GERRIT_USER = defaults.get('gerrit-user')
GERRIT_KEY = defaults.get('gerrit-key')
GITHUB_SECURE_CONFIG = defaults.get('github-config',
                               '/etc/github/github-projects.secure.config')

secure_config = ConfigParser.ConfigParser()
secure_config.read(GITHUB_SECURE_CONFIG)

# Project creation doesn't work via oauth
ghub = github.Github(secure_config.get("github", "username"),
                     secure_config.get("github", "password"))
orgs = ghub.get_user().get_orgs()
orgs_dict = dict(zip([o.login.lower() for o in orgs], orgs))

gerrit = gerritlib.gerrit.Gerrit('localhost',
                                 GERRIT_USER,
                                 29418,
                                 GERRIT_KEY)
project_list = gerrit.listProjects()
ssh_env = make_ssh_wrapper(GERRIT_USER, GERRIT_KEY)
try:

    for section in configs[1]:
        project = section['project']
        options = section.get('options', dict())
        description = section.get('description', None)
        homepage = section.get('homepage', defaults.get('homepage', None))
        upstream = section.get('upstream', None)

        project_git = "%s.git" % project
        project_dir = os.path.join(LOCAL_GIT_DIR, project_git)

        # Find the project's repo
        project_split = project.split('/', 1)
        if len(project_split) > 1:
            repo_name = project_split[1]
        else:
            repo_name = project
        has_issues = 'has-issues' in options or default_has_issues
        has_downloads = 'has-downloads' in options or default_has_downloads
        has_wiki = 'has-wiki' in options or default_has_wiki
        try:
            org = orgs_dict[project_split[0].lower()]
        except KeyError:
            # We do not have control of this github org ignore the project.
            continue
        try:
            repo = org.get_repo(repo_name)
        except github.GithubException:
            repo = org.create_repo(repo_name,
                                   homepage=homepage,
                                   has_issues=has_issues,
                                   has_downloads=has_downloads,
                                   has_wiki=has_wiki)
        if description:
            repo.edit(repo_name, description=description)
        if homepage:
            repo.edit(repo_name, homepage=homepage)

        repo.edit(repo_name, has_issues=has_issues,
                  has_downloads=has_downloads,
                  has_wiki=has_wiki)

        if 'gerrit' not in [team.name for team in repo.get_teams()]:
            teams = org.get_teams()
            teams_dict = dict(zip([t.name.lower() for t in teams], teams))
            teams_dict['gerrit'].add_to_repos(repo)

        remote_url = "ssh://localhost:29418/%s" % project
        if project not in project_list:
            tmpdir = tempfile.mkdtemp()
            try:
                repo_path = os.path.join(tmpdir, 'repo')
                if upstream:
                    run_command("git clone %(upstream)s %(repo_path)s" %
                                dict(upstream=upstream, repo_path=repo_path))
                else:
                    run_command("git init %s" % repo_path)
                    with open(os.path.join(repo_path,
                                           ".gitreview"), 'w') as gitreview:
                        gitreview.write("""
[gerrit]
host=%s
port=29418
project=%s
""" % (GERRIT_HOST, project_git))
                    git_command(repo_path, "add .gitreview")
                    cmd = "commit -a -m'Added .gitreview' --author=" \
                            "'Openstack Project Creator " \
                            "<openstack-infra@lists.openstack.org>'"
                    git_command(repo_path, cmd)
                gerrit.createProject(project)

                if not os.path.exists(project_dir):
                    run_command("git --bare init %s" % project_dir)
                    run_command("chown -R gerrit2:gerrit2 %s" % project_dir)

                git_command(repo_path,
                            "push --all %s" % remote_url,
                            env=ssh_env)
                git_command(repo_path,
                            "push --tags %s" % remote_url, env=ssh_env)
            finally:
                run_command("rm -fr %s" % tmpdir)

        if 'acl_config' in section:
            tmpdir = tempfile.mkdtemp()
            try:
                repo_path = os.path.join(tmpdir, 'repo')
                ret, _ = run_command_status("git init %s" % repo_path)
                if ret != 0:
                    continue
                if (fetch_config(project, remote_url, repo_path, ssh_env) and
                    copy_acl_config(project, repo_path,
                                    section['acl_config']) and
                    create_groups_file(project, gerrit, repo_path)):
                    push_acl_config(project, remote_url, repo_path, ssh_env)
            finally:
                run_command("rm -fr %s" % tmpdir)
finally:
    os.unlink(ssh_env['GIT_SSH'])