jeepyb/cmd/manage_projects.py

#! /usr/bin/env python
# Copyright (C) 2011 OpenStack, LLC.
# Copyright (c) 2012 Hewlett-Packard Development Company, L.P.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.

# manage_projects.py reads a project config file called projects.yaml
# It should look like:

# - homepage: http://openstack.org
#   gerrit-host: review.openstack.org
#   local-git-dir: /var/lib/git
#   gerrit-key: /home/gerrit2/review_site/etc/ssh_host_rsa_key
#   has-wiki: False
#   has-issues: False
#   has-downloads: False
#   acl-dir: /home/gerrit2/acls
#   acl-base: /home/gerrit2/acls/project.config
# ---
# - project: PROJECT_NAME
#   options:
#    - has-wiki
#    - has-issues
#    - has-downloads
#    - has-pull-requests
#   homepage: Some homepage that isn't http://openstack.org
#   description: This is a great project
#   remote: https://gerrit.googlesource.com/gerrit
#   upstream: git://github.com/bushy/beards.git
#   acl-config: /path/to/gerrit/project.config
#   acl-append:
#     - /path/to/gerrit/project.config
#   acl-parameters:
#     project: OTHER_PROJECT_NAME


import ConfigParser
import logging
import os
import re
import shlex
import subprocess
import tempfile
import yaml

import github
import gerritlib.gerrit

import jeepyb.gerritdb

logging.basicConfig(level=logging.ERROR)
log = logging.getLogger("manage_projects")


def run_command(cmd, status=False, env={}):
    cmd_list = shlex.split(str(cmd))
    newenv = os.environ
    newenv.update(env)
    log.debug("Executing command: %s" % " ".join(cmd_list))
    p = subprocess.Popen(cmd_list, stdout=subprocess.PIPE,
                         stderr=subprocess.STDOUT, env=newenv)
    (out, nothing) = p.communicate()
    log.debug("Return code: %s" % p.returncode)
    log.debug("Command said: %s" % out.strip())
    if status:
        return (p.returncode, out.strip())
    return out.strip()


def run_command_status(cmd, env={}):
    return run_command(cmd, True, env)


def git_command(repo_dir, sub_cmd, env={}):
    git_dir = os.path.join(repo_dir, '.git')
    cmd = "git --git-dir=%s --work-tree=%s %s" % (git_dir, repo_dir, sub_cmd)
    status, _ = run_command(cmd, True, env)
    return status


def git_command_output(repo_dir, sub_cmd, env={}):
    git_dir = os.path.join(repo_dir, '.git')
    cmd = "git --git-dir=%s --work-tree=%s %s" % (git_dir, repo_dir, sub_cmd)
    status, out = run_command(cmd, True, env)
    return (status, out)


def write_acl_config(project, acl_dir, acl_base, acl_append, parameters):
    project_parts = os.path.split(project)
    if len(project_parts) > 1:
        repo_base = os.path.join(acl_dir, *project_parts[:-1])
        if not os.path.exists(repo_base):
            os.makedirs(repo_base)
        if not os.path.isdir(repo_base):
            return 1
        project = project_parts[-1]
        config_file = os.path.join(repo_base, "%s.config" % project)
    else:
        config_file = os.path.join(acl_dir, "%s.config" % project)
    if 'project' not in parameters:
        parameters['project'] = project
    with open(config_file, 'w') as config:
        if acl_base and os.path.exists(acl_base):
            config.write(open(acl_base, 'r').read())
        for acl_snippet in acl_append:
            if not os.path.exists(acl_snippet):
                acl_snippet = os.path.join(acl_dir, acl_snippet)
            if not os.path.exists(acl_snippet):
                continue
            with open(acl_snippet, 'r') as append_content:
                config.write(append_content.read() % parameters)


def fetch_config(project, remote_url, repo_path, env={}):
    status = git_command(repo_path, "fetch %s +refs/meta/config:"
                         "refs/remotes/gerrit-meta/config" % remote_url, env)
    if status != 0:
        print "Failed to fetch refs/meta/config for project: %s" % project
        return False
    # Because the following fails if executed more than once you should only
    # run fetch_config once in each repo.
    status = git_command(repo_path, "checkout -b config "
                         "remotes/gerrit-meta/config")
    if status != 0:
        print "Failed to checkout config for project: %s" % project
        return False

    return True


def copy_acl_config(project, repo_path, acl_config):
    if not os.path.exists(acl_config):
        return False

    acl_dest = os.path.join(repo_path, "project.config")
    status, _ = run_command("cp %s %s" %
                            (acl_config, acl_dest), status=True)
    if status == 0:
        status = git_command(repo_path, "diff-index --quiet HEAD --")
        if status != 0:
            return True
    return False


def push_acl_config(project, remote_url, repo_path, env={}):
    cmd = "commit -a -m'Update project config.' --author='Openstack Project " \
          "Creator <openstack-infra@lists.openstack.org>'"
    status = git_command(repo_path, cmd)
    if status != 0:
        print "Failed to commit config for project: %s" % project
        return False
    status, out = git_command_output(repo_path,
                                     "push %s HEAD:refs/meta/config" %
                                     remote_url, env)
    if status != 0:
        print "Failed to push config for project: %s" % project
        print out
        return False
    return True


def _get_group_uuid(gerrit, group):
    cursor = jeepyb.gerritdb.connect().cursor()
    query = "SELECT group_uuid FROM account_groups WHERE name = %s"
    cursor.execute(query, group)
    data = cursor.fetchone()
    if data:
        return data[0]
    return None


def get_group_uuid(gerrit, group):
    uuid = _get_group_uuid(gerrit, group)
    if uuid:
        return uuid
    gerrit.createGroup(group, owner="Administrators")
    uuid = _get_group_uuid(gerrit, group)
    if uuid:
        return uuid
    return None


def create_groups_file(project, gerrit, repo_path):
    acl_config = os.path.join(repo_path, "project.config")
    group_file = os.path.join(repo_path, "groups")
    uuids = {}
    for line in open(acl_config, 'r'):
        r = re.match(r'^\s+.*group\s+(.*)$', line)
        if r:
            group = r.group(1)
            if group in uuids.keys():
                continue
            uuid = get_group_uuid(gerrit, group)
            if uuid:
                uuids[group] = uuid
            else:
                return False
    if uuids:
        with open(group_file, 'w') as fp:
            for group, uuid in uuids.items():
                fp.write("%s\t%s\n" % (uuid, group))
    status = git_command(repo_path, "add groups")
    if status != 0:
        print "Failed to add groups file for project: %s" % project
        return False
    return True


def make_ssh_wrapper(gerrit_user, gerrit_key):
    (fd, name) = tempfile.mkstemp(text=True)
    os.write(fd, '#!/bin/bash\n')
    os.write(fd,
             'ssh -i %s -l %s -o "StrictHostKeyChecking no" $@\n' %
             (gerrit_key, gerrit_user))
    os.close(fd)
    os.chmod(name, 0755)
    return dict(GIT_SSH=name)


def main():

    PROJECTS_YAML = os.environ.get('PROJECTS_YAML',
                                   '/home/gerrit2/projects.yaml')
    configs = [config for config in yaml.load_all(open(PROJECTS_YAML))]
    defaults = configs[0][0]
    default_has_issues = defaults.get('has-issues', False)
    default_has_downloads = defaults.get('has-downloads', False)
    default_has_wiki = defaults.get('has-wiki', False)

    LOCAL_GIT_DIR = defaults.get('local-git-dir', '/var/lib/git')
    ACL_DIR = defaults.get('acl-dir')
    GERRIT_HOST = defaults.get('gerrit-host')
    GERRIT_USER = defaults.get('gerrit-user')
    GERRIT_KEY = defaults.get('gerrit-key')

    GITHUB_SECURE_CONFIG = defaults.get(
        'github-config',
        '/etc/github/github-projects.secure.config')

    secure_config = ConfigParser.ConfigParser()
    secure_config.read(GITHUB_SECURE_CONFIG)

    # Project creation doesn't work via oauth
    ghub = github.Github(secure_config.get("github", "username"),
                         secure_config.get("github", "password"))
    orgs = ghub.get_user().get_orgs()
    orgs_dict = dict(zip([o.login.lower() for o in orgs], orgs))

    gerrit = gerritlib.gerrit.Gerrit('localhost',
                                     GERRIT_USER,
                                     29418,
                                     GERRIT_KEY)
    project_list = gerrit.listProjects()
    ssh_env = make_ssh_wrapper(GERRIT_USER, GERRIT_KEY)
    try:

        for section in configs[1]:
            project = section['project']
            options = section.get('options', dict())
            description = section.get('description', None)
            homepage = section.get('homepage', defaults.get('homepage', None))
            upstream = section.get('upstream', None)

            project_git = "%s.git" % project
            project_dir = os.path.join(LOCAL_GIT_DIR, project_git)

            # Find the project's repo
            project_split = project.split('/', 1)
            if len(project_split) > 1:
                repo_name = project_split[1]
            else:
                repo_name = project
            has_issues = 'has-issues' in options or default_has_issues
            has_downloads = 'has-downloads' in options or default_has_downloads
            has_wiki = 'has-wiki' in options or default_has_wiki
            try:
                org = orgs_dict[project_split[0].lower()]
            except KeyError:
                # We do not have control of this github org ignore the project.
                continue
            try:
                repo = org.get_repo(repo_name)
            except github.GithubException:
                repo = org.create_repo(repo_name,
                                       homepage=homepage,
                                       has_issues=has_issues,
                                       has_downloads=has_downloads,
                                       has_wiki=has_wiki)
            if description:
                repo.edit(repo_name, description=description)
            if homepage:
                repo.edit(repo_name, homepage=homepage)

            repo.edit(repo_name, has_issues=has_issues,
                      has_downloads=has_downloads,
                      has_wiki=has_wiki)

            if 'gerrit' not in [team.name for team in repo.get_teams()]:
                teams = org.get_teams()
                teams_dict = dict(zip([t.name.lower() for t in teams], teams))
                teams_dict['gerrit'].add_to_repos(repo)

            remote_url = "ssh://localhost:29418/%s" % project
            if project not in project_list:
                tmpdir = tempfile.mkdtemp()
                try:
                    repo_path = os.path.join(tmpdir, 'repo')
                    if upstream:
                        run_command("git clone %(upstream)s %(repo_path)s" %
                                    dict(upstream=upstream,
                                         repo_path=repo_path))
                        git_command(repo_path,
                                    "fetch origin "
                                    "+refs/heads/*:refs/copy/heads/*",
                                    env=ssh_env)
                        push_string = "push %s +refs/copy/heads/*:refs/heads/*"
                    else:
                        run_command("git init %s" % repo_path)
                        with open(os.path.join(repo_path,
                                               ".gitreview"),
                                  'w') as gitreview:
                            gitreview.write("""[gerrit]
host=%s
port=29418
project=%s
""" % (GERRIT_HOST, project_git))
                        git_command(repo_path, "add .gitreview")
                        cmd = "commit -a -m'Added .gitreview' --author=" \
                              "'Openstack Project Creator " \
                              "<openstack-infra@lists.openstack.org>'"
                        git_command(repo_path, cmd)
                        push_string = "push --all %s"
                    gerrit.createProject(project)

                    if not os.path.exists(project_dir):
                        run_command("git --bare init %s" % project_dir)
                        run_command("chown -R gerrit2:gerrit2 %s"
                                    % project_dir)

                    git_command(repo_path,
                                push_string % remote_url,
                                env=ssh_env)
                    git_command(repo_path,
                                "push --tags %s" % remote_url, env=ssh_env)
                finally:
                    run_command("rm -fr %s" % tmpdir)

            try:
                acl_config = section.get('acl-config',
                                         '%s.config' % os.path.join(ACL_DIR,
                                                                    project))
            except AttributeError:
                acl_config = None

            if acl_config:
                if not os.path.isfile(acl_config):
                    write_acl_config(project,
                                     ACL_DIR,
                                     section.get('acl-base', None),
                                     section.get('acl-append', []),
                                     section.get('acl-parameters', {}))
                tmpdir = tempfile.mkdtemp()
                try:
                    repo_path = os.path.join(tmpdir, 'repo')
                    ret, _ = run_command_status("git init %s" % repo_path)
                    if ret != 0:
                        continue
                    if (fetch_config(project,
                                     remote_url,
                                     repo_path,
                                     ssh_env) and
                        copy_acl_config(project, repo_path,
                                        acl_config) and
                            create_groups_file(project, gerrit, repo_path)):
                        push_acl_config(project,
                                        remote_url,
                                        repo_path,
                                        ssh_env)
                finally:
                    run_command("rm -fr %s" % tmpdir)
    finally:
        os.unlink(ssh_env['GIT_SSH'])