#! /usr/bin/env python
# Copyright (C) 2011 OpenStack, LLC.
# Copyright (c) 2012 Hewlett-Packard Development Company, L.P.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.

# manage_projects.py reads a project config file called projects.yaml
# It should look like:

# - homepage: http://openstack.org
#   gerrit-host: review.openstack.org
#   local-git-dir: /var/lib/git
#   gerrit-key: /home/gerrit2/review_site/etc/ssh_host_rsa_key
#   gerrit-committer: Project Creator <openstack-infra@lists.openstack.org>
#   has-github: True
#   has-wiki: False
#   has-issues: False
#   has-downloads: False
#   acl-dir: /home/gerrit2/acls
#   acl-base: /home/gerrit2/acls/project.config
# ---
# - project: PROJECT_NAME
#   options:
#    - has-wiki
#    - has-issues
#    - has-downloads
#    - has-pull-requests
#    - track-upstream
#   homepage: Some homepage that isn't http://openstack.org
#   description: This is a great project
#   upstream: https://gerrit.googlesource.com/gerrit
#   upstream-prefix: upstream
#   acl-config: /path/to/gerrit/project.config
#   acl-append:
#     - /path/to/gerrit/project.config
#   acl-parameters:
#     project: OTHER_PROJECT_NAME

import argparse
import ConfigParser
import logging
import os
import re
import shlex
import subprocess
import tempfile
import time
import yaml

import gerritlib.gerrit
import github

import jeepyb.gerritdb

log = logging.getLogger("manage_projects")


def run_command(cmd, status=False, env={}):
    cmd_list = shlex.split(str(cmd))
    newenv = os.environ
    newenv.update(env)
    log.debug("Executing command: %s" % " ".join(cmd_list))
    p = subprocess.Popen(cmd_list, stdout=subprocess.PIPE,
                         stderr=subprocess.STDOUT, env=newenv)
    (out, nothing) = p.communicate()
    log.debug("Return code: %s" % p.returncode)
    log.debug("Command said: %s" % out.strip())
    if status:
        return (p.returncode, out.strip())
    return out.strip()


def run_command_status(cmd, env={}):
    return run_command(cmd, True, env)


def git_command(repo_dir, sub_cmd, env={}):
    git_dir = os.path.join(repo_dir, '.git')
    cmd = "git --git-dir=%s --work-tree=%s %s" % (git_dir, repo_dir, sub_cmd)
    status, _ = run_command(cmd, True, env)
    return status


def git_command_output(repo_dir, sub_cmd, env={}):
    git_dir = os.path.join(repo_dir, '.git')
    cmd = "git --git-dir=%s --work-tree=%s %s" % (git_dir, repo_dir, sub_cmd)
    status, out = run_command(cmd, True, env)
    return (status, out)


def write_acl_config(project, acl_dir, acl_base, acl_append, parameters):
    project_parts = os.path.split(project)
    if len(project_parts) > 1:
        repo_base = os.path.join(acl_dir, *project_parts[:-1])
        if not os.path.exists(repo_base):
            os.makedirs(repo_base)
        if not os.path.isdir(repo_base):
            return 1
        project = project_parts[-1]
        config_file = os.path.join(repo_base, "%s.config" % project)
    else:
        config_file = os.path.join(acl_dir, "%s.config" % project)
    if 'project' not in parameters:
        parameters['project'] = project
    with open(config_file, 'w') as config:
        if acl_base and os.path.exists(acl_base):
            config.write(open(acl_base, 'r').read())
        for acl_snippet in acl_append:
            if not os.path.exists(acl_snippet):
                acl_snippet = os.path.join(acl_dir, acl_snippet)
            if not os.path.exists(acl_snippet):
                continue
            with open(acl_snippet, 'r') as append_content:
                config.write(append_content.read() % parameters)


def fetch_config(project, remote_url, repo_path, env={}):
    status = git_command(repo_path, "fetch %s +refs/meta/config:"
                         "refs/remotes/gerrit-meta/config" % remote_url, env)
    if status != 0:
        log.error("Failed to fetch refs/meta/config for project: %s" % project)
        return False
    # Because the following fails if executed more than once you should only
    # run fetch_config once in each repo.
    status = git_command(repo_path, "checkout -B config "
                         "remotes/gerrit-meta/config")
    if status != 0:
        log.error("Failed to checkout config for project: %s" % project)
        return False

    return True


def copy_acl_config(project, repo_path, acl_config):
    if not os.path.exists(acl_config):
        return False

    acl_dest = os.path.join(repo_path, "project.config")
    status, _ = run_command("cp %s %s" %
                            (acl_config, acl_dest), status=True)
    if status == 0:
        status = git_command(repo_path, "diff --quiet")
        if status != 0:
            return True
    return False


def push_acl_config(project, remote_url, repo_path, gitid, env={}):
    cmd = "commit -a -m'Update project config.' --author='%s'" % gitid
    status = git_command(repo_path, cmd)
    if status != 0:
        log.error("Failed to commit config for project: %s" % project)
        return False
    status, out = git_command_output(repo_path,
                                     "push %s HEAD:refs/meta/config" %
                                     remote_url, env)
    if status != 0:
        log.error("Failed to push config for project: %s" % project)
        return False
    return True


def _get_group_uuid(group):
    cursor = jeepyb.gerritdb.connect().cursor()
    query = "SELECT group_uuid FROM account_groups WHERE name = %s"
    cursor.execute(query, group)
    data = cursor.fetchone()
    if data:
        return data[0]
    return None


def _wait_for_group(gerrit, group):
    """Wait for up to 10 seconds for the group to be created."""
    for x in range(10):
        groups = gerrit.listGroups()
        if group in groups:
            break
        time.sleep(1)


def get_group_uuid(gerrit, group):
    uuid = _get_group_uuid(group)
    if uuid:
        return uuid
    gerrit.createGroup(group)
    _wait_for_group(gerrit, group)
    uuid = _get_group_uuid(group)
    if uuid:
        return uuid
    return None


def create_groups_file(project, gerrit, repo_path):
    acl_config = os.path.join(repo_path, "project.config")
    group_file = os.path.join(repo_path, "groups")
    uuids = {}
    for line in open(acl_config, 'r'):
        r = re.match(r'^\s+.*group\s+(.*)$', line)
        if r:
            group = r.group(1)
            if group in uuids.keys():
                continue
            uuid = get_group_uuid(gerrit, group)
            if uuid:
                uuids[group] = uuid
            else:
                log.error("Unable to get UUID for group %s." % group)
                return False
    if uuids:
        with open(group_file, 'w') as fp:
            for group, uuid in uuids.items():
                fp.write("%s\t%s\n" % (uuid, group))
    status = git_command(repo_path, "add groups")
    if status != 0:
        log.error("Failed to add groups file for project: %s" % project)
        return False
    return True


def make_ssh_wrapper(gerrit_user, gerrit_key):
    (fd, name) = tempfile.mkstemp(text=True)
    os.write(fd, '#!/bin/bash\n')
    os.write(fd,
             'ssh -i %s -l %s -o "StrictHostKeyChecking no" $@\n' %
             (gerrit_key, gerrit_user))
    os.close(fd)
    os.chmod(name, 0o755)
    return dict(GIT_SSH=name)


def create_github_project(defaults, options, project, description, homepage):
    default_has_issues = defaults.get('has-issues', False)
    default_has_downloads = defaults.get('has-downloads', False)
    default_has_wiki = defaults.get('has-wiki', False)
    has_issues = 'has-issues' in options or default_has_issues
    has_downloads = 'has-downloads' in options or default_has_downloads
    has_wiki = 'has-wiki' in options or default_has_wiki

    GITHUB_SECURE_CONFIG = defaults.get(
        'github-config',
        '/etc/github/github-projects.secure.config')

    secure_config = ConfigParser.ConfigParser()
    secure_config.read(GITHUB_SECURE_CONFIG)

    # Project creation doesn't work via oauth
    ghub = github.Github(secure_config.get("github", "username"),
                         secure_config.get("github", "password"))
    orgs = ghub.get_user().get_orgs()
    orgs_dict = dict(zip([o.login.lower() for o in orgs], orgs))

    # Find the project's repo
    project_split = project.split('/', 1)
    org_name = project_split[0]
    if len(project_split) > 1:
        repo_name = project_split[1]
    else:
        repo_name = project

    try:
        org = orgs_dict[org_name.lower()]
    except KeyError:
        # We do not have control of this github org ignore the project.
        return
    try:
        repo = org.get_repo(repo_name)
    except github.GithubException:
        repo = org.create_repo(repo_name,
                               homepage=homepage,
                               has_issues=has_issues,
                               has_downloads=has_downloads,
                               has_wiki=has_wiki)
    if description:
        repo.edit(repo_name, description=description)
    if homepage:
        repo.edit(repo_name, homepage=homepage)

    repo.edit(repo_name, has_issues=has_issues,
              has_downloads=has_downloads,
              has_wiki=has_wiki)

    if 'gerrit' not in [team.name for team in repo.get_teams()]:
        teams = org.get_teams()
        teams_dict = dict(zip([t.name.lower() for t in teams], teams))
        teams_dict['gerrit'].add_to_repos(repo)


# TODO(mordred): Inspect repo_dir:master for a description
#                override
def find_description_override(repo_path):
    return None


def make_local_copy(repo_path, project, project_list,
                    git_opts, ssh_env, upstream, GERRIT_HOST, GERRIT_PORT,
                    project_git, GERRIT_GITID):

    # Ensure that the base location exists
    if not os.path.exists(os.path.dirname(repo_path)):
        os.makedirs(os.path.dirname(repo_path))

    # Three choices
    #  - If gerrit has it, get from gerrit
    #  - If gerrit doesn't have it:
    #    - If it has an upstream, clone that
    #    - If it doesn't, create it

    # Gerrit knows about the project, clone it
    # TODO(mordred): there is a possible failure condition here
    #                we should consider 'gerrit has it' to be
    #                'gerrit repo has a master branch'
    if project in project_list:
        run_command(
            "git clone %(remote_url)s %(repo_path)s" % git_opts,
            env=ssh_env)
        if upstream:
            git_command(
                repo_path,
                "remote add -f upstream %(upstream)s" % git_opts)
        return None

    # Gerrit doesn't have it, but it has an upstream configured
    # We're probably importing it for the first time, clone
    # upstream, but then ongoing we want gerrit to ge origin
    # and upstream to be only there for ongoing tracking
    # purposes, so rename origin to upstream and add a new
    # origin remote that points at gerrit
    elif upstream:
        run_command(
            "git clone %(upstream)s %(repo_path)s" % git_opts,
            env=ssh_env)
        git_command(
            repo_path,
            "fetch origin +refs/heads/*:refs/copy/heads/*",
            env=ssh_env)
        git_command(repo_path, "remote rename origin upstream")
        git_command(
            repo_path,
            "remote add origin %(remote_url)s" % git_opts)
        return "push %s +refs/copy/heads/*:refs/heads/*"

    # Neither gerrit has it, nor does it have an upstream,
    # just create a whole new one
    else:
        run_command("git init %s" % repo_path)
        git_command(
            repo_path,
            "remote add origin %(remote_url)s" % git_opts)
        with open(os.path.join(repo_path,
                               ".gitreview"),
                  'w') as gitreview:
            gitreview.write("""[gerrit]
host=%s
port=%s
project=%s
""" % (GERRIT_HOST, GERRIT_PORT, project_git))
        git_command(repo_path, "add .gitreview")
        cmd = ("commit -a -m'Added .gitreview' --author='%s'"
               % GERRIT_GITID)
        git_command(repo_path, cmd)
        return "push %s HEAD:refs/heads/master"


def update_local_copy(repo_path, track_upstream, git_opts, ssh_env):
    # If we're configured to track upstream but the repo
    # does not have an upstream remote, add one
    if (track_upstream and
        'upstream' not in git_command_output(
            repo_path, 'remote')[1]):
        git_command(
            repo_path,
            "remote add upstream %(upstream)s" % git_opts)

    # If we're configured to track upstream, make sure that
    # the upstream URL matches the config
    else:
        git_command(
            repo_path,
            "remote set-url upstream %(upstream)s" % git_opts)

    # Now that we have any upstreams configured, fetch all of the refs
    # we might need, pruning remote branches that no longer exist
    git_command(
        repo_path, "remote update --prune", env=ssh_env)

    # TODO(mordred): This is here so that later we can
    # inspect the master branch for meta-info
    # Checkout master and reset to the state of origin/master
    git_command(repo_path, "checkout -B master origin/master")


def push_to_gerrit(repo_path, project, push_string, remote_url, ssh_env):
    try:
        git_command(repo_path, push_string % remote_url, env=ssh_env)
        git_command(repo_path, "push --tags %s" % remote_url, env=ssh_env)
    except Exception:
        log.exception(
            "Error pushing %s to Gerrit." % project)


def sync_upstream(repo_path, project, ssh_env, upstream_prefix):
    git_command(
        repo_path,
        "remote update upstream --prune", env=ssh_env)
    # Any branch that exists in the upstream remote, we want
    # a local branch of, optionally prefixed with the
    # upstream prefix value
    for branch in git_command_output(
            repo_path, "branch -a")[1].split():
        if not branch.strip().startswith("remotes/upstream"):
            continue
        if "->" in branch:
            continue
        local_branch = branch[len('remotes/upstream/'):]
        if upstream_prefix:
            local_branch = "%s/%s" % (
                upstream_prefix, local_branch)

        # Check out an up to date copy of the branch, so that
        # we can push it and it will get picked up below
        git_command(repo_path, "checkout -B %s %s" % (
            local_branch, branch))

    try:
        # Push all of the local branches to similarly named
        # Branches on gerrit. Also, push all of the tags
        git_command(
            repo_path,
            "push origin refs/heads/*:refs/heads/*",
            env=ssh_env)
        git_command(repo_path, 'push origin --tags', env=ssh_env)
    except Exception:
        log.exception(
            "Error pushing %s to Gerrit." % project)


def process_acls(acl_config, project, ACL_DIR, section,
                 remote_url, repo_path, ssh_env, gerrit, GERRIT_GITID):
    if not os.path.isfile(acl_config):
        write_acl_config(project,
                         ACL_DIR,
                         section.get('acl-base', None),
                         section.get('acl-append', []),
                         section.get('acl-parameters', {}))
    try:
        if (fetch_config(project,
                         remote_url,
                         repo_path,
                         ssh_env) and
            copy_acl_config(project, repo_path,
                            acl_config) and
                create_groups_file(project, gerrit, repo_path)):
            push_acl_config(project,
                            remote_url,
                            repo_path,
                            GERRIT_GITID,
                            ssh_env)
    except Exception:
        log.exception(
            "Exception processing ACLS for %s." % project)
    finally:
        git_command(repo_path, 'reset --hard')
        git_command(repo_path, 'checkout master')
        git_command(repo_path, 'branch -D config')


def create_gerrit_project(project, project_list, gerrit):
    if project not in project_list:
        try:
            gerrit.createProject(project)
            return True
        except Exception:
            log.exception(
                "Exception creating %s in Gerrit." % project)
            raise
    return False


def create_local_mirror(local_git_dir, project_git,
                        gerrit_system_user, gerrit_system_group):

    git_mirror_path = os.path.join(local_git_dir, project_git)
    if not os.path.exists(git_mirror_path):
        (ret, output) = run_command_status(
            "git --bare init %s" % git_mirror_path)
        if ret:
            run_command("rm -rf git_mirror_path")
            raise Exception(output)
        run_command("chown -R %s:%s %s"
                    % (gerrit_system_user, gerrit_system_group,
                       git_mirror_path))


def main():
    parser = argparse.ArgumentParser(description='Manage projects')
    parser.add_argument('-v', dest='verbose', action='store_true',
                        help='verbose output')
    parser.add_argument('--nocleanup', action='store_true',
                        help='do not remove temp directories')
    parser.add_argument('projects', metavar='project', nargs='*',
                        help='name of project(s) to process')
    args = parser.parse_args()

    if args.verbose:
        logging.basicConfig(level=logging.DEBUG)
    else:
        logging.basicConfig(level=logging.ERROR)

    PROJECTS_YAML = os.environ.get('PROJECTS_YAML',
                                   '/home/gerrit2/projects.yaml')
    configs = [config for config in yaml.load_all(open(PROJECTS_YAML))]
    defaults = configs[0][0]
    default_has_github = defaults.get('has-github', True)

    LOCAL_GIT_DIR = defaults.get('local-git-dir', '/var/lib/git')
    JEEPYB_CACHE_DIR = defaults.get('jeepyb-cache-dir', '/var/lib/jeepyb')
    ACL_DIR = defaults.get('acl-dir')
    GERRIT_HOST = defaults.get('gerrit-host')
    GERRIT_PORT = int(defaults.get('gerrit-port', '29418'))
    GERRIT_USER = defaults.get('gerrit-user')
    GERRIT_KEY = defaults.get('gerrit-key')
    GERRIT_GITID = defaults.get('gerrit-committer')
    GERRIT_SYSTEM_USER = defaults.get('gerrit-system-user', 'gerrit2')
    GERRIT_SYSTEM_GROUP = defaults.get('gerrit-system-group', 'gerrit2')

    gerrit = gerritlib.gerrit.Gerrit('localhost',
                                     GERRIT_USER,
                                     GERRIT_PORT,
                                     GERRIT_KEY)
    project_list = gerrit.listProjects()
    ssh_env = make_ssh_wrapper(GERRIT_USER, GERRIT_KEY)
    try:

        for section in configs[1]:
            project = section['project']
            if args.projects and project not in args.projects:
                continue

            try:
                # Figure out all of the options
                options = section.get('options', dict())
                description = section.get('description', None)
                homepage = section.get(
                    'homepage', defaults.get('homepage', None))
                upstream = section.get('upstream', None)
                upstream_prefix = section.get('upstream-prefix', None)
                track_upstream = 'track-upstream' in options
                repo_path = os.path.join(JEEPYB_CACHE_DIR, project)

                project_git = "%s.git" % project
                remote_url = "ssh://localhost:%s/%s" % (GERRIT_PORT, project)
                git_opts = dict(upstream=upstream,
                                repo_path=repo_path,
                                remote_url=remote_url)
                acl_config = section.get(
                    'acl-config',
                    '%s.config' % os.path.join(ACL_DIR, project))

                # Create the project in Gerrit first, since it will fail
                # spectacularly if its project directory or local replica
                # already exist on disk
                project_created = create_gerrit_project(
                    project, project_list, gerrit)

                # Create the repo for the local git mirror
                create_local_mirror(
                    LOCAL_GIT_DIR, project_git,
                    GERRIT_SYSTEM_USER, GERRIT_SYSTEM_GROUP)

                if not os.path.exists(repo_path) or project_created:
                    # We don't have a local copy already, get one

                    # Make Local repo
                    push_string = make_local_copy(
                        repo_path, project, project_list,
                        git_opts, ssh_env, upstream, GERRIT_HOST, GERRIT_PORT,
                        project_git, GERRIT_GITID)
                else:
                    # We do have a local copy of it already, make sure it's
                    # in shape to have work done.
                    update_local_copy(
                        repo_path, track_upstream, git_opts, ssh_env)

                description = (
                    find_description_override(repo_path) or description)

                if 'has-github' in options or default_has_github:
                    create_github_project(defaults, options, project,
                                          description, homepage)

                if project_created:
                    push_to_gerrit(
                        repo_path, project, push_string, remote_url, ssh_env)

                # If we're configured to track upstream, make sure we have
                # upstream's refs, and then push them to the appropriate
                # branches in gerrit
                if track_upstream:
                    sync_upstream(repo_path, project, ssh_env, upstream_prefix)

                if acl_config:
                    process_acls(
                        acl_config, project, ACL_DIR, section,
                        remote_url, repo_path, ssh_env, gerrit, GERRIT_GITID)

            except Exception:
                log.exception(
                    "Problems creating %s, moving on." % project)
                continue
    finally:
        os.unlink(ssh_env['GIT_SSH'])

if __name__ == "__main__":
    main()