| #! /usr/bin/env python |
| # Copyright (C) 2011 OpenStack, LLC. |
| # Copyright (c) 2012 Hewlett-Packard Development Company, L.P. |
| # |
| # Licensed under the Apache License, Version 2.0 (the "License"); |
| # you may not use this file except in compliance with the License. |
| # You may obtain a copy of the License at |
| # |
| # http://www.apache.org/licenses/LICENSE-2.0 |
| # |
| # Unless required by applicable law or agreed to in writing, software |
| # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT |
| # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the |
| # License for the specific language governing permissions and limitations |
| # under the License. |
| |
| # manage_projects.py reads a project config file called projects.yaml |
| # It should look like: |
| |
| # - homepage: http://openstack.org |
| # gerrit-host: review.openstack.org |
| # local-git-dir: /var/lib/git |
| # gerrit-key: /home/gerrit2/review_site/etc/ssh_host_rsa_key |
| # gerrit-committer: Project Creator <openstack-infra@lists.openstack.org> |
| # has-github: True |
| # has-wiki: False |
| # has-issues: False |
| # has-downloads: False |
| # acl-dir: /home/gerrit2/acls |
| # acl-base: /home/gerrit2/acls/project.config |
| # --- |
| # - project: PROJECT_NAME |
| # options: |
| # - has-wiki |
| # - has-issues |
| # - has-downloads |
| # - has-pull-requests |
| # - track-upstream |
| # homepage: Some homepage that isn't http://openstack.org |
| # description: This is a great project |
| # upstream: https://gerrit.googlesource.com/gerrit |
| # upstream-prefix: upstream |
| # acl-config: /path/to/gerrit/project.config |
| # acl-append: |
| # - /path/to/gerrit/project.config |
| # acl-parameters: |
| # project: OTHER_PROJECT_NAME |
| |
| import argparse |
| import ConfigParser |
| import logging |
| import os |
| import re |
| import shlex |
| import subprocess |
| import tempfile |
| import time |
| import yaml |
| |
| import gerritlib.gerrit |
| import github |
| |
| import jeepyb.gerritdb |
| |
| log = logging.getLogger("manage_projects") |
| |
| |
| def run_command(cmd, status=False, env={}): |
| cmd_list = shlex.split(str(cmd)) |
| newenv = os.environ |
| newenv.update(env) |
| log.debug("Executing command: %s" % " ".join(cmd_list)) |
| p = subprocess.Popen(cmd_list, stdout=subprocess.PIPE, |
| stderr=subprocess.STDOUT, env=newenv) |
| (out, nothing) = p.communicate() |
| log.debug("Return code: %s" % p.returncode) |
| log.debug("Command said: %s" % out.strip()) |
| if status: |
| return (p.returncode, out.strip()) |
| return out.strip() |
| |
| |
| def run_command_status(cmd, env={}): |
| return run_command(cmd, True, env) |
| |
| |
| def git_command(repo_dir, sub_cmd, env={}): |
| git_dir = os.path.join(repo_dir, '.git') |
| cmd = "git --git-dir=%s --work-tree=%s %s" % (git_dir, repo_dir, sub_cmd) |
| status, _ = run_command(cmd, True, env) |
| return status |
| |
| |
| def git_command_output(repo_dir, sub_cmd, env={}): |
| git_dir = os.path.join(repo_dir, '.git') |
| cmd = "git --git-dir=%s --work-tree=%s %s" % (git_dir, repo_dir, sub_cmd) |
| status, out = run_command(cmd, True, env) |
| return (status, out) |
| |
| |
| def write_acl_config(project, acl_dir, acl_base, acl_append, parameters): |
| project_parts = os.path.split(project) |
| if len(project_parts) > 1: |
| repo_base = os.path.join(acl_dir, *project_parts[:-1]) |
| if not os.path.exists(repo_base): |
| os.makedirs(repo_base) |
| if not os.path.isdir(repo_base): |
| return 1 |
| project = project_parts[-1] |
| config_file = os.path.join(repo_base, "%s.config" % project) |
| else: |
| config_file = os.path.join(acl_dir, "%s.config" % project) |
| if 'project' not in parameters: |
| parameters['project'] = project |
| with open(config_file, 'w') as config: |
| if acl_base and os.path.exists(acl_base): |
| config.write(open(acl_base, 'r').read()) |
| for acl_snippet in acl_append: |
| if not os.path.exists(acl_snippet): |
| acl_snippet = os.path.join(acl_dir, acl_snippet) |
| if not os.path.exists(acl_snippet): |
| continue |
| with open(acl_snippet, 'r') as append_content: |
| config.write(append_content.read() % parameters) |
| |
| |
| def fetch_config(project, remote_url, repo_path, env={}): |
| status = git_command(repo_path, "fetch %s +refs/meta/config:" |
| "refs/remotes/gerrit-meta/config" % remote_url, env) |
| if status != 0: |
| log.error("Failed to fetch refs/meta/config for project: %s" % project) |
| return False |
| # Because the following fails if executed more than once you should only |
| # run fetch_config once in each repo. |
| status = git_command(repo_path, "checkout -B config " |
| "remotes/gerrit-meta/config") |
| if status != 0: |
| log.error("Failed to checkout config for project: %s" % project) |
| return False |
| |
| return True |
| |
| |
| def copy_acl_config(project, repo_path, acl_config): |
| if not os.path.exists(acl_config): |
| return False |
| |
| acl_dest = os.path.join(repo_path, "project.config") |
| status, _ = run_command("cp %s %s" % |
| (acl_config, acl_dest), status=True) |
| if status == 0: |
| status = git_command(repo_path, "diff --quiet") |
| if status != 0: |
| return True |
| return False |
| |
| |
| def push_acl_config(project, remote_url, repo_path, gitid, env={}): |
| cmd = "commit -a -m'Update project config.' --author='%s'" % gitid |
| status = git_command(repo_path, cmd) |
| if status != 0: |
| log.error("Failed to commit config for project: %s" % project) |
| return False |
| status, out = git_command_output(repo_path, |
| "push %s HEAD:refs/meta/config" % |
| remote_url, env) |
| if status != 0: |
| log.error("Failed to push config for project: %s" % project) |
| return False |
| return True |
| |
| |
| def _get_group_uuid(group): |
| cursor = jeepyb.gerritdb.connect().cursor() |
| query = "SELECT group_uuid FROM account_groups WHERE name = %s" |
| cursor.execute(query, group) |
| data = cursor.fetchone() |
| if data: |
| return data[0] |
| return None |
| |
| |
| def _wait_for_group(gerrit, group): |
| """Wait for up to 10 seconds for the group to be created.""" |
| for x in range(10): |
| groups = gerrit.listGroups() |
| if group in groups: |
| break |
| time.sleep(1) |
| |
| |
| def get_group_uuid(gerrit, group): |
| uuid = _get_group_uuid(group) |
| if uuid: |
| return uuid |
| gerrit.createGroup(group) |
| _wait_for_group(gerrit, group) |
| uuid = _get_group_uuid(group) |
| if uuid: |
| return uuid |
| return None |
| |
| |
| def create_groups_file(project, gerrit, repo_path): |
| acl_config = os.path.join(repo_path, "project.config") |
| group_file = os.path.join(repo_path, "groups") |
| uuids = {} |
| for line in open(acl_config, 'r'): |
| r = re.match(r'^\s+.*group\s+(.*)$', line) |
| if r: |
| group = r.group(1) |
| if group in uuids.keys(): |
| continue |
| uuid = get_group_uuid(gerrit, group) |
| if uuid: |
| uuids[group] = uuid |
| else: |
| log.error("Unable to get UUID for group %s." % group) |
| return False |
| if uuids: |
| with open(group_file, 'w') as fp: |
| for group, uuid in uuids.items(): |
| fp.write("%s\t%s\n" % (uuid, group)) |
| status = git_command(repo_path, "add groups") |
| if status != 0: |
| log.error("Failed to add groups file for project: %s" % project) |
| return False |
| return True |
| |
| |
| def make_ssh_wrapper(gerrit_user, gerrit_key): |
| (fd, name) = tempfile.mkstemp(text=True) |
| os.write(fd, '#!/bin/bash\n') |
| os.write(fd, |
| 'ssh -i %s -l %s -o "StrictHostKeyChecking no" $@\n' % |
| (gerrit_key, gerrit_user)) |
| os.close(fd) |
| os.chmod(name, 0o755) |
| return dict(GIT_SSH=name) |
| |
| |
| def create_github_project(defaults, options, project, description, homepage): |
| default_has_issues = defaults.get('has-issues', False) |
| default_has_downloads = defaults.get('has-downloads', False) |
| default_has_wiki = defaults.get('has-wiki', False) |
| has_issues = 'has-issues' in options or default_has_issues |
| has_downloads = 'has-downloads' in options or default_has_downloads |
| has_wiki = 'has-wiki' in options or default_has_wiki |
| |
| GITHUB_SECURE_CONFIG = defaults.get( |
| 'github-config', |
| '/etc/github/github-projects.secure.config') |
| |
| secure_config = ConfigParser.ConfigParser() |
| secure_config.read(GITHUB_SECURE_CONFIG) |
| |
| # Project creation doesn't work via oauth |
| ghub = github.Github(secure_config.get("github", "username"), |
| secure_config.get("github", "password")) |
| orgs = ghub.get_user().get_orgs() |
| orgs_dict = dict(zip([o.login.lower() for o in orgs], orgs)) |
| |
| # Find the project's repo |
| project_split = project.split('/', 1) |
| org_name = project_split[0] |
| if len(project_split) > 1: |
| repo_name = project_split[1] |
| else: |
| repo_name = project |
| |
| try: |
| org = orgs_dict[org_name.lower()] |
| except KeyError: |
| # We do not have control of this github org ignore the project. |
| return |
| try: |
| repo = org.get_repo(repo_name) |
| except github.GithubException: |
| repo = org.create_repo(repo_name, |
| homepage=homepage, |
| has_issues=has_issues, |
| has_downloads=has_downloads, |
| has_wiki=has_wiki) |
| if description: |
| repo.edit(repo_name, description=description) |
| if homepage: |
| repo.edit(repo_name, homepage=homepage) |
| |
| repo.edit(repo_name, has_issues=has_issues, |
| has_downloads=has_downloads, |
| has_wiki=has_wiki) |
| |
| if 'gerrit' not in [team.name for team in repo.get_teams()]: |
| teams = org.get_teams() |
| teams_dict = dict(zip([t.name.lower() for t in teams], teams)) |
| teams_dict['gerrit'].add_to_repos(repo) |
| |
| |
| # TODO(mordred): Inspect repo_dir:master for a description |
| # override |
| def find_description_override(repo_path): |
| return None |
| |
| |
| def main(): |
| parser = argparse.ArgumentParser(description='Manage projects') |
| parser.add_argument('-v', dest='verbose', action='store_true', |
| help='verbose output') |
| parser.add_argument('--nocleanup', action='store_true', |
| help='do not remove temp directories') |
| parser.add_argument('projects', metavar='project', nargs='*', |
| help='name of project(s) to process') |
| args = parser.parse_args() |
| |
| if args.verbose: |
| logging.basicConfig(level=logging.DEBUG) |
| else: |
| logging.basicConfig(level=logging.ERROR) |
| |
| PROJECTS_YAML = os.environ.get('PROJECTS_YAML', |
| '/home/gerrit2/projects.yaml') |
| configs = [config for config in yaml.load_all(open(PROJECTS_YAML))] |
| defaults = configs[0][0] |
| default_has_github = defaults.get('has-github', True) |
| |
| LOCAL_GIT_DIR = defaults.get('local-git-dir', '/var/lib/git') |
| JEEPYB_CACHE_DIR = defaults.get('jeepyb-cache-dir', '/var/lib/jeepyb') |
| ACL_DIR = defaults.get('acl-dir') |
| GERRIT_HOST = defaults.get('gerrit-host') |
| GERRIT_PORT = int(defaults.get('gerrit-port', '29418')) |
| GERRIT_USER = defaults.get('gerrit-user') |
| GERRIT_KEY = defaults.get('gerrit-key') |
| GERRIT_GITID = defaults.get('gerrit-committer') |
| GERRIT_SYSTEM_USER = defaults.get('gerrit-system-user', 'gerrit2') |
| GERRIT_SYSTEM_GROUP = defaults.get('gerrit-system-group', 'gerrit2') |
| |
| gerrit = gerritlib.gerrit.Gerrit('localhost', |
| GERRIT_USER, |
| GERRIT_PORT, |
| GERRIT_KEY) |
| project_list = gerrit.listProjects() |
| ssh_env = make_ssh_wrapper(GERRIT_USER, GERRIT_KEY) |
| try: |
| |
| for section in configs[1]: |
| project = section['project'] |
| if args.projects and project not in args.projects: |
| continue |
| options = section.get('options', dict()) |
| description = section.get('description', None) |
| homepage = section.get('homepage', defaults.get('homepage', None)) |
| upstream = section.get('upstream', None) |
| upstream_prefix = section.get('upstream-prefix', None) |
| track_upstream = 'track-upstream' in options |
| |
| project_git = "%s.git" % project |
| remote_url = "ssh://localhost:%s/%s" % (GERRIT_PORT, project) |
| |
| # Create the project in Gerrit first, since it will fail |
| # spectacularly if its project directory or local replica |
| # already exist on disk |
| project_created = False |
| if project not in project_list: |
| try: |
| gerrit.createProject(project) |
| project_created = True |
| except Exception: |
| log.exception( |
| "Exception creating %s in Gerrit." % project) |
| |
| # Create the repo for the local git mirror |
| git_mirror_path = os.path.join(LOCAL_GIT_DIR, project_git) |
| if not os.path.exists(git_mirror_path): |
| run_command("git --bare init %s" % git_mirror_path) |
| run_command("chown -R %s:%s %s" |
| % (GERRIT_SYSTEM_USER, GERRIT_SYSTEM_GROUP, |
| git_mirror_path)) |
| |
| # Start processing the local JEEPYB cache |
| repo_path = os.path.join(JEEPYB_CACHE_DIR, project) |
| git_opts = dict(upstream=upstream, |
| repo_path=repo_path, |
| remote_url=remote_url) |
| |
| # If we don't have a local copy already, get one |
| if not os.path.exists(repo_path) or project_created: |
| if not os.path.exists(os.path.dirname(repo_path)): |
| os.makedirs(os.path.dirname(repo_path)) |
| # Three choices |
| # - If gerrit has it, get from gerrit |
| # - If gerrit doesn't have it: |
| # - If it has an upstream, clone that |
| # - If it doesn't, create it |
| |
| # Gerrit knows about the project, clone it |
| if project in project_list: |
| run_command( |
| "git clone %(remote_url)s %(repo_path)s" % git_opts, |
| env=ssh_env) |
| if upstream: |
| git_command( |
| repo_path, |
| "remote add -f upstream %(upstream)s" % git_opts) |
| |
| # Gerrit doesn't have it, but it has an upstream configured |
| # We're probably importing it for the first time, clone |
| # upstream, but then ongoing we want gerrit to ge origin |
| # and upstream to be only there for ongoing tracking |
| # purposes, so rename origin to upstream and add a new |
| # origin remote that points at gerrit |
| elif upstream: |
| run_command( |
| "git clone %(upstream)s %(repo_path)s" % git_opts, |
| env=ssh_env) |
| git_command( |
| repo_path, |
| "fetch origin +refs/heads/*:refs/copy/heads/*", |
| env=ssh_env) |
| git_command(repo_path, "remote rename origin upstream") |
| git_command( |
| repo_path, |
| "remote add origin %(remote_url)s" % git_opts) |
| push_string = "push %s +refs/copy/heads/*:refs/heads/*" |
| |
| # Neither gerrit has it, nor does it have an upstream, |
| # just create a whole new one |
| else: |
| run_command("git init %s" % repo_path) |
| git_command( |
| repo_path, |
| "remote add origin %(remote_url)s" % git_opts) |
| with open(os.path.join(repo_path, |
| ".gitreview"), |
| 'w') as gitreview: |
| gitreview.write("""[gerrit] |
| host=%s |
| port=%s |
| project=%s |
| """ % (GERRIT_HOST, GERRIT_PORT, project_git)) |
| git_command(repo_path, "add .gitreview") |
| cmd = ("commit -a -m'Added .gitreview' --author='%s'" |
| % GERRIT_GITID) |
| git_command(repo_path, cmd) |
| push_string = "push %s HEAD:refs/heads/master" |
| |
| # We do have a local copy of it already, make sure it's |
| # in shape to have work done. |
| else: |
| if project in project_list: |
| |
| # If we're configured to track upstream but the repo |
| # does not have an upstream remote, add one |
| if (track_upstream and |
| 'upstream' not in git_command_output( |
| repo_path, 'remote')[1]): |
| git_command( |
| repo_path, |
| "remote add upstream %(upstream)s" % git_opts) |
| |
| # If we're configured to track upstream, make sure that |
| # the upstream URL matches the config |
| else: |
| git_command( |
| repo_path, |
| "remote set-url upstream %(upstream)s" % git_opts) |
| |
| # Now that we have any upstreams configured, fetch all |
| # of the refs we might need, pruning remote branches |
| # that no longer exist |
| git_command( |
| repo_path, "remote update --prune", env=ssh_env) |
| |
| # TODO(mordred): This is here so that later we can |
| # inspect the master branch for meta-info |
| # Checkout master and reset to the state of origin/master |
| git_command(repo_path, "checkout -B master origin/master") |
| |
| description = find_description_override(repo_path) or description |
| |
| if 'has-github' in options or default_has_github: |
| create_github_project(defaults, options, project, |
| description, homepage) |
| |
| if project_created: |
| try: |
| git_command(repo_path, |
| push_string % remote_url, |
| env=ssh_env) |
| git_command(repo_path, |
| "push --tags %s" % remote_url, env=ssh_env) |
| except Exception: |
| log.exception( |
| "Error pushing %s to Gerrit." % project) |
| |
| # If we're configured to track upstream, make sure we have |
| # upstream's refs, and then push them to the appropriate |
| # branches in gerrit |
| if track_upstream: |
| git_command( |
| repo_path, |
| "remote update upstream --prune", env=ssh_env) |
| # Any branch that exists in the upstream remote, we want |
| # a local branch of, optionally prefixed with the |
| # upstream prefix value |
| for branch in git_command_output( |
| repo_path, "branch -a")[1].split(): |
| if not branch.strip().startswith("remotes/upstream"): |
| continue |
| if "->" in branch: |
| continue |
| local_branch = branch[len('remotes/upstream/'):] |
| if upstream_prefix: |
| local_branch = "%s/%s" % ( |
| upstream_prefix, local_branch) |
| |
| # Check out an up to date copy of the branch, so that |
| # we can push it and it will get picked up below |
| git_command(repo_path, "checkout -B %s %s" % ( |
| local_branch, branch)) |
| |
| try: |
| # Push all of the local branches to similarly named |
| # Branches on gerrit. Also, push all of the tags |
| git_command( |
| repo_path, |
| "push origin refs/heads/*:refs/heads/*", |
| env=ssh_env) |
| git_command(repo_path, 'push origin --tags', env=ssh_env) |
| except Exception: |
| log.exception( |
| "Error pushing %s to Gerrit." % project) |
| |
| try: |
| acl_config = section.get('acl-config', |
| '%s.config' % os.path.join(ACL_DIR, |
| project)) |
| except AttributeError: |
| acl_config = None |
| |
| if acl_config: |
| if not os.path.isfile(acl_config): |
| write_acl_config(project, |
| ACL_DIR, |
| section.get('acl-base', None), |
| section.get('acl-append', []), |
| section.get('acl-parameters', {})) |
| try: |
| if (fetch_config(project, |
| remote_url, |
| repo_path, |
| ssh_env) and |
| copy_acl_config(project, repo_path, |
| acl_config) and |
| create_groups_file(project, gerrit, repo_path)): |
| push_acl_config(project, |
| remote_url, |
| repo_path, |
| GERRIT_GITID, |
| ssh_env) |
| except Exception: |
| log.exception( |
| "Exception processing ACLS for %s." % project) |
| finally: |
| git_command(repo_path, 'reset --hard') |
| git_command(repo_path, 'checkout master') |
| git_command(repo_path, 'branch -D config') |
| |
| finally: |
| os.unlink(ssh_env['GIT_SSH']) |
| |
| if __name__ == "__main__": |
| main() |