| #! /usr/bin/env python |
| # Copyright (C) 2011 OpenStack, LLC. |
| # Copyright (c) 2012 Hewlett-Packard Development Company, L.P. |
| # |
| # Licensed under the Apache License, Version 2.0 (the "License"); |
| # you may not use this file except in compliance with the License. |
| # You may obtain a copy of the License at |
| # |
| # http://www.apache.org/licenses/LICENSE-2.0 |
| # |
| # Unless required by applicable law or agreed to in writing, software |
| # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT |
| # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the |
| # License for the specific language governing permissions and limitations |
| # under the License. |
| |
| # manage_projects.py reads a config file called projects.ini |
| # It should look like: |
| |
| # [projects] |
| # homepage=http://openstack.org |
| # gerrit-host=review.openstack.org |
| # local-git-dir=/var/lib/git |
| # gerrit-key=/home/gerrit2/review_site/etc/ssh_host_rsa_key |
| # gerrit-committer=Project Creator <openstack-infra@lists.openstack.org> |
| # gerrit-replicate=True |
| # has-github=True |
| # has-wiki=False |
| # has-issues=False |
| # has-downloads=False |
| # acl-dir=/home/gerrit2/acls |
| # acl-base=/home/gerrit2/acls/project.config |
| # |
| # manage_projects.py reads a project listing file called projects.yaml |
| # It should look like: |
| # - project: PROJECT_NAME |
| # options: |
| # - has-wiki |
| # - has-issues |
| # - has-downloads |
| # - has-pull-requests |
| # - track-upstream |
| # homepage: Some homepage that isn't http://openstack.org |
| # description: This is a great project |
| # upstream: https://gerrit.googlesource.com/gerrit |
| # upstream-prefix: upstream |
| # acl-config: /path/to/gerrit/project.config |
| # acl-append: |
| # - /path/to/gerrit/project.config |
| # acl-parameters: |
| # project: OTHER_PROJECT_NAME |
| |
| import argparse |
| import ConfigParser |
| import logging |
| import os |
| import re |
| import shlex |
| import subprocess |
| import tempfile |
| import time |
| |
| import gerritlib.gerrit |
| import github |
| |
| import jeepyb.gerritdb |
| import jeepyb.utils as u |
| |
| registry = u.ProjectsRegistry() |
| |
| log = logging.getLogger("manage_projects") |
| |
| |
| class FetchConfigException(Exception): |
| pass |
| |
| |
| class CopyACLException(Exception): |
| pass |
| |
| |
| class CreateGroupException(Exception): |
| pass |
| |
| |
| def run_command(cmd, status=False, env=None): |
| env = env or {} |
| cmd_list = shlex.split(str(cmd)) |
| newenv = os.environ |
| newenv.update(env) |
| log.info("Executing command: %s" % " ".join(cmd_list)) |
| p = subprocess.Popen(cmd_list, stdout=subprocess.PIPE, |
| stderr=subprocess.STDOUT, env=newenv) |
| (out, nothing) = p.communicate() |
| log.debug("Return code: %s" % p.returncode) |
| log.debug("Command said: %s" % out.strip()) |
| if status: |
| return (p.returncode, out.strip()) |
| return out.strip() |
| |
| |
| def run_command_status(cmd, env=None): |
| env = env or {} |
| return run_command(cmd, True, env) |
| |
| |
| def git_command(repo_dir, sub_cmd, env=None): |
| env = env or {} |
| git_dir = os.path.join(repo_dir, '.git') |
| cmd = "git --git-dir=%s --work-tree=%s %s" % (git_dir, repo_dir, sub_cmd) |
| status, _ = run_command(cmd, True, env) |
| return status |
| |
| |
| def git_command_output(repo_dir, sub_cmd, env=None): |
| env = env or {} |
| git_dir = os.path.join(repo_dir, '.git') |
| cmd = "git --git-dir=%s --work-tree=%s %s" % (git_dir, repo_dir, sub_cmd) |
| status, out = run_command(cmd, True, env) |
| return (status, out) |
| |
| |
| def fetch_config(project, remote_url, repo_path, env=None): |
| env = env or {} |
| # Poll for refs/meta/config as gerrit may not have written it out for |
| # us yet. |
| for x in range(10): |
| status = git_command(repo_path, "fetch %s +refs/meta/config:" |
| "refs/remotes/gerrit-meta/config" % |
| remote_url, env) |
| if status == 0: |
| break |
| else: |
| log.debug("Failed to fetch refs/meta/config for project: %s" % |
| project) |
| time.sleep(2) |
| if status != 0: |
| log.error("Failed to fetch refs/meta/config for project: %s" % project) |
| raise FetchConfigException() |
| |
| # Poll for project.config as gerrit may not have committed an empty |
| # one yet. |
| output = "" |
| for x in range(10): |
| status = git_command(repo_path, "remote update --prune", env) |
| if status != 0: |
| log.error("Failed to update remote: %s" % remote_url) |
| time.sleep(2) |
| continue |
| else: |
| status, output = git_command_output( |
| repo_path, "ls-files --with-tree=remotes/gerrit-meta/config " |
| "project.config", env) |
| if output.strip() != "project.config" or status != 0: |
| log.debug("Failed to find project.config for project: %s" % |
| project) |
| time.sleep(2) |
| else: |
| break |
| if output.strip() != "project.config" or status != 0: |
| log.error("Failed to find project.config for project: %s" % project) |
| raise FetchConfigException() |
| |
| # Because the following fails if executed more than once you should only |
| # run fetch_config once in each repo. |
| status = git_command(repo_path, "checkout -B config " |
| "remotes/gerrit-meta/config") |
| if status != 0: |
| log.error("Failed to checkout config for project: %s" % project) |
| raise FetchConfigException() |
| |
| |
| def copy_acl_config(project, repo_path, acl_config): |
| if not os.path.exists(acl_config): |
| raise CopyACLException() |
| |
| acl_dest = os.path.join(repo_path, "project.config") |
| status, _ = run_command("cp %s %s" % |
| (acl_config, acl_dest), status=True) |
| if status != 0: |
| raise CopyACLException() |
| |
| status = git_command(repo_path, "diff --quiet") |
| return status != 0 |
| |
| |
| def push_acl_config(project, remote_url, repo_path, gitid, env=None): |
| env = env or {} |
| cmd = "commit -a -m'Update project config.' --author='%s'" % gitid |
| status = git_command(repo_path, cmd) |
| if status != 0: |
| log.error("Failed to commit config for project: %s" % project) |
| return False |
| status, out = git_command_output(repo_path, |
| "push %s HEAD:refs/meta/config" % |
| remote_url, env) |
| if status != 0: |
| log.error("Failed to push config for project: %s" % project) |
| return False |
| return True |
| |
| |
| def _get_group_uuid(group): |
| """Wait for up to 10 seconds for the group to be created in the DB.""" |
| query = "SELECT group_uuid FROM account_groups WHERE name = %s" |
| con = jeepyb.gerritdb.connect() |
| for x in range(10): |
| cursor = con.cursor() |
| cursor.execute(query, (group,)) |
| data = cursor.fetchone() |
| cursor.close() |
| con.commit() |
| if data: |
| return data[0] |
| time.sleep(1) |
| return None |
| |
| |
| def get_group_uuid(gerrit, group): |
| uuid = _get_group_uuid(group) |
| if uuid: |
| return uuid |
| gerrit.createGroup(group) |
| uuid = _get_group_uuid(group) |
| if uuid: |
| return uuid |
| return None |
| |
| |
| def create_groups_file(project, gerrit, repo_path): |
| acl_config = os.path.join(repo_path, "project.config") |
| group_file = os.path.join(repo_path, "groups") |
| uuids = {} |
| for line in open(acl_config, 'r'): |
| r = re.match(r'^.*\sgroup\s+(.*)$', line) |
| if r: |
| group = r.group(1) |
| if group in uuids.keys(): |
| continue |
| uuid = get_group_uuid(gerrit, group) |
| if uuid: |
| uuids[group] = uuid |
| else: |
| log.error("Unable to get UUID for group %s." % group) |
| raise CreateGroupException() |
| if uuids: |
| with open(group_file, 'w') as fp: |
| for group, uuid in uuids.items(): |
| fp.write("%s\t%s\n" % (uuid, group)) |
| status = git_command(repo_path, "add groups") |
| if status != 0: |
| log.error("Failed to add groups file for project: %s" % project) |
| raise CreateGroupException() |
| |
| |
| def make_ssh_wrapper(gerrit_user, gerrit_key): |
| (fd, name) = tempfile.mkstemp(text=True) |
| os.write(fd, '#!/bin/bash\n') |
| os.write(fd, |
| 'ssh -i %s -l %s -o "StrictHostKeyChecking no" $@\n' % |
| (gerrit_key, gerrit_user)) |
| os.close(fd) |
| os.chmod(name, 0o755) |
| return dict(GIT_SSH=name) |
| |
| |
| def create_github_project( |
| default_has_issues, default_has_downloads, default_has_wiki, |
| github_secure_config, options, project, description, homepage): |
| created = False |
| has_issues = 'has-issues' in options or default_has_issues |
| has_downloads = 'has-downloads' in options or default_has_downloads |
| has_wiki = 'has-wiki' in options or default_has_wiki |
| |
| secure_config = ConfigParser.ConfigParser() |
| secure_config.read(github_secure_config) |
| |
| # Project creation doesn't work via oauth |
| ghub = github.Github(secure_config.get("github", "username"), |
| secure_config.get("github", "password")) |
| orgs = ghub.get_user().get_orgs() |
| orgs_dict = dict(zip([o.login.lower() for o in orgs], orgs)) |
| |
| # Find the project's repo |
| project_split = project.split('/', 1) |
| org_name = project_split[0] |
| if len(project_split) > 1: |
| repo_name = project_split[1] |
| else: |
| repo_name = project |
| |
| try: |
| org = orgs_dict[org_name.lower()] |
| except KeyError: |
| # We do not have control of this github org ignore the project. |
| return False |
| try: |
| repo = org.get_repo(repo_name) |
| except github.GithubException: |
| repo = org.create_repo(repo_name, |
| homepage=homepage, |
| has_issues=has_issues, |
| has_downloads=has_downloads, |
| has_wiki=has_wiki) |
| if description: |
| repo.edit(repo_name, description=description) |
| if homepage: |
| repo.edit(repo_name, homepage=homepage) |
| repo.edit(repo_name, has_issues=has_issues, |
| has_downloads=has_downloads, |
| has_wiki=has_wiki) |
| |
| if 'gerrit' not in [team.name for team in repo.get_teams()]: |
| teams = org.get_teams() |
| teams_dict = dict(zip([t.name.lower() for t in teams], teams)) |
| teams_dict['gerrit'].add_to_repos(repo) |
| created = True |
| |
| return created |
| |
| |
| # TODO(mordred): Inspect repo_dir:master for a description |
| # override |
| def find_description_override(repo_path): |
| return None |
| |
| |
| def make_local_copy(repo_path, project, project_list, |
| git_opts, ssh_env, upstream, GERRIT_HOST, GERRIT_PORT, |
| project_git, GERRIT_GITID): |
| |
| # Ensure that the base location exists |
| if not os.path.exists(os.path.dirname(repo_path)): |
| os.makedirs(os.path.dirname(repo_path)) |
| |
| # Three choices |
| # - If gerrit has it, get from gerrit |
| # - If gerrit doesn't have it: |
| # - If it has an upstream, clone that |
| # - If it doesn't, create it |
| |
| # Gerrit knows about the project, clone it |
| # TODO(mordred): there is a possible failure condition here |
| # we should consider 'gerrit has it' to be |
| # 'gerrit repo has a master branch' |
| if project in project_list: |
| run_command( |
| "git clone %(remote_url)s %(repo_path)s" % git_opts, |
| env=ssh_env) |
| if upstream: |
| git_command( |
| repo_path, |
| "remote add -f upstream %(upstream)s" % git_opts) |
| return None |
| |
| # Gerrit doesn't have it, but it has an upstream configured |
| # We're probably importing it for the first time, clone |
| # upstream, but then ongoing we want gerrit to ge origin |
| # and upstream to be only there for ongoing tracking |
| # purposes, so rename origin to upstream and add a new |
| # origin remote that points at gerrit |
| elif upstream: |
| run_command( |
| "git clone %(upstream)s %(repo_path)s" % git_opts, |
| env=ssh_env) |
| git_command( |
| repo_path, |
| "fetch origin +refs/heads/*:refs/copy/heads/*", |
| env=ssh_env) |
| git_command(repo_path, "remote rename origin upstream") |
| git_command( |
| repo_path, |
| "remote add origin %(remote_url)s" % git_opts) |
| return "push %s +refs/copy/heads/*:refs/heads/*" |
| |
| # Neither gerrit has it, nor does it have an upstream, |
| # just create a whole new one |
| else: |
| run_command("git init %s" % repo_path) |
| git_command( |
| repo_path, |
| "remote add origin %(remote_url)s" % git_opts) |
| with open(os.path.join(repo_path, |
| ".gitreview"), |
| 'w') as gitreview: |
| gitreview.write("""[gerrit] |
| host=%s |
| port=%s |
| project=%s |
| """ % (GERRIT_HOST, GERRIT_PORT, project_git)) |
| git_command(repo_path, "add .gitreview") |
| cmd = ("commit -a -m'Added .gitreview' --author='%s'" |
| % GERRIT_GITID) |
| git_command(repo_path, cmd) |
| return "push %s HEAD:refs/heads/master" |
| |
| |
| def update_local_copy(repo_path, track_upstream, git_opts, ssh_env): |
| # first do a clean of the branch to prevent possible |
| # problems due to previous runs |
| git_command(repo_path, "clean -fdx") |
| |
| has_upstream_remote = ( |
| 'upstream' in git_command_output(repo_path, 'remote')[1]) |
| if track_upstream: |
| # If we're configured to track upstream but the repo |
| # does not have an upstream remote, add one |
| if not has_upstream_remote: |
| git_command( |
| repo_path, |
| "remote add upstream %(upstream)s" % git_opts) |
| |
| # If we're configured to track upstream, make sure that |
| # the upstream URL matches the config |
| else: |
| git_command( |
| repo_path, |
| "remote set-url upstream %(upstream)s" % git_opts) |
| |
| # Now that we have any upstreams configured, fetch all of the refs |
| # we might need, pruning remote branches that no longer exist |
| git_command( |
| repo_path, "remote update --prune", env=ssh_env) |
| else: |
| # If we are not tracking upstream, then we do not need |
| # an upstream remote configured |
| if has_upstream_remote: |
| git_command(repo_path, "remote rm upstream") |
| |
| # TODO(mordred): This is here so that later we can |
| # inspect the master branch for meta-info |
| # Checkout master and reset to the state of origin/master |
| git_command(repo_path, "checkout -B master origin/master") |
| |
| |
| def push_to_gerrit(repo_path, project, push_string, remote_url, ssh_env): |
| try: |
| git_command(repo_path, push_string % remote_url, env=ssh_env) |
| git_command(repo_path, "push --tags %s" % remote_url, env=ssh_env) |
| except Exception: |
| log.exception( |
| "Error pushing %s to Gerrit." % project) |
| |
| |
| def sync_upstream(repo_path, project, ssh_env, upstream_prefix): |
| git_command( |
| repo_path, |
| "remote update upstream --prune", env=ssh_env) |
| # Any branch that exists in the upstream remote, we want |
| # a local branch of, optionally prefixed with the |
| # upstream prefix value |
| for branch in git_command_output( |
| repo_path, "branch -a")[1].split('\n'): |
| if not branch.strip().startswith("remotes/upstream"): |
| continue |
| if "->" in branch: |
| continue |
| local_branch = branch.split()[0][len('remotes/upstream/'):] |
| if upstream_prefix: |
| local_branch = "%s/%s" % ( |
| upstream_prefix, local_branch) |
| |
| # Check out an up to date copy of the branch, so that |
| # we can push it and it will get picked up below |
| git_command(repo_path, "checkout -B %s %s" % ( |
| local_branch, branch)) |
| |
| try: |
| # Push all of the local branches to similarly named |
| # Branches on gerrit. Also, push all of the tags |
| git_command( |
| repo_path, |
| "push origin refs/heads/*:refs/heads/*", |
| env=ssh_env) |
| git_command(repo_path, 'push origin --tags', env=ssh_env) |
| except Exception: |
| log.exception( |
| "Error pushing %s to Gerrit." % project) |
| |
| |
| def process_acls(acl_config, project, ACL_DIR, section, |
| remote_url, repo_path, ssh_env, gerrit, GERRIT_GITID): |
| if not os.path.isfile(acl_config): |
| return |
| try: |
| fetch_config(project, remote_url, repo_path, ssh_env) |
| if not copy_acl_config(project, repo_path, acl_config): |
| # nothing was copied, so we're done |
| return |
| create_groups_file(project, gerrit, repo_path) |
| push_acl_config(project, remote_url, repo_path, |
| GERRIT_GITID, ssh_env) |
| except Exception: |
| log.exception( |
| "Exception processing ACLS for %s." % project) |
| finally: |
| git_command(repo_path, 'reset --hard') |
| git_command(repo_path, 'checkout master') |
| git_command(repo_path, 'branch -D config') |
| |
| |
| def create_gerrit_project(project, project_list, gerrit): |
| if project not in project_list: |
| try: |
| gerrit.createProject(project) |
| return True |
| except Exception: |
| log.exception( |
| "Exception creating %s in Gerrit." % project) |
| raise |
| return False |
| |
| |
| def create_local_mirror(local_git_dir, project_git, |
| gerrit_system_user, gerrit_system_group): |
| |
| git_mirror_path = os.path.join(local_git_dir, project_git) |
| if not os.path.exists(git_mirror_path): |
| (ret, output) = run_command_status( |
| "git --bare init %s" % git_mirror_path) |
| if ret: |
| run_command("rm -rf git_mirror_path") |
| raise Exception(output) |
| run_command("chown -R %s:%s %s" |
| % (gerrit_system_user, gerrit_system_group, |
| git_mirror_path)) |
| |
| |
| def main(): |
| parser = argparse.ArgumentParser(description='Manage projects') |
| parser.add_argument('-v', dest='verbose', action='store_true', |
| help='verbose output') |
| parser.add_argument('-d', dest='debug', action='store_true', |
| help='debug output') |
| parser.add_argument('--nocleanup', action='store_true', |
| help='do not remove temp directories') |
| parser.add_argument('projects', metavar='project', nargs='*', |
| help='name of project(s) to process') |
| args = parser.parse_args() |
| |
| if args.debug: |
| logging.basicConfig(level=logging.DEBUG, |
| format='%(asctime)-6s: %(name)s - %(levelname)s' |
| ' - %(message)s') |
| elif args.verbose: |
| logging.basicConfig(level=logging.INFO, |
| format='%(asctime)-6s: %(name)s - %(levelname)s' |
| ' - %(message)s') |
| else: |
| logging.basicConfig(level=logging.ERROR, |
| format='%(asctime)-6s: %(name)s - %(levelname)s' |
| ' - %(message)s') |
| |
| default_has_github = registry.get_defaults('has-github', True) |
| |
| LOCAL_GIT_DIR = registry.get_defaults('local-git-dir', '/var/lib/git') |
| JEEPYB_CACHE_DIR = registry.get_defaults('jeepyb-cache-dir', |
| '/var/lib/jeepyb') |
| ACL_DIR = registry.get_defaults('acl-dir') |
| GERRIT_HOST = registry.get_defaults('gerrit-host') |
| GERRIT_PORT = int(registry.get_defaults('gerrit-port', '29418')) |
| GERRIT_USER = registry.get_defaults('gerrit-user') |
| GERRIT_KEY = registry.get_defaults('gerrit-key') |
| GERRIT_GITID = registry.get_defaults('gerrit-committer') |
| GERRIT_REPLICATE = registry.get_defaults('gerrit-replicate', True) |
| GERRIT_SYSTEM_USER = registry.get_defaults('gerrit-system-user', 'gerrit2') |
| GERRIT_SYSTEM_GROUP = registry.get_defaults('gerrit-system-group', |
| 'gerrit2') |
| DEFAULT_HOMEPAGE = registry.get_defaults('homepage') |
| DEFAULT_HAS_ISSUES = registry.get_defaults('has-issues', False) |
| DEFAULT_HAS_DOWNLOADS = registry.get_defaults('has-downloads', False) |
| DEFAULT_HAS_WIKI = registry.get_defaults('has-wiki', False) |
| GITHUB_SECURE_CONFIG = registry.get_defaults( |
| 'github-config', |
| '/etc/github/github-projects.secure.config') |
| |
| gerrit = gerritlib.gerrit.Gerrit(GERRIT_HOST, |
| GERRIT_USER, |
| GERRIT_PORT, |
| GERRIT_KEY) |
| project_list = gerrit.listProjects() |
| ssh_env = make_ssh_wrapper(GERRIT_USER, GERRIT_KEY) |
| try: |
| |
| for section in registry.configs_list: |
| project = section['project'] |
| if args.projects and project not in args.projects: |
| continue |
| |
| try: |
| log.info("Processing project: %s" % project) |
| |
| # Figure out all of the options |
| options = section.get('options', dict()) |
| description = section.get('description', None) |
| homepage = section.get('homepage', DEFAULT_HOMEPAGE) |
| upstream = section.get('upstream', None) |
| upstream_prefix = section.get('upstream-prefix', None) |
| track_upstream = 'track-upstream' in options |
| repo_path = os.path.join(JEEPYB_CACHE_DIR, project) |
| |
| # If this project doesn't want to use gerrit, exit cleanly. |
| if 'no-gerrit' in options: |
| continue |
| |
| project_git = "%s.git" % project |
| remote_url = "ssh://%s:%s/%s" % ( |
| GERRIT_HOST, |
| GERRIT_PORT, |
| project) |
| git_opts = dict(upstream=upstream, |
| repo_path=repo_path, |
| remote_url=remote_url) |
| acl_config = section.get( |
| 'acl-config', |
| '%s.config' % os.path.join(ACL_DIR, project)) |
| |
| # Create the project in Gerrit first, since it will fail |
| # spectacularly if its project directory or local replica |
| # already exist on disk |
| project_created = create_gerrit_project( |
| project, project_list, gerrit) |
| |
| # Create the repo for the local git mirror |
| create_local_mirror( |
| LOCAL_GIT_DIR, project_git, |
| GERRIT_SYSTEM_USER, GERRIT_SYSTEM_GROUP) |
| |
| if not os.path.exists(repo_path) or project_created: |
| # We don't have a local copy already, get one |
| |
| # Make Local repo |
| push_string = make_local_copy( |
| repo_path, project, project_list, |
| git_opts, ssh_env, upstream, GERRIT_HOST, GERRIT_PORT, |
| project_git, GERRIT_GITID) |
| else: |
| # We do have a local copy of it already, make sure it's |
| # in shape to have work done. |
| update_local_copy( |
| repo_path, track_upstream, git_opts, ssh_env) |
| |
| description = ( |
| find_description_override(repo_path) or description) |
| |
| if project_created: |
| push_to_gerrit( |
| repo_path, project, push_string, remote_url, ssh_env) |
| if GERRIT_REPLICATE: |
| gerrit.replicate(project) |
| |
| # If we're configured to track upstream, make sure we have |
| # upstream's refs, and then push them to the appropriate |
| # branches in gerrit |
| if track_upstream: |
| sync_upstream(repo_path, project, ssh_env, upstream_prefix) |
| |
| if acl_config: |
| process_acls( |
| acl_config, project, ACL_DIR, section, |
| remote_url, repo_path, ssh_env, gerrit, GERRIT_GITID) |
| |
| if 'has-github' in options or default_has_github: |
| created = create_github_project( |
| DEFAULT_HAS_ISSUES, DEFAULT_HAS_DOWNLOADS, |
| DEFAULT_HAS_WIKI, GITHUB_SECURE_CONFIG, |
| options, project, description, homepage) |
| if created and GERRIT_REPLICATE: |
| gerrit.replicate(project) |
| |
| except Exception: |
| log.exception( |
| "Problems creating %s, moving on." % project) |
| continue |
| finally: |
| os.unlink(ssh_env['GIT_SSH']) |
| |
| if __name__ == "__main__": |
| main() |