| #! /usr/bin/env python |
| # Copyright (C) 2011 OpenStack, LLC. |
| # Copyright (c) 2012 Hewlett-Packard Development Company, L.P. |
| # |
| # Licensed under the Apache License, Version 2.0 (the "License"); |
| # you may not use this file except in compliance with the License. |
| # You may obtain a copy of the License at |
| # |
| # http://www.apache.org/licenses/LICENSE-2.0 |
| # |
| # Unless required by applicable law or agreed to in writing, software |
| # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT |
| # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the |
| # License for the specific language governing permissions and limitations |
| # under the License. |
| |
| # manage_projects.py reads a config file called projects.ini |
| # It should look like: |
| |
| # [projects] |
| # homepage=http://openstack.org |
| # gerrit-host=review.openstack.org |
| # local-git-dir=/var/lib/git |
| # gerrit-key=/home/gerrit2/review_site/etc/ssh_host_rsa_key |
| # gerrit-committer=Project Creator <openstack-infra@lists.openstack.org> |
| # gerrit-replicate=True |
| # has-github=True |
| # has-wiki=False |
| # has-issues=False |
| # has-downloads=False |
| # acl-dir=/home/gerrit2/acls |
| # acl-base=/home/gerrit2/acls/project.config |
| # |
| # manage_projects.py reads a project listing file called projects.yaml |
| # It should look like: |
| # - project: PROJECT_NAME |
| # options: |
| # - has-wiki |
| # - has-issues |
| # - has-downloads |
| # - has-pull-requests |
| # - track-upstream |
| # homepage: Some homepage that isn't http://openstack.org |
| # description: This is a great project |
| # upstream: https://gerrit.googlesource.com/gerrit |
| # upstream-prefix: upstream |
| # acl-config: /path/to/gerrit/project.config |
| # acl-append: |
| # - /path/to/gerrit/project.config |
| # acl-parameters: |
| # project: OTHER_PROJECT_NAME |
| |
| import argparse |
| import json |
| import logging |
| import os |
| |
| import gerritlib.gerrit |
| |
| import jeepyb.log as l |
| import jeepyb.utils as u |
| |
| registry = u.ProjectsRegistry() |
| |
| log = logging.getLogger("track_upstream") |
| orgs = None |
| |
| |
| def update_local_copy(repo_path, track_upstream, git_opts, ssh_env): |
| # first do a clean of the branch to prevent possible |
| # problems due to previous runs |
| u.git_command(repo_path, "clean -fdx") |
| |
| has_upstream_remote = ( |
| 'upstream' in u.git_command_output(repo_path, 'remote')[1]) |
| if track_upstream: |
| # If we're configured to track upstream but the repo |
| # does not have an upstream remote, add one |
| if not has_upstream_remote: |
| u.git_command( |
| repo_path, |
| "remote add upstream %(upstream)s" % git_opts) |
| |
| # If we're configured to track upstream, make sure that |
| # the upstream URL matches the config |
| else: |
| u.git_command( |
| repo_path, |
| "remote set-url upstream %(upstream)s" % git_opts) |
| |
| # Now that we have any upstreams configured, fetch all of the refs |
| # we might need, pruning remote branches that no longer exist |
| u.git_command( |
| repo_path, "remote update --prune", env=ssh_env) |
| else: |
| # If we are not tracking upstream, then we do not need |
| # an upstream remote configured |
| if has_upstream_remote: |
| u.git_command(repo_path, "remote rm upstream") |
| |
| # TODO(mordred): This is here so that later we can |
| # inspect the master branch for meta-info |
| # Checkout master and reset to the state of origin/master |
| u.git_command(repo_path, "checkout -B master origin/master") |
| |
| |
| def sync_upstream(repo_path, project, ssh_env, upstream_prefix): |
| u.git_command( |
| repo_path, |
| "remote update upstream --prune", env=ssh_env) |
| # Any branch that exists in the upstream remote, we want |
| # a local branch of, optionally prefixed with the |
| # upstream prefix value |
| for branch in u.git_command_output( |
| repo_path, "branch -a")[1].split('\n'): |
| if not branch.strip().startswith("remotes/upstream"): |
| continue |
| if "->" in branch: |
| continue |
| local_branch = branch.split()[0][len('remotes/upstream/'):] |
| if upstream_prefix: |
| local_branch = "%s/%s" % ( |
| upstream_prefix, local_branch) |
| |
| # Check out an up to date copy of the branch, so that |
| # we can push it and it will get picked up below |
| u.git_command( |
| repo_path, "checkout -B %s %s" % (local_branch, branch)) |
| |
| try: |
| # Push all of the local branches to similarly named |
| # Branches on gerrit. Also, push all of the tags |
| u.git_command( |
| repo_path, |
| "push origin refs/heads/*:refs/heads/*", |
| env=ssh_env) |
| u.git_command(repo_path, 'push origin --tags', env=ssh_env) |
| except Exception: |
| log.exception( |
| "Error pushing %s to Gerrit." % project) |
| |
| |
| def main(): |
| parser = argparse.ArgumentParser(description='Manage projects') |
| l.setup_logging_arguments(parser) |
| parser.add_argument('--nocleanup', action='store_true', |
| help='do not remove temp directories') |
| parser.add_argument('projects', metavar='project', nargs='*', |
| help='name of project(s) to process') |
| args = parser.parse_args() |
| l.configure_logging(args) |
| |
| JEEPYB_CACHE_DIR = registry.get_defaults('jeepyb-cache-dir', |
| '/var/lib/jeepyb') |
| IMPORT_DIR = os.path.join(JEEPYB_CACHE_DIR, 'import') |
| GERRIT_HOST = registry.get_defaults('gerrit-host') |
| GERRIT_PORT = int(registry.get_defaults('gerrit-port', '29418')) |
| GERRIT_USER = registry.get_defaults('gerrit-user') |
| GERRIT_KEY = registry.get_defaults('gerrit-key') |
| GERRIT_GITID = registry.get_defaults('gerrit-committer') |
| |
| PROJECT_CACHE_FILE = os.path.join(JEEPYB_CACHE_DIR, 'project.cache') |
| project_cache = {} |
| if os.path.exists(PROJECT_CACHE_FILE): |
| project_cache = json.loads(open(PROJECT_CACHE_FILE, 'r').read()) |
| |
| gerrit = gerritlib.gerrit.Gerrit(GERRIT_HOST, |
| GERRIT_USER, |
| GERRIT_PORT, |
| GERRIT_KEY) |
| project_list = gerrit.listProjects() |
| ssh_env = u.make_ssh_wrapper(GERRIT_USER, GERRIT_KEY) |
| try: |
| |
| for section in registry.configs_list: |
| project = section['project'] |
| if args.projects and project not in args.projects: |
| continue |
| |
| try: |
| log.info("Processing project: %s" % project) |
| |
| # Figure out all of the options |
| options = section.get('options', dict()) |
| track_upstream = 'track-upstream' in options |
| if not track_upstream: |
| continue |
| |
| # If this project doesn't want to use gerrit, exit cleanly. |
| if 'no-gerrit' in options: |
| continue |
| |
| upstream = section.get('upstream', None) |
| upstream_prefix = section.get('upstream-prefix', None) |
| repo_path = os.path.join(IMPORT_DIR, project) |
| |
| project_git = "%s.git" % project |
| remote_url = "ssh://%s:%s/%s" % ( |
| GERRIT_HOST, |
| GERRIT_PORT, |
| project) |
| git_opts = dict(upstream=upstream, |
| repo_path=repo_path, |
| remote_url=remote_url) |
| project_cache.setdefault(project, {}) |
| if not project_cache[project]['pushed-to-gerrit']: |
| continue |
| |
| # Make Local repo |
| if not os.path.exists(repo_path): |
| u.make_local_copy( |
| repo_path, project, project_list, |
| git_opts, ssh_env, upstream, GERRIT_HOST, |
| GERRIT_PORT, project_git, GERRIT_GITID) |
| else: |
| update_local_copy( |
| repo_path, track_upstream, git_opts, ssh_env) |
| |
| u.fsck_repo(repo_path) |
| sync_upstream(repo_path, project, ssh_env, upstream_prefix) |
| |
| except Exception: |
| log.exception( |
| "Problems creating %s, moving on." % project) |
| continue |
| finally: |
| os.unlink(ssh_env['GIT_SSH']) |
| |
| if __name__ == "__main__": |
| main() |