Do repo processing in an existing local dir
Keep repos around so that we can do smarter things with them
aroud processing remotes and tracking descriptions.
Changes upstream tracking from mucking with repos behind gerrit to
pushing in changes with the push command, which should allow for zuul
to take action based on receiving new refs from upstream.`
Change-Id: I9e79489b7c724db1320272f75a7da2c2b4c24607
diff --git a/jeepyb/cmd/manage_projects.py b/jeepyb/cmd/manage_projects.py
index fa8e055..23b121f 100644
--- a/jeepyb/cmd/manage_projects.py
+++ b/jeepyb/cmd/manage_projects.py
@@ -35,10 +35,11 @@
# - has-issues
# - has-downloads
# - has-pull-requests
+# - track-upstream
# homepage: Some homepage that isn't http://openstack.org
# description: This is a great project
-# remote: https://gerrit.googlesource.com/gerrit
-# upstream: git://github.com/bushy/beards.git
+# upstream: https://gerrit.googlesource.com/gerrit
+# upstream-prefix: upstream
# acl-config: /path/to/gerrit/project.config
# acl-append:
# - /path/to/gerrit/project.config
@@ -295,6 +296,12 @@
teams_dict['gerrit'].add_to_repos(repo)
+# TODO(mordred): Inspect repo_dir:master for a description
+# override
+def find_description_override(repo_path):
+ return None
+
+
def main():
parser = argparse.ArgumentParser(description='Manage projects')
parser.add_argument('-v', dest='verbose', action='store_true',
@@ -317,6 +324,7 @@
default_has_github = defaults.get('has-github', True)
LOCAL_GIT_DIR = defaults.get('local-git-dir', '/var/lib/git')
+ JEEPYB_CACHE_DIR = defaults.get('jeepyb-cache-dir', '/var/lib/jeepyb')
ACL_DIR = defaults.get('acl-dir')
GERRIT_HOST = defaults.get('gerrit-host')
GERRIT_PORT = int(defaults.get('gerrit-port', '29418'))
@@ -342,51 +350,129 @@
description = section.get('description', None)
homepage = section.get('homepage', defaults.get('homepage', None))
upstream = section.get('upstream', None)
+ upstream_prefix = section.get('upstream-prefix', None)
+ track_upstream = 'track-upstream' in options
project_git = "%s.git" % project
- project_dir = os.path.join(LOCAL_GIT_DIR, project_git)
+ remote_url = "ssh://localhost:%s/%s" % (GERRIT_PORT, project)
+
+ # Create the repo for the local git mirror
+ git_mirror_path = os.path.join(LOCAL_GIT_DIR, project_git)
+ if not os.path.exists(git_mirror_path):
+ run_command("git --bare init %s" % git_mirror_path)
+ run_command("chown -R %s:%s %s"
+ % (GERRIT_SYSTEM_USER, GERRIT_SYSTEM_GROUP,
+ git_mirror_path))
+
+ # Start processing the local JEEPYB cache
+ repo_path = os.path.join(JEEPYB_CACHE_DIR, project)
+ git_opts = dict(upstream=upstream,
+ repo_path=repo_path,
+ remote_url=remote_url)
+
+ # If we don't have a local copy already, get one
+ if not os.path.exists(repo_path):
+ if not os.path.exists(os.path.dirname(repo_path)):
+ os.makedirs(os.path.dirname(repo_path))
+ # Three choices
+ # - If gerrit has it, get from gerrit
+ # - If gerrit doesn't have it:
+ # - If it has an upstream, clone that
+ # - If it doesn't, create it
+
+ # Gerrit knows about the project, clone it
+ if project in project_list:
+ run_command(
+ "git clone %(remote_url)s %(repo_path)s" % git_opts,
+ env=ssh_env)
+ if upstream:
+ git_command(
+ repo_path,
+ "remote add -f upstream %(upstream)s" % git_opts)
+
+ # Gerrit doesn't have it, but it has an upstream configured
+ # We're probably importing it for the first time, clone
+ # upstream, but then ongoing we want gerrit to ge origin
+ # and upstream to be only there for ongoing tracking
+ # purposes, so rename origin to upstream and add a new
+ # origin remote that points at gerrit
+ elif upstream:
+ run_command(
+ "git clone %(upstream)s %(repo_path)s" % git_opts,
+ env=ssh_env)
+ git_command(
+ repo_path,
+ "fetch origin +refs/heads/*:refs/copy/heads/*",
+ env=ssh_env)
+ git_command(repo_path, "remote rename origin upstream")
+ git_command(
+ repo_path,
+ "remote add origin %(remote_url)s" % git_opts)
+ push_string = "push %s +refs/copy/heads/*:refs/heads/*"
+
+ # Neither gerrit has it, nor does it have an upstream,
+ # just create a whole new one
+ else:
+ run_command("git init %s" % repo_path)
+ git_command(
+ repo_path,
+ "remote add origin %(remote_url)" % git_opts)
+ with open(os.path.join(repo_path,
+ ".gitreview"),
+ 'w') as gitreview:
+ gitreview.write("""[gerrit]
+host=%s
+port=%s
+project=%s
+""" % (GERRIT_HOST, GERRIT_PORT, project_git))
+ git_command(repo_path, "add .gitreview")
+ cmd = ("commit -a -m'Added .gitreview' --author='%s'"
+ % GERRIT_GITID)
+ git_command(repo_path, cmd)
+ push_string = "push %s HEAD:refs/heads/master"
+
+ # We do have a local copy of it already, make sure it's
+ # in shape to have work done.
+ else:
+ if project in project_list:
+
+ # If we're configured to track upstream but the repo
+ # does not have an upstream remote, add one
+ if (track_upstream and
+ 'upstream' not in git_command_output(
+ repo_path, 'remote')[1]):
+ git_command(
+ repo_path,
+ "remote add upstream %(upstream)s" % git_opts)
+
+ # If we're configured to track upstream, make sure that
+ # the upstream URL matches the config
+ else:
+ git_command(
+ repo_path,
+ "remote set-url upstream %(upstream)s" % git_opts)
+
+ # Now that we have any upstreams configured, fetch all
+ # of the refs we might need, pruning remote branches
+ # that no longer exist
+ git_command(
+ repo_path, "remote update --prune", env=ssh_env)
+
+ # TODO(mordred): This is here so that later we can
+ # inspect the master branch for meta-info
+ # Checkout master and reset to the state of origin/master
+ git_command(repo_path, "checkout -B master origin/master")
+
+ description = find_description_override(repo_path) or description
if 'has-github' in options or default_has_github:
create_github_project(defaults, options, project,
description, homepage)
- remote_url = "ssh://localhost:%s/%s" % (GERRIT_PORT, project)
if project not in project_list:
- tmpdir = tempfile.mkdtemp()
try:
- repo_path = os.path.join(tmpdir, 'repo')
- if upstream:
- run_command("git clone %(upstream)s %(repo_path)s" %
- dict(upstream=upstream,
- repo_path=repo_path))
- git_command(repo_path,
- "fetch origin "
- "+refs/heads/*:refs/copy/heads/*",
- env=ssh_env)
- push_string = "push %s +refs/copy/heads/*:refs/heads/*"
- else:
- run_command("git init %s" % repo_path)
- with open(os.path.join(repo_path,
- ".gitreview"),
- 'w') as gitreview:
- gitreview.write("""[gerrit]
-host=%s
-port=%s
-project=%s
-""" % (GERRIT_HOST, GERRIT_PORT, project_git))
- git_command(repo_path, "add .gitreview")
- cmd = ("commit -a -m'Added .gitreview' --author='%s'"
- % GERRIT_GITID)
- git_command(repo_path, cmd)
- push_string = "push --all %s"
gerrit.createProject(project)
- if not os.path.exists(project_dir):
- run_command("git --bare init %s" % project_dir)
- run_command("chown -R %s:%s %s"
- % (GERRIT_SYSTEM_USER, GERRIT_SYSTEM_GROUP,
- project_dir))
-
git_command(repo_path,
push_string % remote_url,
env=ssh_env)
@@ -395,9 +481,44 @@
except Exception:
log.exception(
"Exception creating %s in Gerrit." % project)
- finally:
- if not args.nocleanup:
- run_command("rm -fr %s" % tmpdir)
+
+ # If we're configured to track upstream, make sure we have
+ # upstream's refs, and then push them to the appropriate
+ # branches in gerrit
+ if track_upstream:
+ git_command(
+ repo_path,
+ "remote update upstream --prune", env=ssh_env)
+ # Any branch that exists in the upstream remote, we want
+ # a local branch of, optionally prefixed with the
+ # upstream prefix value
+ for branch in git_command_output(
+ repo_path, "branch -a")[1].split():
+ if not branch.strip().startswith("remotes/upstream"):
+ continue
+ if "->" in branch:
+ continue
+ local_branch = branch[len('remotes/upstream/'):]
+ if upstream_prefix:
+ local_branch = "%s/%s" % (
+ upstream_prefix, local_branch)
+
+ # Check out an up to date copy of the branch, so that
+ # we can push it and it will get picked up below
+ git_command(repo_path, "checkout -B %s %s" % (
+ local_branch, branch))
+
+ try:
+ # Push all of the local branches to similarly named
+ # Branches on gerrit. Also, push all of the tags
+ git_command(
+ repo_path,
+ "push origin +refs/heads/*:refs/heads/*",
+ env=ssh_env)
+ git_command(repo_path, 'push --tags', env=ssh_env)
+ except Exception:
+ log.exception(
+ "Error pushing %s to Gerrit." % project)
try:
acl_config = section.get('acl-config',
@@ -413,12 +534,7 @@
section.get('acl-base', None),
section.get('acl-append', []),
section.get('acl-parameters', {}))
- tmpdir = tempfile.mkdtemp()
try:
- repo_path = os.path.join(tmpdir, 'repo')
- ret, _ = run_command_status("git init %s" % repo_path)
- if ret != 0:
- continue
if (fetch_config(project,
remote_url,
repo_path,
@@ -435,8 +551,9 @@
log.exception(
"Exception processing ACLS for %s." % project)
finally:
- if not args.nocleanup:
- run_command("rm -fr %s" % tmpdir)
+ git_command(repo_path, 'checkout master')
+ git_command(repo_path, 'branch -D config')
+
finally:
os.unlink(ssh_env['GIT_SSH'])