Catch errors centrally, and bail out more
We want to be more resilient against parts of things failing. For that,
we need to be able to bail out of the loop when things bork. Put in a
central loop exception processor, and throw things from the functions.
Change-Id: I5d9b30e1403634909ad45c27af7c975d815a9538
diff --git a/jeepyb/cmd/manage_projects.py b/jeepyb/cmd/manage_projects.py
index 52eed60..d13191e 100644
--- a/jeepyb/cmd/manage_projects.py
+++ b/jeepyb/cmd/manage_projects.py
@@ -302,111 +302,108 @@
return None
-def make_local_copy(repo_path, project_created, project, project_list,
+def make_local_copy(repo_path, project, project_list,
git_opts, ssh_env, upstream, GERRIT_HOST, GERRIT_PORT,
- project_git, GERRIT_GITID, track_upstream):
- # If we don't have a local copy already, get one
- if not os.path.exists(repo_path) or project_created:
- if not os.path.exists(os.path.dirname(repo_path)):
- os.makedirs(os.path.dirname(repo_path))
- # Three choices
- # - If gerrit has it, get from gerrit
- # - If gerrit doesn't have it:
- # - If it has an upstream, clone that
- # - If it doesn't, create it
+ project_git, GERRIT_GITID):
- # Gerrit knows about the project, clone it
- if project in project_list:
- run_command(
- "git clone %(remote_url)s %(repo_path)s" % git_opts,
- env=ssh_env)
- if upstream:
- git_command(
- repo_path,
- "remote add -f upstream %(upstream)s" % git_opts)
+ # Ensure that the base location exists
+ if not os.path.exists(os.path.dirname(repo_path)):
+ os.makedirs(os.path.dirname(repo_path))
- # Gerrit doesn't have it, but it has an upstream configured
- # We're probably importing it for the first time, clone
- # upstream, but then ongoing we want gerrit to ge origin
- # and upstream to be only there for ongoing tracking
- # purposes, so rename origin to upstream and add a new
- # origin remote that points at gerrit
- elif upstream:
- run_command(
- "git clone %(upstream)s %(repo_path)s" % git_opts,
- env=ssh_env)
+ # Three choices
+ # - If gerrit has it, get from gerrit
+ # - If gerrit doesn't have it:
+ # - If it has an upstream, clone that
+ # - If it doesn't, create it
+
+ # Gerrit knows about the project, clone it
+ # TODO(mordred): there is a possible failure condition here
+ # we should consider 'gerrit has it' to be
+ # 'gerrit repo has a master branch'
+ if project in project_list:
+ run_command(
+ "git clone %(remote_url)s %(repo_path)s" % git_opts,
+ env=ssh_env)
+ if upstream:
git_command(
repo_path,
- "fetch origin +refs/heads/*:refs/copy/heads/*",
- env=ssh_env)
- git_command(repo_path, "remote rename origin upstream")
- git_command(
- repo_path,
- "remote add origin %(remote_url)s" % git_opts)
- push_string = "push %s +refs/copy/heads/*:refs/heads/*"
+ "remote add -f upstream %(upstream)s" % git_opts)
+ return None
- # Neither gerrit has it, nor does it have an upstream,
- # just create a whole new one
- else:
- run_command("git init %s" % repo_path)
- git_command(
- repo_path,
- "remote add origin %(remote_url)s" % git_opts)
- with open(os.path.join(repo_path,
- ".gitreview"),
- 'w') as gitreview:
- gitreview.write("""[gerrit]
+ # Gerrit doesn't have it, but it has an upstream configured
+ # We're probably importing it for the first time, clone
+ # upstream, but then ongoing we want gerrit to ge origin
+ # and upstream to be only there for ongoing tracking
+ # purposes, so rename origin to upstream and add a new
+ # origin remote that points at gerrit
+ elif upstream:
+ run_command(
+ "git clone %(upstream)s %(repo_path)s" % git_opts,
+ env=ssh_env)
+ git_command(
+ repo_path,
+ "fetch origin +refs/heads/*:refs/copy/heads/*",
+ env=ssh_env)
+ git_command(repo_path, "remote rename origin upstream")
+ git_command(
+ repo_path,
+ "remote add origin %(remote_url)s" % git_opts)
+ return "push %s +refs/copy/heads/*:refs/heads/*"
+
+ # Neither gerrit has it, nor does it have an upstream,
+ # just create a whole new one
+ else:
+ run_command("git init %s" % repo_path)
+ git_command(
+ repo_path,
+ "remote add origin %(remote_url)s" % git_opts)
+ with open(os.path.join(repo_path,
+ ".gitreview"),
+ 'w') as gitreview:
+ gitreview.write("""[gerrit]
host=%s
port=%s
project=%s
""" % (GERRIT_HOST, GERRIT_PORT, project_git))
- git_command(repo_path, "add .gitreview")
- cmd = ("commit -a -m'Added .gitreview' --author='%s'"
- % GERRIT_GITID)
- git_command(repo_path, cmd)
- push_string = "push %s HEAD:refs/heads/master"
+ git_command(repo_path, "add .gitreview")
+ cmd = ("commit -a -m'Added .gitreview' --author='%s'"
+ % GERRIT_GITID)
+ git_command(repo_path, cmd)
+ return "push %s HEAD:refs/heads/master"
- # We do have a local copy of it already, make sure it's
- # in shape to have work done.
+
+def update_local_copy(repo_path, track_upstream, git_opts, ssh_env):
+ # If we're configured to track upstream but the repo
+ # does not have an upstream remote, add one
+ if (track_upstream and
+ 'upstream' not in git_command_output(
+ repo_path, 'remote')[1]):
+ git_command(
+ repo_path,
+ "remote add upstream %(upstream)s" % git_opts)
+
+ # If we're configured to track upstream, make sure that
+ # the upstream URL matches the config
else:
- if project in project_list:
+ git_command(
+ repo_path,
+ "remote set-url upstream %(upstream)s" % git_opts)
- # If we're configured to track upstream but the repo
- # does not have an upstream remote, add one
- if (track_upstream and
- 'upstream' not in git_command_output(
- repo_path, 'remote')[1]):
- git_command(
- repo_path,
- "remote add upstream %(upstream)s" % git_opts)
+ # Now that we have any upstreams configured, fetch all of the refs
+ # we might need, pruning remote branches that no longer exist
+ git_command(
+ repo_path, "remote update --prune", env=ssh_env)
- # If we're configured to track upstream, make sure that
- # the upstream URL matches the config
- else:
- git_command(
- repo_path,
- "remote set-url upstream %(upstream)s" % git_opts)
-
- # Now that we have any upstreams configured, fetch all
- # of the refs we might need, pruning remote branches
- # that no longer exist
- git_command(
- repo_path, "remote update --prune", env=ssh_env)
-
- # TODO(mordred): This is here so that later we can
- # inspect the master branch for meta-info
- # Checkout master and reset to the state of origin/master
- git_command(repo_path, "checkout -B master origin/master")
- return push_string
+ # TODO(mordred): This is here so that later we can
+ # inspect the master branch for meta-info
+ # Checkout master and reset to the state of origin/master
+ git_command(repo_path, "checkout -B master origin/master")
def push_to_gerrit(repo_path, project, push_string, remote_url, ssh_env):
try:
- git_command(repo_path,
- push_string % remote_url,
- env=ssh_env)
- git_command(repo_path,
- "push --tags %s" % remote_url, env=ssh_env)
+ git_command(repo_path, push_string % remote_url, env=ssh_env)
+ git_command(repo_path, "push --tags %s" % remote_url, env=ssh_env)
except Exception:
log.exception(
"Error pushing %s to Gerrit." % project)
@@ -478,7 +475,7 @@
git_command(repo_path, 'branch -D config')
-def create_local_project(project, project_list, gerrit):
+def create_gerrit_project(project, project_list, gerrit):
if project not in project_list:
try:
gerrit.createProject(project)
@@ -486,17 +483,22 @@
except Exception:
log.exception(
"Exception creating %s in Gerrit." % project)
+ raise
return False
-def create_local_mirror(LOCAL_GIT_DIR, project_git,
- GERRIT_SYSTEM_USER, GERRIT_SYSTEM_GROUP):
+def create_local_mirror(local_git_dir, project_git,
+ gerrit_system_user, gerrit_system_group):
- git_mirror_path = os.path.join(LOCAL_GIT_DIR, project_git)
+ git_mirror_path = os.path.join(local_git_dir, project_git)
if not os.path.exists(git_mirror_path):
- run_command("git --bare init %s" % git_mirror_path)
+ (ret, output) = run_command_status(
+ "git --bare init %s" % git_mirror_path)
+ if ret:
+ run_command("rm -rf git_mirror_path")
+ raise Exception(output)
run_command("chown -R %s:%s %s"
- % (GERRIT_SYSTEM_USER, GERRIT_SYSTEM_GROUP,
+ % (gerrit_system_user, gerrit_system_group,
git_mirror_path))
@@ -544,64 +546,78 @@
project = section['project']
if args.projects and project not in args.projects:
continue
- options = section.get('options', dict())
- description = section.get('description', None)
- homepage = section.get('homepage', defaults.get('homepage', None))
- upstream = section.get('upstream', None)
- upstream_prefix = section.get('upstream-prefix', None)
- track_upstream = 'track-upstream' in options
- repo_path = os.path.join(JEEPYB_CACHE_DIR, project)
- project_git = "%s.git" % project
- remote_url = "ssh://localhost:%s/%s" % (GERRIT_PORT, project)
- git_opts = dict(upstream=upstream,
- repo_path=repo_path,
- remote_url=remote_url)
try:
- acl_config = section.get('acl-config',
- '%s.config' % os.path.join(ACL_DIR,
- project))
- except AttributeError:
- acl_config = None
+ # Figure out all of the options
+ options = section.get('options', dict())
+ description = section.get('description', None)
+ homepage = section.get(
+ 'homepage', defaults.get('homepage', None))
+ upstream = section.get('upstream', None)
+ upstream_prefix = section.get('upstream-prefix', None)
+ track_upstream = 'track-upstream' in options
+ repo_path = os.path.join(JEEPYB_CACHE_DIR, project)
- # Create the project in Gerrit first, since it will fail
- # spectacularly if its project directory or local replica
- # already exist on disk
- project_created = create_local_project(
- project, project_list, gerrit)
+ project_git = "%s.git" % project
+ remote_url = "ssh://localhost:%s/%s" % (GERRIT_PORT, project)
+ git_opts = dict(upstream=upstream,
+ repo_path=repo_path,
+ remote_url=remote_url)
+ acl_config = section.get(
+ 'acl-config',
+ '%s.config' % os.path.join(ACL_DIR, project))
- # Create the repo for the local git mirror
- create_local_mirror(
- LOCAL_GIT_DIR, project_git,
- GERRIT_SYSTEM_USER, GERRIT_SYSTEM_GROUP)
+ # Create the project in Gerrit first, since it will fail
+ # spectacularly if its project directory or local replica
+ # already exist on disk
+ project_created = create_gerrit_project(
+ project, project_list, gerrit)
- # Make Local repo
- push_string = make_local_copy(
- repo_path, project_created, project, project_list,
- git_opts, ssh_env, upstream, GERRIT_HOST, GERRIT_PORT,
- project_git, GERRIT_GITID, track_upstream)
+ # Create the repo for the local git mirror
+ create_local_mirror(
+ LOCAL_GIT_DIR, project_git,
+ GERRIT_SYSTEM_USER, GERRIT_SYSTEM_GROUP)
- description = find_description_override(repo_path) or description
+ if not os.path.exists(repo_path) or project_created:
+ # We don't have a local copy already, get one
- if 'has-github' in options or default_has_github:
- create_github_project(defaults, options, project,
- description, homepage)
+ # Make Local repo
+ push_string = make_local_copy(
+ repo_path, project, project_list,
+ git_opts, ssh_env, upstream, GERRIT_HOST, GERRIT_PORT,
+ project_git, GERRIT_GITID)
+ else:
+ # We do have a local copy of it already, make sure it's
+ # in shape to have work done.
+ update_local_copy(
+ repo_path, track_upstream, git_opts, ssh_env)
- if project_created:
- push_to_gerrit(
- repo_path, project, push_string, remote_url, ssh_env)
+ description = (
+ find_description_override(repo_path) or description)
- # If we're configured to track upstream, make sure we have
- # upstream's refs, and then push them to the appropriate
- # branches in gerrit
- if track_upstream:
- sync_upstream(repo_path, project, ssh_env, upstream_prefix)
+ if 'has-github' in options or default_has_github:
+ create_github_project(defaults, options, project,
+ description, homepage)
- if acl_config:
- process_acls(
- acl_config, project, ACL_DIR, section,
- remote_url, repo_path, ssh_env, gerrit, GERRIT_GITID)
+ if project_created:
+ push_to_gerrit(
+ repo_path, project, push_string, remote_url, ssh_env)
+ # If we're configured to track upstream, make sure we have
+ # upstream's refs, and then push them to the appropriate
+ # branches in gerrit
+ if track_upstream:
+ sync_upstream(repo_path, project, ssh_env, upstream_prefix)
+
+ if acl_config:
+ process_acls(
+ acl_config, project, ACL_DIR, section,
+ remote_url, repo_path, ssh_env, gerrit, GERRIT_GITID)
+
+ except Exception:
+ log.exception(
+ "Problems creating %s, moving on." % project)
+ continue
finally:
os.unlink(ssh_env['GIT_SSH'])