Merge "Add script to post a gerrit welcome message"
diff --git a/jeepyb/cmd/manage_projects.py b/jeepyb/cmd/manage_projects.py
index 4f360a6..1737543 100644
--- a/jeepyb/cmd/manage_projects.py
+++ b/jeepyb/cmd/manage_projects.py
@@ -183,10 +183,10 @@
def _wait_for_group(gerrit, group):
"""Wait for up to 10 seconds for the group to be created."""
for x in range(10):
+ time.sleep(1)
groups = gerrit.listGroups()
if group in groups:
break
- time.sleep(1)
def get_group_uuid(gerrit, group):
@@ -240,6 +240,7 @@
def create_github_project(defaults, options, project, description, homepage):
+ created = False
default_has_issues = defaults.get('has-issues', False)
default_has_downloads = defaults.get('has-downloads', False)
default_has_wiki = defaults.get('has-wiki', False)
@@ -272,7 +273,7 @@
org = orgs_dict[org_name.lower()]
except KeyError:
# We do not have control of this github org ignore the project.
- return
+ return False
try:
repo = org.get_repo(repo_name)
except github.GithubException:
@@ -281,6 +282,7 @@
has_issues=has_issues,
has_downloads=has_downloads,
has_wiki=has_wiki)
+ created = True
if description:
repo.edit(repo_name, description=description)
if homepage:
@@ -294,6 +296,7 @@
teams = org.get_teams()
teams_dict = dict(zip([t.name.lower() for t in teams], teams))
teams_dict['gerrit'].add_to_repos(repo)
+ return created
# TODO(mordred): Inspect repo_dir:master for a description
@@ -302,6 +305,206 @@
return None
+def make_local_copy(repo_path, project, project_list,
+ git_opts, ssh_env, upstream, GERRIT_HOST, GERRIT_PORT,
+ project_git, GERRIT_GITID):
+
+ # Ensure that the base location exists
+ if not os.path.exists(os.path.dirname(repo_path)):
+ os.makedirs(os.path.dirname(repo_path))
+
+ # Three choices
+ # - If gerrit has it, get from gerrit
+ # - If gerrit doesn't have it:
+ # - If it has an upstream, clone that
+ # - If it doesn't, create it
+
+ # Gerrit knows about the project, clone it
+ # TODO(mordred): there is a possible failure condition here
+ # we should consider 'gerrit has it' to be
+ # 'gerrit repo has a master branch'
+ if project in project_list:
+ run_command(
+ "git clone %(remote_url)s %(repo_path)s" % git_opts,
+ env=ssh_env)
+ if upstream:
+ git_command(
+ repo_path,
+ "remote add -f upstream %(upstream)s" % git_opts)
+ return None
+
+ # Gerrit doesn't have it, but it has an upstream configured
+ # We're probably importing it for the first time, clone
+ # upstream, but then ongoing we want gerrit to ge origin
+ # and upstream to be only there for ongoing tracking
+ # purposes, so rename origin to upstream and add a new
+ # origin remote that points at gerrit
+ elif upstream:
+ run_command(
+ "git clone %(upstream)s %(repo_path)s" % git_opts,
+ env=ssh_env)
+ git_command(
+ repo_path,
+ "fetch origin +refs/heads/*:refs/copy/heads/*",
+ env=ssh_env)
+ git_command(repo_path, "remote rename origin upstream")
+ git_command(
+ repo_path,
+ "remote add origin %(remote_url)s" % git_opts)
+ return "push %s +refs/copy/heads/*:refs/heads/*"
+
+ # Neither gerrit has it, nor does it have an upstream,
+ # just create a whole new one
+ else:
+ run_command("git init %s" % repo_path)
+ git_command(
+ repo_path,
+ "remote add origin %(remote_url)s" % git_opts)
+ with open(os.path.join(repo_path,
+ ".gitreview"),
+ 'w') as gitreview:
+ gitreview.write("""[gerrit]
+host=%s
+port=%s
+project=%s
+""" % (GERRIT_HOST, GERRIT_PORT, project_git))
+ git_command(repo_path, "add .gitreview")
+ cmd = ("commit -a -m'Added .gitreview' --author='%s'"
+ % GERRIT_GITID)
+ git_command(repo_path, cmd)
+ return "push %s HEAD:refs/heads/master"
+
+
+def update_local_copy(repo_path, track_upstream, git_opts, ssh_env):
+ # If we're configured to track upstream but the repo
+ # does not have an upstream remote, add one
+ if (track_upstream and
+ 'upstream' not in git_command_output(
+ repo_path, 'remote')[1]):
+ git_command(
+ repo_path,
+ "remote add upstream %(upstream)s" % git_opts)
+
+ # If we're configured to track upstream, make sure that
+ # the upstream URL matches the config
+ else:
+ git_command(
+ repo_path,
+ "remote set-url upstream %(upstream)s" % git_opts)
+
+ # Now that we have any upstreams configured, fetch all of the refs
+ # we might need, pruning remote branches that no longer exist
+ git_command(
+ repo_path, "remote update --prune", env=ssh_env)
+
+ # TODO(mordred): This is here so that later we can
+ # inspect the master branch for meta-info
+ # Checkout master and reset to the state of origin/master
+ git_command(repo_path, "checkout -B master origin/master")
+
+
+def push_to_gerrit(repo_path, project, push_string, remote_url, ssh_env):
+ try:
+ git_command(repo_path, push_string % remote_url, env=ssh_env)
+ git_command(repo_path, "push --tags %s" % remote_url, env=ssh_env)
+ except Exception:
+ log.exception(
+ "Error pushing %s to Gerrit." % project)
+
+
+def sync_upstream(repo_path, project, ssh_env, upstream_prefix):
+ git_command(
+ repo_path,
+ "remote update upstream --prune", env=ssh_env)
+ # Any branch that exists in the upstream remote, we want
+ # a local branch of, optionally prefixed with the
+ # upstream prefix value
+ for branch in git_command_output(
+ repo_path, "branch -a")[1].split():
+ if not branch.strip().startswith("remotes/upstream"):
+ continue
+ if "->" in branch:
+ continue
+ local_branch = branch[len('remotes/upstream/'):]
+ if upstream_prefix:
+ local_branch = "%s/%s" % (
+ upstream_prefix, local_branch)
+
+ # Check out an up to date copy of the branch, so that
+ # we can push it and it will get picked up below
+ git_command(repo_path, "checkout -B %s %s" % (
+ local_branch, branch))
+
+ try:
+ # Push all of the local branches to similarly named
+ # Branches on gerrit. Also, push all of the tags
+ git_command(
+ repo_path,
+ "push origin refs/heads/*:refs/heads/*",
+ env=ssh_env)
+ git_command(repo_path, 'push origin --tags', env=ssh_env)
+ except Exception:
+ log.exception(
+ "Error pushing %s to Gerrit." % project)
+
+
+def process_acls(acl_config, project, ACL_DIR, section,
+ remote_url, repo_path, ssh_env, gerrit, GERRIT_GITID):
+ if not os.path.isfile(acl_config):
+ write_acl_config(project,
+ ACL_DIR,
+ section.get('acl-base', None),
+ section.get('acl-append', []),
+ section.get('acl-parameters', {}))
+ try:
+ if (fetch_config(project,
+ remote_url,
+ repo_path,
+ ssh_env) and
+ copy_acl_config(project, repo_path,
+ acl_config) and
+ create_groups_file(project, gerrit, repo_path)):
+ push_acl_config(project,
+ remote_url,
+ repo_path,
+ GERRIT_GITID,
+ ssh_env)
+ except Exception:
+ log.exception(
+ "Exception processing ACLS for %s." % project)
+ finally:
+ git_command(repo_path, 'reset --hard')
+ git_command(repo_path, 'checkout master')
+ git_command(repo_path, 'branch -D config')
+
+
+def create_gerrit_project(project, project_list, gerrit):
+ if project not in project_list:
+ try:
+ gerrit.createProject(project)
+ return True
+ except Exception:
+ log.exception(
+ "Exception creating %s in Gerrit." % project)
+ raise
+ return False
+
+
+def create_local_mirror(local_git_dir, project_git,
+ gerrit_system_user, gerrit_system_group):
+
+ git_mirror_path = os.path.join(local_git_dir, project_git)
+ if not os.path.exists(git_mirror_path):
+ (ret, output) = run_command_status(
+ "git --bare init %s" % git_mirror_path)
+ if ret:
+ run_command("rm -rf git_mirror_path")
+ raise Exception(output)
+ run_command("chown -R %s:%s %s"
+ % (gerrit_system_user, gerrit_system_group,
+ git_mirror_path))
+
+
def main():
parser = argparse.ArgumentParser(description='Manage projects')
parser.add_argument('-v', dest='verbose', action='store_true',
@@ -346,225 +549,81 @@
project = section['project']
if args.projects and project not in args.projects:
continue
- options = section.get('options', dict())
- description = section.get('description', None)
- homepage = section.get('homepage', defaults.get('homepage', None))
- upstream = section.get('upstream', None)
- upstream_prefix = section.get('upstream-prefix', None)
- track_upstream = 'track-upstream' in options
-
- project_git = "%s.git" % project
- remote_url = "ssh://localhost:%s/%s" % (GERRIT_PORT, project)
-
- # Create the project in Gerrit first, since it will fail
- # spectacularly if its project directory or local replica
- # already exist on disk
- project_created = False
- if project not in project_list:
- try:
- gerrit.createProject(project)
- project_created = True
- except Exception:
- log.exception(
- "Exception creating %s in Gerrit." % project)
-
- # Create the repo for the local git mirror
- git_mirror_path = os.path.join(LOCAL_GIT_DIR, project_git)
- if not os.path.exists(git_mirror_path):
- run_command("git --bare init %s" % git_mirror_path)
- run_command("chown -R %s:%s %s"
- % (GERRIT_SYSTEM_USER, GERRIT_SYSTEM_GROUP,
- git_mirror_path))
-
- # Start processing the local JEEPYB cache
- repo_path = os.path.join(JEEPYB_CACHE_DIR, project)
- git_opts = dict(upstream=upstream,
- repo_path=repo_path,
- remote_url=remote_url)
-
- # If we don't have a local copy already, get one
- if not os.path.exists(repo_path) or project_created:
- if not os.path.exists(os.path.dirname(repo_path)):
- os.makedirs(os.path.dirname(repo_path))
- # Three choices
- # - If gerrit has it, get from gerrit
- # - If gerrit doesn't have it:
- # - If it has an upstream, clone that
- # - If it doesn't, create it
-
- # Gerrit knows about the project, clone it
- if project in project_list:
- run_command(
- "git clone %(remote_url)s %(repo_path)s" % git_opts,
- env=ssh_env)
- if upstream:
- git_command(
- repo_path,
- "remote add -f upstream %(upstream)s" % git_opts)
-
- # Gerrit doesn't have it, but it has an upstream configured
- # We're probably importing it for the first time, clone
- # upstream, but then ongoing we want gerrit to ge origin
- # and upstream to be only there for ongoing tracking
- # purposes, so rename origin to upstream and add a new
- # origin remote that points at gerrit
- elif upstream:
- run_command(
- "git clone %(upstream)s %(repo_path)s" % git_opts,
- env=ssh_env)
- git_command(
- repo_path,
- "fetch origin +refs/heads/*:refs/copy/heads/*",
- env=ssh_env)
- git_command(repo_path, "remote rename origin upstream")
- git_command(
- repo_path,
- "remote add origin %(remote_url)s" % git_opts)
- push_string = "push %s +refs/copy/heads/*:refs/heads/*"
-
- # Neither gerrit has it, nor does it have an upstream,
- # just create a whole new one
- else:
- run_command("git init %s" % repo_path)
- git_command(
- repo_path,
- "remote add origin %(remote_url)s" % git_opts)
- with open(os.path.join(repo_path,
- ".gitreview"),
- 'w') as gitreview:
- gitreview.write("""[gerrit]
-host=%s
-port=%s
-project=%s
-""" % (GERRIT_HOST, GERRIT_PORT, project_git))
- git_command(repo_path, "add .gitreview")
- cmd = ("commit -a -m'Added .gitreview' --author='%s'"
- % GERRIT_GITID)
- git_command(repo_path, cmd)
- push_string = "push %s HEAD:refs/heads/master"
-
- # We do have a local copy of it already, make sure it's
- # in shape to have work done.
- else:
- if project in project_list:
-
- # If we're configured to track upstream but the repo
- # does not have an upstream remote, add one
- if (track_upstream and
- 'upstream' not in git_command_output(
- repo_path, 'remote')[1]):
- git_command(
- repo_path,
- "remote add upstream %(upstream)s" % git_opts)
-
- # If we're configured to track upstream, make sure that
- # the upstream URL matches the config
- else:
- git_command(
- repo_path,
- "remote set-url upstream %(upstream)s" % git_opts)
-
- # Now that we have any upstreams configured, fetch all
- # of the refs we might need, pruning remote branches
- # that no longer exist
- git_command(
- repo_path, "remote update --prune", env=ssh_env)
-
- # TODO(mordred): This is here so that later we can
- # inspect the master branch for meta-info
- # Checkout master and reset to the state of origin/master
- git_command(repo_path, "checkout -B master origin/master")
-
- description = find_description_override(repo_path) or description
-
- if 'has-github' in options or default_has_github:
- create_github_project(defaults, options, project,
- description, homepage)
-
- if project_created:
- try:
- git_command(repo_path,
- push_string % remote_url,
- env=ssh_env)
- git_command(repo_path,
- "push --tags %s" % remote_url, env=ssh_env)
- except Exception:
- log.exception(
- "Error pushing %s to Gerrit." % project)
-
- # If we're configured to track upstream, make sure we have
- # upstream's refs, and then push them to the appropriate
- # branches in gerrit
- if track_upstream:
- git_command(
- repo_path,
- "remote update upstream --prune", env=ssh_env)
- # Any branch that exists in the upstream remote, we want
- # a local branch of, optionally prefixed with the
- # upstream prefix value
- for branch in git_command_output(
- repo_path, "branch -a")[1].split():
- if not branch.strip().startswith("remotes/upstream"):
- continue
- if "->" in branch:
- continue
- local_branch = branch[len('remotes/upstream/'):]
- if upstream_prefix:
- local_branch = "%s/%s" % (
- upstream_prefix, local_branch)
-
- # Check out an up to date copy of the branch, so that
- # we can push it and it will get picked up below
- git_command(repo_path, "checkout -B %s %s" % (
- local_branch, branch))
-
- try:
- # Push all of the local branches to similarly named
- # Branches on gerrit. Also, push all of the tags
- git_command(
- repo_path,
- "push origin refs/heads/*:refs/heads/*",
- env=ssh_env)
- git_command(repo_path, 'push origin --tags', env=ssh_env)
- except Exception:
- log.exception(
- "Error pushing %s to Gerrit." % project)
try:
- acl_config = section.get('acl-config',
- '%s.config' % os.path.join(ACL_DIR,
- project))
- except AttributeError:
- acl_config = None
+ # Figure out all of the options
+ options = section.get('options', dict())
+ description = section.get('description', None)
+ homepage = section.get(
+ 'homepage', defaults.get('homepage', None))
+ upstream = section.get('upstream', None)
+ upstream_prefix = section.get('upstream-prefix', None)
+ track_upstream = 'track-upstream' in options
+ repo_path = os.path.join(JEEPYB_CACHE_DIR, project)
- if acl_config:
- if not os.path.isfile(acl_config):
- write_acl_config(project,
- ACL_DIR,
- section.get('acl-base', None),
- section.get('acl-append', []),
- section.get('acl-parameters', {}))
- try:
- if (fetch_config(project,
- remote_url,
- repo_path,
- ssh_env) and
- copy_acl_config(project, repo_path,
- acl_config) and
- create_groups_file(project, gerrit, repo_path)):
- push_acl_config(project,
- remote_url,
- repo_path,
- GERRIT_GITID,
- ssh_env)
- except Exception:
- log.exception(
- "Exception processing ACLS for %s." % project)
- finally:
- git_command(repo_path, 'reset --hard')
- git_command(repo_path, 'checkout master')
- git_command(repo_path, 'branch -D config')
+ project_git = "%s.git" % project
+ remote_url = "ssh://localhost:%s/%s" % (GERRIT_PORT, project)
+ git_opts = dict(upstream=upstream,
+ repo_path=repo_path,
+ remote_url=remote_url)
+ acl_config = section.get(
+ 'acl-config',
+ '%s.config' % os.path.join(ACL_DIR, project))
+ # Create the project in Gerrit first, since it will fail
+ # spectacularly if its project directory or local replica
+ # already exist on disk
+ project_created = create_gerrit_project(
+ project, project_list, gerrit)
+
+ # Create the repo for the local git mirror
+ create_local_mirror(
+ LOCAL_GIT_DIR, project_git,
+ GERRIT_SYSTEM_USER, GERRIT_SYSTEM_GROUP)
+
+ if not os.path.exists(repo_path) or project_created:
+ # We don't have a local copy already, get one
+
+ # Make Local repo
+ push_string = make_local_copy(
+ repo_path, project, project_list,
+ git_opts, ssh_env, upstream, GERRIT_HOST, GERRIT_PORT,
+ project_git, GERRIT_GITID)
+ else:
+ # We do have a local copy of it already, make sure it's
+ # in shape to have work done.
+ update_local_copy(
+ repo_path, track_upstream, git_opts, ssh_env)
+
+ description = (
+ find_description_override(repo_path) or description)
+
+ if project_created:
+ push_to_gerrit(
+ repo_path, project, push_string, remote_url, ssh_env)
+ gerrit.replicate(project)
+
+ # If we're configured to track upstream, make sure we have
+ # upstream's refs, and then push them to the appropriate
+ # branches in gerrit
+ if track_upstream:
+ sync_upstream(repo_path, project, ssh_env, upstream_prefix)
+
+ if acl_config:
+ process_acls(
+ acl_config, project, ACL_DIR, section,
+ remote_url, repo_path, ssh_env, gerrit, GERRIT_GITID)
+
+ if 'has-github' in options or default_has_github:
+ created = create_github_project(
+ defaults, options, project, description, homepage)
+ if created:
+ gerrit.replicate(project)
+
+ except Exception:
+ log.exception(
+ "Problems creating %s, moving on." % project)
+ continue
finally:
os.unlink(ssh_env['GIT_SSH'])
diff --git a/jeepyb/projects.py b/jeepyb/projects.py
index 19f178b..cf126a2 100644
--- a/jeepyb/projects.py
+++ b/jeepyb/projects.py
@@ -82,10 +82,6 @@
description: Best project ever.
"""
return project_full_name in [
- 'openstack/openstack-manuals',
- 'openstack/api-site',
- 'openstack/tripleo-incubator',
- 'openstack/tempest',
'openstack-dev/devstack',
'openstack-infra/askbot-theme',
'openstack-infra/config',
@@ -100,6 +96,12 @@
'openstack-infra/publications',
'openstack-infra/reviewday',
'openstack-infra/statusbot',
+ 'openstack/api-site',
+ 'openstack/openstack-manuals',
+ 'openstack/tempest',
+ 'openstack/tripleo-heat-templates',
+ 'openstack/tripleo-image-elements',
+ 'openstack/tripleo-incubator',
'stackforge/cookbook-openstack-block-storage',
'stackforge/cookbook-openstack-common',
'stackforge/cookbook-openstack-compute',
@@ -113,8 +115,6 @@
'stackforge/cookbook-openstack-ops-messaging',
'stackforge/cookbook-openstack-orchestration',
'stackforge/openstack-chef-repo',
- 'stackforge/tripleo-heat-templates',
- 'stackforge/tripleo-image-elements',
]
@@ -132,24 +132,6 @@
"""
project_map = {
- 'openstack/api-site': 'openstack-api-site',
- 'openstack/identity-api': 'openstack-api-site',
- 'openstack/object-api': 'openstack-api-site',
- 'openstack/volume-api': 'openstack-api-site',
- 'openstack/netconn-api': 'openstack-api-site',
- 'openstack/compute-api': 'openstack-api-site',
- 'openstack/image-api': 'openstack-api-site',
- 'openstack/database-api': 'openstack-api-site',
- 'openstack/quantum': 'neutron',
- 'openstack/python-quantumclient': 'python-neutronclient',
- 'openstack/oslo-incubator': 'oslo',
- 'openstack/tripleo-incubator': 'tripleo',
- 'openstack/django_openstack_auth': 'django-openstack-auth',
- 'openstack/savanna': 'savanna',
- 'openstack/python-savannaclient': 'savanna',
- 'openstack/savanna-dashboard': 'savanna',
- 'openstack/savanna-image-elements': 'savanna',
- 'openstack/savanna-extra': 'savanna',
'openstack-infra/askbot-theme': 'openstack-ci',
'openstack-infra/config': 'openstack-ci',
'openstack-infra/devstack-gate': 'openstack-ci',
@@ -170,6 +152,25 @@
'openstack-infra/reviewday': 'openstack-ci',
'openstack-infra/statusbot': 'openstack-ci',
'openstack-infra/zmq-event-publisher': 'openstack-ci',
+ 'openstack/api-site': 'openstack-api-site',
+ 'openstack/compute-api': 'openstack-api-site',
+ 'openstack/database-api': 'openstack-api-site',
+ 'openstack/django_openstack_auth': 'django-openstack-auth',
+ 'openstack/identity-api': 'openstack-api-site',
+ 'openstack/image-api': 'openstack-api-site',
+ 'openstack/netconn-api': 'openstack-api-site',
+ 'openstack/object-api': 'openstack-api-site',
+ 'openstack/oslo-incubator': 'oslo',
+ 'openstack/python-quantumclient': 'python-neutronclient',
+ 'openstack/quantum': 'neutron',
+ 'openstack/savanna': 'savanna',
+ 'openstack/savanna-dashboard': 'savanna',
+ 'openstack/savanna-extra': 'savanna',
+ 'openstack/savanna-image-elements': 'savanna',
+ 'openstack/tripleo-heat-templates': 'tripleo',
+ 'openstack/tripleo-image-elements': 'tripleo',
+ 'openstack/tripleo-incubator': 'tripleo',
+ 'openstack/volume-api': 'openstack-api-site',
'stackforge/cookbook-openstack-block-storage': 'openstack-chef',
'stackforge/cookbook-openstack-common': 'openstack-chef',
'stackforge/cookbook-openstack-compute': 'openstack-chef',
@@ -182,16 +183,14 @@
'stackforge/cookbook-openstack-ops-database': 'openstack-chef',
'stackforge/cookbook-openstack-ops-messaging': 'openstack-chef',
'stackforge/cookbook-openstack-orchestration': 'openstack-chef',
+ 'stackforge/fuel-astute': 'fuel',
+ 'stackforge/fuel-main': 'fuel',
+ 'stackforge/fuel-ostf': 'fuel',
+ 'stackforge/fuel-web': 'fuel',
'stackforge/openstack-chef-repo': 'openstack-chef',
'stackforge/puppet-openstack_dev_env': 'puppet-openstack',
'stackforge/puppet-quantum': 'puppet-neutron',
- 'stackforge/tripleo-heat-templates': 'tripleo',
- 'stackforge/tripleo-image-elements': 'tripleo',
'stackforge/puppet-savanna': 'savanna',
- 'stackforge/fuel-web': 'fuel',
- 'stackforge/fuel-astute': 'fuel',
- 'stackforge/fuel-ostf': 'fuel',
- 'stackforge/fuel-main': 'fuel'
}
return project_map.get(project_full_name,
u.short_project_name(project_full_name))
diff --git a/requirements.txt b/requirements.txt
index 994ad98..f2b536e 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,5 +1,5 @@
argparse
-gerritlib
+gerritlib>=0.3.0
MySQL-python
paramiko
PyGithub
diff --git a/tox.ini b/tox.ini
index 47067e0..925577c 100644
--- a/tox.ini
+++ b/tox.ini
@@ -16,5 +16,7 @@
commands = {posargs}
[flake8]
+# E125 and H are intentionally ignored
+ignore = E125,H
show-source = True
exclude = .venv,.tox,dist,doc,build,*.egg