Merge "Keep py3.X compatibility for urllib"
diff --git a/jeepyb/cmd/close_pull_requests.py b/jeepyb/cmd/close_pull_requests.py
index 19d2d25..f890bb1 100644
--- a/jeepyb/cmd/close_pull_requests.py
+++ b/jeepyb/cmd/close_pull_requests.py
@@ -42,7 +42,8 @@
import github
import logging
import os
-import yaml
+
+import jeepyb.utils as u
MESSAGE = """Thank you for contributing to %(project)s!
@@ -57,20 +58,12 @@
logging.basicConfig(level=logging.ERROR)
- PROJECTS_YAML = os.environ.get('PROJECTS_YAML',
- '/home/gerrit2/projects.yaml')
- PROJECTS_INI = os.environ.get('PROJECTS_INI',
- '/home/gerrit2/projects.ini')
GITHUB_SECURE_CONFIG = os.environ.get('GITHUB_SECURE_CONFIG',
'/etc/github/github.secure.config')
secure_config = ConfigParser.ConfigParser()
secure_config.read(GITHUB_SECURE_CONFIG)
- yaml_docs = [config for config in yaml.load_all(open(PROJECTS_YAML))]
- if os.path.exists(PROJECTS_INI):
- config = yaml_docs[0]
- else:
- config = yaml_docs[1]
+ registry = u.ProjectsRegistry()
if secure_config.has_option("github", "oauth_token"):
ghub = github.Github(secure_config.get("github", "oauth_token"))
@@ -80,7 +73,7 @@
orgs = ghub.get_user().get_orgs()
orgs_dict = dict(zip([o.login.lower() for o in orgs], orgs))
- for section in config:
+ for section in registry.configs_list:
project = section['project']
# Make sure we're supposed to close pull requests for this project:
diff --git a/jeepyb/cmd/create_cgitrepos.py b/jeepyb/cmd/create_cgitrepos.py
index 1572a44..6950655 100644
--- a/jeepyb/cmd/create_cgitrepos.py
+++ b/jeepyb/cmd/create_cgitrepos.py
@@ -22,15 +22,13 @@
import os
import subprocess
-import yaml
+
+import jeepyb.utils as u
-PROJECTS_YAML = os.environ.get('PROJECTS_YAML',
- '/home/cgit/projects.yaml')
-CGIT_REPOS = os.environ.get('CGIT_REPOS',
- '/etc/cgitrepos')
-REPO_PATH = os.environ.get('REPO_PATH',
- '/var/lib/git')
+PROJECTS_YAML = os.environ.get('PROJECTS_YAML', '/home/cgit/projects.yaml')
+CGIT_REPOS = os.environ.get('CGIT_REPOS', '/etc/cgitrepos')
+REPO_PATH = os.environ.get('REPO_PATH', '/var/lib/git')
SCRATCH_SUBPATH = os.environ.get('SCRATCH_SUBPATH')
SCRATCH_OWNER = os.environ.get('SCRATCH_OWNER', 'scratch')
SCRATCH_GROUP = os.environ.get('SCRATCH_GROUP', 'scratch')
@@ -39,11 +37,10 @@
def main():
- yaml_docs = [config for config in yaml.safe_load_all(open(PROJECTS_YAML))]
- config = yaml_docs[-1]
+ registry = u.ProjectsRegistry(PROJECTS_YAML)
gitorgs = {}
names = set()
- for entry in config:
+ for entry in registry.configs_list:
project = entry['project']
(org, name) = project.split('/')
description = entry.get('description', name)
diff --git a/jeepyb/cmd/manage_projects.py b/jeepyb/cmd/manage_projects.py
index 372e2ed..2dece22 100644
--- a/jeepyb/cmd/manage_projects.py
+++ b/jeepyb/cmd/manage_projects.py
@@ -58,12 +58,14 @@
import subprocess
import tempfile
import time
-import yaml
import gerritlib.gerrit
import github
import jeepyb.gerritdb
+import jeepyb.utils as u
+
+registry = u.ProjectsRegistry()
log = logging.getLogger("manage_projects")
@@ -76,8 +78,8 @@
p = subprocess.Popen(cmd_list, stdout=subprocess.PIPE,
stderr=subprocess.STDOUT, env=newenv)
(out, nothing) = p.communicate()
- log.debug("Return code: %s" % p.returncode)
- log.debug("Command said: %s" % out.strip())
+ log.info("Return code: %s" % p.returncode)
+ log.info("Command said: %s" % out.strip())
if status:
return (p.returncode, out.strip())
return out.strip()
@@ -128,8 +130,19 @@
def fetch_config(project, remote_url, repo_path, env={}):
- status = git_command(repo_path, "fetch %s +refs/meta/config:"
- "refs/remotes/gerrit-meta/config" % remote_url, env)
+ # Poll for refs/meta/config as gerrit may not have written it out for
+ # us yet.
+ for x in range(10):
+ status = git_command(repo_path, "fetch %s +refs/meta/config:"
+ "refs/remotes/gerrit-meta/config" %
+ remote_url, env)
+ if status == 0:
+ break
+ else:
+ log.debug("Failed to fetch refs/meta/config for project: %s" %
+ project)
+ time.sleep(2)
+
if status != 0:
log.error("Failed to fetch refs/meta/config for project: %s" % project)
return False
@@ -370,26 +383,32 @@
def update_local_copy(repo_path, track_upstream, git_opts, ssh_env):
- # If we're configured to track upstream but the repo
- # does not have an upstream remote, add one
- if (track_upstream and
- 'upstream' not in git_command_output(
- repo_path, 'remote')[1]):
- git_command(
- repo_path,
- "remote add upstream %(upstream)s" % git_opts)
+ has_upstream_remote = (
+ 'upstream' in git_command_output(repo_path, 'remote')[1])
+ if track_upstream:
+ # If we're configured to track upstream but the repo
+ # does not have an upstream remote, add one
+ if not has_upstream_remote:
+ git_command(
+ repo_path,
+ "remote add upstream %(upstream)s" % git_opts)
- # If we're configured to track upstream, make sure that
- # the upstream URL matches the config
+ # If we're configured to track upstream, make sure that
+ # the upstream URL matches the config
+ else:
+ git_command(
+ repo_path,
+ "remote set-url upstream %(upstream)s" % git_opts)
+
+ # Now that we have any upstreams configured, fetch all of the refs
+ # we might need, pruning remote branches that no longer exist
+ git_command(
+ repo_path, "remote update --prune", env=ssh_env)
else:
- git_command(
- repo_path,
- "remote set-url upstream %(upstream)s" % git_opts)
-
- # Now that we have any upstreams configured, fetch all of the refs
- # we might need, pruning remote branches that no longer exist
- git_command(
- repo_path, "remote update --prune", env=ssh_env)
+ # If we are not tracking upstream, then we do not need
+ # an upstream remote configured
+ if has_upstream_remote:
+ git_command(repo_path, "remote rm upstream")
# TODO(mordred): This is here so that later we can
# inspect the master branch for meta-info
@@ -414,12 +433,12 @@
# a local branch of, optionally prefixed with the
# upstream prefix value
for branch in git_command_output(
- repo_path, "branch -a")[1].split():
+ repo_path, "branch -a")[1].split('\n'):
if not branch.strip().startswith("remotes/upstream"):
continue
if "->" in branch:
continue
- local_branch = branch[len('remotes/upstream/'):]
+ local_branch = branch.split()[0][len('remotes/upstream/'):]
if upstream_prefix:
local_branch = "%s/%s" % (
upstream_prefix, local_branch)
@@ -499,19 +518,12 @@
git_mirror_path))
-def get_option(options, section, key, default):
- if options.has_option(section, key):
- if type(default) is bool:
- return options.getboolean(section, key)
- else:
- return options.get(section, key)
- return default
-
-
def main():
parser = argparse.ArgumentParser(description='Manage projects')
parser.add_argument('-v', dest='verbose', action='store_true',
help='verbose output')
+ parser.add_argument('-d', dest='debug', action='store_true',
+ help='debug output')
parser.add_argument('--nocleanup', action='store_true',
help='do not remove temp directories')
parser.add_argument('projects', metavar='project', nargs='*',
@@ -519,72 +531,33 @@
args = parser.parse_args()
if args.verbose:
+ logging.basicConfig(level=logging.INFO)
+ elif args.debug:
logging.basicConfig(level=logging.DEBUG)
else:
logging.basicConfig(level=logging.ERROR)
- PROJECTS_YAML = os.environ.get('PROJECTS_YAML',
- '/home/gerrit2/projects.yaml')
- yaml_docs = [config for config in yaml.safe_load_all(open(PROJECTS_YAML))]
+ default_has_github = registry.get('has-github', True)
- PROJECTS_INI = os.environ.get('PROJECTS_INI',
- '/home/gerrit2/projects.ini')
- if os.path.exists(PROJECTS_INI):
- # New-style - supports projects.ini
- projects_yaml_list = yaml_docs[0]
- defaults = ConfigParser.ConfigParser()
- defaults.read(PROJECTS_INI)
-
- default_has_github = get_option(
- defaults, 'projects', 'has-github', True)
-
- LOCAL_GIT_DIR = get_option(
- defaults, 'projects', 'local-git-dir', '/var/lib/git')
- JEEPYB_CACHE_DIR = get_option(
- defaults, 'projects', 'jeepyb-cache-dir', '/var/lib/jeepyb')
- ACL_DIR = defaults.get('projects', 'acl-dir')
- GERRIT_HOST = defaults.get('projects', 'gerrit-host')
- GERRIT_PORT = int(get_option(
- defaults, 'projects', 'gerrit-port', '29418'))
- GERRIT_USER = defaults.get('projects', 'gerrit-user')
- GERRIT_KEY = defaults.get('projects', 'gerrit-key')
- GERRIT_GITID = defaults.get('projects', 'gerrit-committer')
- GERRIT_SYSTEM_USER = get_option(
- defaults, 'projects', 'gerrit-system-user', 'gerrit2')
- GERRIT_SYSTEM_GROUP = get_option(
- defaults, 'projects', 'gerrit-system-group', 'gerrit2')
- DEFAULT_HOMEPAGE = get_option(defaults, 'projects', 'homepage', None)
- DEFAULT_HAS_ISSUES = get_option(
- defaults, 'projects', 'has-issues', False)
- DEFAULT_HAS_DOWNLOADS = get_option(
- defaults, 'projects', 'has-downloads', False)
- DEFAULT_HAS_WIKI = get_option(defaults, 'projects', 'has-wiki', False)
- GITHUB_SECURE_CONFIG = get_option(
- defaults, 'projects', 'github-config',
- '/etc/github/github-projects.secure.config')
- else:
- # Old-style - embedded
- projects_yaml_list = yaml_docs[1]
- defaults = yaml_docs[0][0]
- default_has_github = defaults.get('has-github', True)
-
- LOCAL_GIT_DIR = defaults.get('local-git-dir', '/var/lib/git')
- JEEPYB_CACHE_DIR = defaults.get('jeepyb-cache-dir', '/var/lib/jeepyb')
- ACL_DIR = defaults.get('acl-dir')
- GERRIT_HOST = defaults.get('gerrit-host')
- GERRIT_PORT = int(defaults.get('gerrit-port', '29418'))
- GERRIT_USER = defaults.get('gerrit-user')
- GERRIT_KEY = defaults.get('gerrit-key')
- GERRIT_GITID = defaults.get('gerrit-committer')
- GERRIT_SYSTEM_USER = defaults.get('gerrit-system-user', 'gerrit2')
- GERRIT_SYSTEM_GROUP = defaults.get('gerrit-system-group', 'gerrit2')
- DEFAULT_HOMEPAGE = defaults.get('homepage', None)
- DEFAULT_HAS_ISSUES = defaults.get('has-issues', False)
- DEFAULT_HAS_DOWNLOADS = defaults.get('has-downloads', False)
- DEFAULT_HAS_WIKI = defaults.get('has-wiki', False)
- GITHUB_SECURE_CONFIG = defaults.get(
- 'github-config',
- '/etc/github/github-projects.secure.config')
+ LOCAL_GIT_DIR = registry.get_defaults('local-git-dir', '/var/lib/git')
+ JEEPYB_CACHE_DIR = registry.get_defaults('jeepyb-cache-dir',
+ '/var/lib/jeepyb')
+ ACL_DIR = registry.get_defaults('acl-dir')
+ GERRIT_HOST = registry.get_defaults('gerrit-host')
+ GERRIT_PORT = int(registry.get_defaults('gerrit-port', '29418'))
+ GERRIT_USER = registry.get_defaults('gerrit-user')
+ GERRIT_KEY = registry.get_defaults('gerrit-key')
+ GERRIT_GITID = registry.get_defaults('gerrit-committer')
+ GERRIT_SYSTEM_USER = registry.get_defaults('gerrit-system-user', 'gerrit2')
+ GERRIT_SYSTEM_GROUP = registry.get_defaults('gerrit-system-group',
+ 'gerrit2')
+ DEFAULT_HOMEPAGE = registry.get_defaults('homepage')
+ DEFAULT_HAS_ISSUES = registry.get_defaults('has-issues', False)
+ DEFAULT_HAS_DOWNLOADS = registry.get_defaults('has-downloads', False)
+ DEFAULT_HAS_WIKI = registry.get_defaults('has-wiki', False)
+ GITHUB_SECURE_CONFIG = registry.get_defaults(
+ 'github-config',
+ '/etc/github/github-projects.secure.config')
gerrit = gerritlib.gerrit.Gerrit('localhost',
GERRIT_USER,
@@ -594,7 +567,7 @@
ssh_env = make_ssh_wrapper(GERRIT_USER, GERRIT_KEY)
try:
- for section in projects_yaml_list:
+ for section in registry.configs_list:
project = section['project']
if args.projects and project not in args.projects:
continue
diff --git a/jeepyb/cmd/welcome_message.py b/jeepyb/cmd/welcome_message.py
index e031ff7..3536769 100644
--- a/jeepyb/cmd/welcome_message.py
+++ b/jeepyb/cmd/welcome_message.py
@@ -58,7 +58,7 @@
cursor.execute(query, searchkey)
data = cursor.fetchone()
if data:
- if data[0] == "1":
+ if data[0] == 1:
logger.info('We found a newbie: %s', uploader)
return True
else:
@@ -136,7 +136,7 @@
# Don't actually post the message
parser.add_argument('--dryrun', dest='dryrun', action='store_true')
parser.add_argument('--no-dryrun', dest='dryrun', action='store_false')
- parser.add_argument('-v', dest='verbose', action='store_true',
+ parser.add_argument('-v', '--verbose', dest='verbose', action='store_true',
help='verbose output')
parser.set_defaults(dryrun=False)
diff --git a/jeepyb/projects.py b/jeepyb/projects.py
index 4cf9ffe..a801830 100644
--- a/jeepyb/projects.py
+++ b/jeepyb/projects.py
@@ -27,8 +27,7 @@
import jeepyb.utils as u
-registry = u.ProjectsYamlRegistry('/home/gerrit2/projects.yaml',
- 'PROJECTS_YAML')
+registry = u.ProjectsRegistry()
def project_to_group(project_full_name):
@@ -55,67 +54,11 @@
def is_direct_release(project_full_name):
try:
- direct = 'direct-release' in registry[project_full_name]['options']
- # return ...
+ return 'direct-release' in registry[project_full_name]['options']
except KeyError:
- direct = False
- # return False
-
- return direct or _hardcoded_is_direct_release(project_full_name)
+ return False
def docimpact_target(project_full_name):
return registry.get_project_item(project_full_name, 'docimpact-group',
'unknown')
-
-
-# The following functions should be deleted when projects.yaml will be updated
-
-def _hardcoded_is_direct_release(project_full_name):
- """Test against a list of projects who directly release changes.
-
- This function should be removed when projects.yaml will be updated.
- To specify direct_release you just need add option 'direct_relese' to your
- project declaration in projects.yaml
-
- Example:
- - project: some/project
- options:
- - direct-release
- description: Best project ever.
- """
- return project_full_name in [
- 'openstack-dev/devstack',
- 'openstack-infra/askbot-theme',
- 'openstack-infra/config',
- 'openstack-infra/devstack-gate',
- 'openstack-infra/gerrit',
- 'openstack-infra/gerritbot',
- 'openstack-infra/gerritlib',
- 'openstack-infra/gitdm',
- 'openstack-infra/lodgeit',
- 'openstack-infra/meetbot',
- 'openstack-infra/nose-html-output',
- 'openstack-infra/publications',
- 'openstack-infra/reviewday',
- 'openstack-infra/statusbot',
- 'openstack/api-site',
- 'openstack/openstack-manuals',
- 'openstack/tempest',
- 'openstack/tripleo-heat-templates',
- 'openstack/tripleo-image-elements',
- 'openstack/tripleo-incubator',
- 'stackforge/cookbook-openstack-block-storage',
- 'stackforge/cookbook-openstack-common',
- 'stackforge/cookbook-openstack-compute',
- 'stackforge/cookbook-openstack-dashboard',
- 'stackforge/cookbook-openstack-identity',
- 'stackforge/cookbook-openstack-image',
- 'stackforge/cookbook-openstack-metering',
- 'stackforge/cookbook-openstack-network',
- 'stackforge/cookbook-openstack-object-storage',
- 'stackforge/cookbook-openstack-ops-database',
- 'stackforge/cookbook-openstack-ops-messaging',
- 'stackforge/cookbook-openstack-orchestration',
- 'stackforge/openstack-chef-repo',
- ]
diff --git a/jeepyb/utils.py b/jeepyb/utils.py
index f92852e..7b04dc1 100644
--- a/jeepyb/utils.py
+++ b/jeepyb/utils.py
@@ -12,38 +12,49 @@
# License for the specific language governing permissions and limitations
# under the License.
+import ConfigParser
import os
import yaml
+PROJECTS_INI = os.environ.get('PROJECTS_INI', '/home/gerrit2/projects.ini')
+PROJECTS_YAML = os.environ.get('PROJECTS_YAML', '/home/gerrit2/projects.yaml')
+
def short_project_name(full_project_name):
"""Return the project part of the git repository name."""
return full_project_name.split('/')[-1]
-class ProjectsYamlRegistry(object):
- """review.projects.yaml style config file parser.
+class ProjectsRegistry(object):
+ """read config from ini or yaml file.
It could be used as dict 'project name' -> 'project properties'.
"""
-
- def __init__(self, file_path, env_name=None, single_doc=True):
- self.file_path = file_path
- self.env_name = env_name
+ def __init__(self, yaml_file=PROJECTS_YAML, single_doc=True):
+ self.yaml_doc = [c for c in yaml.safe_load_all(open(yaml_file))]
self.single_doc = single_doc
+ self.configs_list = []
+ self.defaults = {}
self._parse_file()
def _parse_file(self):
- file_path = os.environ.get(self.env_name, self.file_path)
- yaml_docs = [config for config in yaml.safe_load_all(open(file_path))]
if self.single_doc:
- configs_list = yaml_docs[0]
+ self.configs_list = self.yaml_doc[0]
else:
- configs_list = yaml_docs[1]
+ self.configs_list = self.yaml_doc[1]
+
+ if os.path.exists(PROJECTS_INI):
+ self.defaults = ConfigParser.ConfigParser()
+ self.defaults.read(PROJECTS_INI)
+ else:
+ try:
+ self.defaults = self.yaml_doc[0][0]
+ except IndexError:
+ pass
configs = {}
- for section in configs_list:
+ for section in self.configs_list:
configs[section['project']] = section
self.configs = configs
@@ -56,3 +67,18 @@
return self.configs[project].get(item, default)
else:
return default
+
+ def get(self, item, default=None):
+ return self.configs.get(item, default)
+
+ def get_defaults(self, item, default=None):
+ if os.path.exists(PROJECTS_INI):
+ section = 'projects'
+ if self.defaults.has_option(section, item):
+ if type(default) == bool:
+ return self.defaults.getboolean(section, item)
+ else:
+ return self.defaults.get(section, item)
+ return default
+ else:
+ return self.defaults.get(item, default)