Retry http requests to generate plugins list
This commit retries http requests to generate a plugins list. We often
see internal server errors(500) in the gate. It makes sanity jobs fail.
So, it would be better to retry internally instead of 'recheck'. To
accomplish it, this uses urllib3 module.
NOTE: The default total number of retries[0] is 10. It should be enough
for us.
[0] https://urllib3.readthedocs.io/en/latest/reference/urllib3.util.html#urllib3.util.retry.Retry
Change-Id: Id3da3e2895fc2324d5a2db91f82b12ec5bb610f8
diff --git a/tools/generate-tempest-plugins-list.py b/tools/generate-tempest-plugins-list.py
index f1003a0..165af45 100644
--- a/tools/generate-tempest-plugins-list.py
+++ b/tools/generate-tempest-plugins-list.py
@@ -27,14 +27,8 @@
import re
import sys
-try:
- # For Python 3.0 and later
- from urllib.error import HTTPError
- import urllib.request as urllib
-except ImportError:
- # Fall back to Python 2's urllib2
- import urllib2 as urllib
- from urllib2 import HTTPError
+import urllib3
+from urllib3.util import retry
# List of projects having tempest plugin stale or unmaintained for a long time
# (6 months or more)
@@ -69,23 +63,21 @@
},
'''
+http = urllib3.PoolManager(cert_reqs='CERT_REQUIRED')
+retries = retry.Retry(status_forcelist=[500], backoff_factor=1.0)
-# Rather than returning a 404 for a nonexistent file, cgit delivers a
-# 0-byte response to a GET request. It also does not provide a
-# Content-Length in a HEAD response, so the way we tell if a file exists
-# is to check the length of the entire GET response body.
+
def has_tempest_plugin(proj):
try:
- r = urllib.urlopen(
- "https://opendev.org/%s/raw/branch/"
- "master/setup.cfg" % proj)
- except HTTPError as err:
- if err.code == 404:
+ r = http.request('GET', "https://opendev.org/%s/raw/branch/"
+ "master/setup.cfg" % proj, retries=retries)
+ if r.status == 404:
return False
+ except urllib3.exceptions.MaxRetryError as err:
# We should not ignore non 404 errors.
raise err
p = re.compile(r'^tempest\.test_plugins', re.M)
- if p.findall(r.read().decode('utf-8')):
+ if p.findall(r.data.decode('utf-8')):
return True
else:
False
@@ -98,11 +90,11 @@
# So, this exits here.
sys.exit()
-r = urllib.urlopen(url)
+r = http.request('GET', url, retries=retries)
# Gerrit prepends 4 garbage octets to the JSON, in order to counter
# cross-site scripting attacks. Therefore we must discard it so the
# json library won't choke.
-content = r.read().decode('utf-8')[4:]
+content = r.data.decode('utf-8')[4:]
projects = sorted(json.loads(content))
# Retrieve projects having no deployment tool repo (such as deb,