Redrive rate limited API calls.
Transparently redrive calls throttled by nova rate limiting, with
an intra-retry delay governed by the Retry-After response header.
Invocations are retried at most twice.
This avoids spurious test failures when the target nova installation
is using the default rate limits, or explicitly configured limits too
low for the cadence of tempest test cases.
Change-Id: I6c701b24f59998a98e6f515a2861a458470ee40c
diff --git a/tempest/common/rest_client.py b/tempest/common/rest_client.py
index 1530313..7dd3a86 100644
--- a/tempest/common/rest_client.py
+++ b/tempest/common/rest_client.py
@@ -2,9 +2,14 @@
import httplib2
import logging
import sys
+import time
from tempest import exceptions
+# redrive rate limited calls at most twice
+MAX_RECURSION_DEPTH = 2
+
+
class RestClient(object):
def __init__(self, config, user, key, auth_url, service, tenant_name=None):
@@ -107,7 +112,7 @@
self.log.error('Response Headers: ' + str(resp))
self.log.error('Response Body: ' + str(resp_body))
- def request(self, method, url, headers=None, body=None):
+ def request(self, method, url, headers=None, body=None, depth=0):
"""A simple HTTP request interface."""
self.http_obj = httplib2.Http()
@@ -138,6 +143,10 @@
self._log(req_url, body, resp, resp_body)
if 'overLimit' in resp_body:
raise exceptions.OverLimit(resp_body['overLimit']['message'])
+ elif depth < MAX_RECURSION_DEPTH:
+ delay = resp['Retry-After'] if 'Retry-After' in resp else 60
+ time.sleep(int(delay))
+ return self.request(method, url, headers, body, depth + 1)
else:
raise exceptions.RateLimitExceeded(
message=resp_body['overLimitFault']['message'],