Merge "Swift object client: use urllib3 builtin support for chunked transfer"
diff --git a/releasenotes/notes/support-chunked-encoding-d71f53225f68edf3.yaml b/releasenotes/notes/support-chunked-encoding-d71f53225f68edf3.yaml
new file mode 100644
index 0000000..eb45523
--- /dev/null
+++ b/releasenotes/notes/support-chunked-encoding-d71f53225f68edf3.yaml
@@ -0,0 +1,9 @@
+---
+features:
+ - The RestClient (in tempest.lib.common.rest_client) now supports POSTing
+ and PUTing data with chunked transfer encoding. Just pass an `iterable`
+ object as the `body` argument and set the `chunked` argument to `True`.
+ - A new generator called `chunkify` is added in
+ tempest.lib.common.utils.data_utils that yields fixed-size chunks (slices)
+ from a Python sequence.
+
diff --git a/tempest/api/object_storage/test_object_services.py b/tempest/api/object_storage/test_object_services.py
index dc926e0..a88e4f4 100644
--- a/tempest/api/object_storage/test_object_services.py
+++ b/tempest/api/object_storage/test_object_services.py
@@ -20,7 +20,6 @@
import zlib
import six
-from six import moves
from tempest.api.object_storage import base
from tempest.common import custom_matchers
@@ -201,8 +200,8 @@
status, _, resp_headers = self.object_client.put_object_with_chunk(
container=self.container_name,
name=object_name,
- contents=moves.cStringIO(data),
- chunk_size=512)
+ contents=data_utils.chunkify(data, 512)
+ )
self.assertHeaders(resp_headers, 'Object', 'PUT')
# check uploaded content
diff --git a/tempest/lib/common/rest_client.py b/tempest/lib/common/rest_client.py
index 30750de..179db17 100644
--- a/tempest/lib/common/rest_client.py
+++ b/tempest/lib/common/rest_client.py
@@ -243,7 +243,8 @@
details = pattern.format(read_code, expected_code)
raise exceptions.InvalidHttpSuccessCode(details)
- def post(self, url, body, headers=None, extra_headers=False):
+ def post(self, url, body, headers=None, extra_headers=False,
+ chunked=False):
"""Send a HTTP POST request using keystone auth
:param str url: the relative url to send the post request to
@@ -253,11 +254,12 @@
returned by the get_headers() method are to
be used but additional headers are needed in
the request pass them in as a dict.
+ :param bool chunked: sends the body with chunked encoding
:return: a tuple with the first entry containing the response headers
and the second the response body
:rtype: tuple
"""
- return self.request('POST', url, extra_headers, headers, body)
+ return self.request('POST', url, extra_headers, headers, body, chunked)
def get(self, url, headers=None, extra_headers=False):
"""Send a HTTP GET request using keystone service catalog and auth
@@ -306,7 +308,7 @@
"""
return self.request('PATCH', url, extra_headers, headers, body)
- def put(self, url, body, headers=None, extra_headers=False):
+ def put(self, url, body, headers=None, extra_headers=False, chunked=False):
"""Send a HTTP PUT request using keystone service catalog and auth
:param str url: the relative url to send the post request to
@@ -316,11 +318,12 @@
returned by the get_headers() method are to
be used but additional headers are needed in
the request pass them in as a dict.
+ :param bool chunked: sends the body with chunked encoding
:return: a tuple with the first entry containing the response headers
and the second the response body
:rtype: tuple
"""
- return self.request('PUT', url, extra_headers, headers, body)
+ return self.request('PUT', url, extra_headers, headers, body, chunked)
def head(self, url, headers=None, extra_headers=False):
"""Send a HTTP HEAD request using keystone service catalog and auth
@@ -520,7 +523,7 @@
if method != 'HEAD' and not resp_body and resp.status >= 400:
self.LOG.warning("status >= 400 response with empty body")
- def _request(self, method, url, headers=None, body=None):
+ def _request(self, method, url, headers=None, body=None, chunked=False):
"""A simple HTTP request interface."""
# Authenticate the request with the auth provider
req_url, req_headers, req_body = self.auth_provider.auth_request(
@@ -530,7 +533,9 @@
start = time.time()
self._log_request_start(method, req_url)
resp, resp_body = self.raw_request(
- req_url, method, headers=req_headers, body=req_body)
+ req_url, method, headers=req_headers, body=req_body,
+ chunked=chunked
+ )
end = time.time()
self._log_request(method, req_url, resp, secs=(end - start),
req_headers=req_headers, req_body=req_body,
@@ -541,7 +546,7 @@
return resp, resp_body
- def raw_request(self, url, method, headers=None, body=None):
+ def raw_request(self, url, method, headers=None, body=None, chunked=False):
"""Send a raw HTTP request without the keystone catalog or auth
This method sends a HTTP request in the same manner as the request()
@@ -554,17 +559,18 @@
:param str headers: Headers to use for the request if none are specifed
the headers
:param str body: Body to send with the request
+ :param bool chunked: sends the body with chunked encoding
:rtype: tuple
:return: a tuple with the first entry containing the response headers
and the second the response body
"""
if headers is None:
headers = self.get_headers()
- return self.http_obj.request(url, method,
- headers=headers, body=body)
+ return self.http_obj.request(url, method, headers=headers,
+ body=body, chunked=chunked)
def request(self, method, url, extra_headers=False, headers=None,
- body=None):
+ body=None, chunked=False):
"""Send a HTTP request with keystone auth and using the catalog
This method will send an HTTP request using keystone auth in the
@@ -590,6 +596,7 @@
get_headers() method are used. If the request
explicitly requires no headers use an empty dict.
:param str body: Body to send with the request
+ :param bool chunked: sends the body with chunked encoding
:rtype: tuple
:return: a tuple with the first entry containing the response headers
and the second the response body
@@ -629,8 +636,8 @@
except (ValueError, TypeError):
headers = self.get_headers()
- resp, resp_body = self._request(method, url,
- headers=headers, body=body)
+ resp, resp_body = self._request(method, url, headers=headers,
+ body=body, chunked=chunked)
while (resp.status == 413 and
'retry-after' in resp and
diff --git a/tempest/lib/common/utils/data_utils.py b/tempest/lib/common/utils/data_utils.py
index 9605479..45e5067 100644
--- a/tempest/lib/common/utils/data_utils.py
+++ b/tempest/lib/common/utils/data_utils.py
@@ -19,6 +19,8 @@
import string
import uuid
+import six.moves
+
def rand_uuid():
"""Generate a random UUID string
@@ -196,3 +198,10 @@
except TypeError:
raise TypeError('Bad prefix type for generate IPv6 address by '
'EUI-64: %s' % cidr)
+
+
+# Courtesy of http://stackoverflow.com/a/312464
+def chunkify(sequence, chunksize):
+ """Yield successive chunks from `sequence`."""
+ for i in six.moves.xrange(0, len(sequence), chunksize):
+ yield sequence[i:i + chunksize]
diff --git a/tempest/lib/services/compute/base_compute_client.py b/tempest/lib/services/compute/base_compute_client.py
index 9161abb..a387b85 100644
--- a/tempest/lib/services/compute/base_compute_client.py
+++ b/tempest/lib/services/compute/base_compute_client.py
@@ -48,9 +48,9 @@
return headers
def request(self, method, url, extra_headers=False, headers=None,
- body=None):
+ body=None, chunked=False):
resp, resp_body = super(BaseComputeClient, self).request(
- method, url, extra_headers, headers, body)
+ method, url, extra_headers, headers, body, chunked)
if (COMPUTE_MICROVERSION and
COMPUTE_MICROVERSION != api_version_utils.LATEST_MICROVERSION):
api_version_utils.assert_version_header_matches_request(
diff --git a/tempest/lib/services/identity/v2/token_client.py b/tempest/lib/services/identity/v2/token_client.py
index 0350175..5716027 100644
--- a/tempest/lib/services/identity/v2/token_client.py
+++ b/tempest/lib/services/identity/v2/token_client.py
@@ -75,8 +75,12 @@
return rest_client.ResponseBody(resp, body['access'])
def request(self, method, url, extra_headers=False, headers=None,
- body=None):
- """A simple HTTP request interface."""
+ body=None, chunked=False):
+ """A simple HTTP request interface.
+
+ Note: this overloads the `request` method from the parent class and
+ thus must implement the same method signature.
+ """
if headers is None:
headers = self.get_headers(accept_type="json")
elif extra_headers:
diff --git a/tempest/lib/services/identity/v3/token_client.py b/tempest/lib/services/identity/v3/token_client.py
index f342a49..964d43f 100644
--- a/tempest/lib/services/identity/v3/token_client.py
+++ b/tempest/lib/services/identity/v3/token_client.py
@@ -122,8 +122,12 @@
return rest_client.ResponseBody(resp, body)
def request(self, method, url, extra_headers=False, headers=None,
- body=None):
- """A simple HTTP request interface."""
+ body=None, chunked=False):
+ """A simple HTTP request interface.
+
+ Note: this overloads the `request` method from the parent class and
+ thus must implement the same method signature.
+ """
if headers is None:
# Always accept 'json', for xml token client too.
# Because XML response is not easily
diff --git a/tempest/services/object_storage/object_client.py b/tempest/services/object_storage/object_client.py
index fa43d94..33dba6e 100644
--- a/tempest/services/object_storage/object_client.py
+++ b/tempest/services/object_storage/object_client.py
@@ -149,25 +149,30 @@
self.expected_success(201, resp.status)
return resp, body
- def put_object_with_chunk(self, container, name, contents, chunk_size):
- """Put an object with Transfer-Encoding header"""
+ def put_object_with_chunk(self, container, name, contents):
+ """Put an object with Transfer-Encoding header
+
+ :param container: name of the container
+ :type container: string
+ :param name: name of the object
+ :type name: string
+ :param contents: object data
+ :type contents: iterable
+ """
headers = {'Transfer-Encoding': 'chunked'}
if self.token:
headers['X-Auth-Token'] = self.token
- conn = put_object_connection(self.base_url, container, name, contents,
- chunk_size, headers)
-
- resp = conn.getresponse()
- body = resp.read()
-
- resp_headers = {}
- for header, value in resp.getheaders():
- resp_headers[header.lower()] = value
+ url = "%s/%s" % (container, name)
+ resp, body = self.put(
+ url, headers=headers,
+ body=contents,
+ chunked=True
+ )
self._error_checker('PUT', None, headers, contents, resp, body)
self.expected_success(201, resp.status)
- return resp.status, resp.reason, resp_headers
+ return resp.status, resp.reason, resp
def create_object_continue(self, container, object_name,
data, metadata=None):
@@ -262,30 +267,7 @@
headers = dict(headers)
else:
headers = {}
- if hasattr(contents, 'read'):
- conn.putrequest('PUT', path)
- for header, value in six.iteritems(headers):
- conn.putheader(header, value)
- if 'Content-Length' not in headers:
- if 'Transfer-Encoding' not in headers:
- conn.putheader('Transfer-Encoding', 'chunked')
- conn.endheaders()
- chunk = contents.read(chunk_size)
- while chunk:
- conn.send('%x\r\n%s\r\n' % (len(chunk), chunk))
- chunk = contents.read(chunk_size)
- conn.send('0\r\n\r\n')
- else:
- conn.endheaders()
- left = headers['Content-Length']
- while left > 0:
- size = chunk_size
- if size > left:
- size = left
- chunk = contents.read(size)
- conn.send(chunk)
- left -= len(chunk)
- else:
- conn.request('PUT', path, contents, headers)
+
+ conn.request('PUT', path, contents, headers)
return conn
diff --git a/tempest/tests/lib/common/utils/test_data_utils.py b/tempest/tests/lib/common/utils/test_data_utils.py
index f9e1f44..399c4af 100644
--- a/tempest/tests/lib/common/utils/test_data_utils.py
+++ b/tempest/tests/lib/common/utils/test_data_utils.py
@@ -169,3 +169,10 @@
bad_mac = 99999999999999999999
self.assertRaises(TypeError, data_utils.get_ipv6_addr_by_EUI64,
cidr, bad_mac)
+
+ def test_chunkify(self):
+ data = "aaa"
+ chunks = data_utils.chunkify(data, 2)
+ self.assertEqual("aa", next(chunks))
+ self.assertEqual("a", next(chunks))
+ self.assertRaises(StopIteration, next, chunks)
diff --git a/tempest/tests/lib/fake_http.py b/tempest/tests/lib/fake_http.py
index 397c856..cfa4b93 100644
--- a/tempest/tests/lib/fake_http.py
+++ b/tempest/tests/lib/fake_http.py
@@ -21,7 +21,7 @@
self.return_type = return_type
def request(self, uri, method="GET", body=None, headers=None,
- redirections=5, connection_type=None):
+ redirections=5, connection_type=None, chunked=False):
if not self.return_type:
fake_headers = fake_http_response(headers)
return_obj = {