Swift object client: use urllib3 builtin support for chunked transfer
Urllib3 has native support for chunked encoding, so let's use this
instead of rolling our own. Less code to maintain, additional logging
and timing (thanks to our common RestClient). Yeah \O/.
Change-Id: I4a253a5cec0fc35009af25872239363625d417e3
diff --git a/releasenotes/notes/support-chunked-encoding-d71f53225f68edf3.yaml b/releasenotes/notes/support-chunked-encoding-d71f53225f68edf3.yaml
new file mode 100644
index 0000000..eb45523
--- /dev/null
+++ b/releasenotes/notes/support-chunked-encoding-d71f53225f68edf3.yaml
@@ -0,0 +1,9 @@
+---
+features:
+ - The RestClient (in tempest.lib.common.rest_client) now supports POSTing
+ and PUTing data with chunked transfer encoding. Just pass an `iterable`
+ object as the `body` argument and set the `chunked` argument to `True`.
+ - A new generator called `chunkify` is added in
+ tempest.lib.common.utils.data_utils that yields fixed-size chunks (slices)
+ from a Python sequence.
+
diff --git a/tempest/api/object_storage/test_object_services.py b/tempest/api/object_storage/test_object_services.py
index dc926e0..a88e4f4 100644
--- a/tempest/api/object_storage/test_object_services.py
+++ b/tempest/api/object_storage/test_object_services.py
@@ -20,7 +20,6 @@
import zlib
import six
-from six import moves
from tempest.api.object_storage import base
from tempest.common import custom_matchers
@@ -201,8 +200,8 @@
status, _, resp_headers = self.object_client.put_object_with_chunk(
container=self.container_name,
name=object_name,
- contents=moves.cStringIO(data),
- chunk_size=512)
+ contents=data_utils.chunkify(data, 512)
+ )
self.assertHeaders(resp_headers, 'Object', 'PUT')
# check uploaded content
diff --git a/tempest/lib/common/rest_client.py b/tempest/lib/common/rest_client.py
index 30750de..179db17 100644
--- a/tempest/lib/common/rest_client.py
+++ b/tempest/lib/common/rest_client.py
@@ -243,7 +243,8 @@
details = pattern.format(read_code, expected_code)
raise exceptions.InvalidHttpSuccessCode(details)
- def post(self, url, body, headers=None, extra_headers=False):
+ def post(self, url, body, headers=None, extra_headers=False,
+ chunked=False):
"""Send a HTTP POST request using keystone auth
:param str url: the relative url to send the post request to
@@ -253,11 +254,12 @@
returned by the get_headers() method are to
be used but additional headers are needed in
the request pass them in as a dict.
+ :param bool chunked: sends the body with chunked encoding
:return: a tuple with the first entry containing the response headers
and the second the response body
:rtype: tuple
"""
- return self.request('POST', url, extra_headers, headers, body)
+ return self.request('POST', url, extra_headers, headers, body, chunked)
def get(self, url, headers=None, extra_headers=False):
"""Send a HTTP GET request using keystone service catalog and auth
@@ -306,7 +308,7 @@
"""
return self.request('PATCH', url, extra_headers, headers, body)
- def put(self, url, body, headers=None, extra_headers=False):
+ def put(self, url, body, headers=None, extra_headers=False, chunked=False):
"""Send a HTTP PUT request using keystone service catalog and auth
:param str url: the relative url to send the post request to
@@ -316,11 +318,12 @@
returned by the get_headers() method are to
be used but additional headers are needed in
the request pass them in as a dict.
+ :param bool chunked: sends the body with chunked encoding
:return: a tuple with the first entry containing the response headers
and the second the response body
:rtype: tuple
"""
- return self.request('PUT', url, extra_headers, headers, body)
+ return self.request('PUT', url, extra_headers, headers, body, chunked)
def head(self, url, headers=None, extra_headers=False):
"""Send a HTTP HEAD request using keystone service catalog and auth
@@ -520,7 +523,7 @@
if method != 'HEAD' and not resp_body and resp.status >= 400:
self.LOG.warning("status >= 400 response with empty body")
- def _request(self, method, url, headers=None, body=None):
+ def _request(self, method, url, headers=None, body=None, chunked=False):
"""A simple HTTP request interface."""
# Authenticate the request with the auth provider
req_url, req_headers, req_body = self.auth_provider.auth_request(
@@ -530,7 +533,9 @@
start = time.time()
self._log_request_start(method, req_url)
resp, resp_body = self.raw_request(
- req_url, method, headers=req_headers, body=req_body)
+ req_url, method, headers=req_headers, body=req_body,
+ chunked=chunked
+ )
end = time.time()
self._log_request(method, req_url, resp, secs=(end - start),
req_headers=req_headers, req_body=req_body,
@@ -541,7 +546,7 @@
return resp, resp_body
- def raw_request(self, url, method, headers=None, body=None):
+ def raw_request(self, url, method, headers=None, body=None, chunked=False):
"""Send a raw HTTP request without the keystone catalog or auth
This method sends a HTTP request in the same manner as the request()
@@ -554,17 +559,18 @@
:param str headers: Headers to use for the request if none are specifed
the headers
:param str body: Body to send with the request
+ :param bool chunked: sends the body with chunked encoding
:rtype: tuple
:return: a tuple with the first entry containing the response headers
and the second the response body
"""
if headers is None:
headers = self.get_headers()
- return self.http_obj.request(url, method,
- headers=headers, body=body)
+ return self.http_obj.request(url, method, headers=headers,
+ body=body, chunked=chunked)
def request(self, method, url, extra_headers=False, headers=None,
- body=None):
+ body=None, chunked=False):
"""Send a HTTP request with keystone auth and using the catalog
This method will send an HTTP request using keystone auth in the
@@ -590,6 +596,7 @@
get_headers() method are used. If the request
explicitly requires no headers use an empty dict.
:param str body: Body to send with the request
+ :param bool chunked: sends the body with chunked encoding
:rtype: tuple
:return: a tuple with the first entry containing the response headers
and the second the response body
@@ -629,8 +636,8 @@
except (ValueError, TypeError):
headers = self.get_headers()
- resp, resp_body = self._request(method, url,
- headers=headers, body=body)
+ resp, resp_body = self._request(method, url, headers=headers,
+ body=body, chunked=chunked)
while (resp.status == 413 and
'retry-after' in resp and
diff --git a/tempest/lib/common/utils/data_utils.py b/tempest/lib/common/utils/data_utils.py
index 9605479..45e5067 100644
--- a/tempest/lib/common/utils/data_utils.py
+++ b/tempest/lib/common/utils/data_utils.py
@@ -19,6 +19,8 @@
import string
import uuid
+import six.moves
+
def rand_uuid():
"""Generate a random UUID string
@@ -196,3 +198,10 @@
except TypeError:
raise TypeError('Bad prefix type for generate IPv6 address by '
'EUI-64: %s' % cidr)
+
+
+# Courtesy of http://stackoverflow.com/a/312464
+def chunkify(sequence, chunksize):
+ """Yield successive chunks from `sequence`."""
+ for i in six.moves.xrange(0, len(sequence), chunksize):
+ yield sequence[i:i + chunksize]
diff --git a/tempest/lib/services/compute/base_compute_client.py b/tempest/lib/services/compute/base_compute_client.py
index 9161abb..a387b85 100644
--- a/tempest/lib/services/compute/base_compute_client.py
+++ b/tempest/lib/services/compute/base_compute_client.py
@@ -48,9 +48,9 @@
return headers
def request(self, method, url, extra_headers=False, headers=None,
- body=None):
+ body=None, chunked=False):
resp, resp_body = super(BaseComputeClient, self).request(
- method, url, extra_headers, headers, body)
+ method, url, extra_headers, headers, body, chunked)
if (COMPUTE_MICROVERSION and
COMPUTE_MICROVERSION != api_version_utils.LATEST_MICROVERSION):
api_version_utils.assert_version_header_matches_request(
diff --git a/tempest/lib/services/identity/v2/token_client.py b/tempest/lib/services/identity/v2/token_client.py
index 0350175..5716027 100644
--- a/tempest/lib/services/identity/v2/token_client.py
+++ b/tempest/lib/services/identity/v2/token_client.py
@@ -75,8 +75,12 @@
return rest_client.ResponseBody(resp, body['access'])
def request(self, method, url, extra_headers=False, headers=None,
- body=None):
- """A simple HTTP request interface."""
+ body=None, chunked=False):
+ """A simple HTTP request interface.
+
+ Note: this overloads the `request` method from the parent class and
+ thus must implement the same method signature.
+ """
if headers is None:
headers = self.get_headers(accept_type="json")
elif extra_headers:
diff --git a/tempest/lib/services/identity/v3/token_client.py b/tempest/lib/services/identity/v3/token_client.py
index f342a49..964d43f 100644
--- a/tempest/lib/services/identity/v3/token_client.py
+++ b/tempest/lib/services/identity/v3/token_client.py
@@ -122,8 +122,12 @@
return rest_client.ResponseBody(resp, body)
def request(self, method, url, extra_headers=False, headers=None,
- body=None):
- """A simple HTTP request interface."""
+ body=None, chunked=False):
+ """A simple HTTP request interface.
+
+ Note: this overloads the `request` method from the parent class and
+ thus must implement the same method signature.
+ """
if headers is None:
# Always accept 'json', for xml token client too.
# Because XML response is not easily
diff --git a/tempest/services/object_storage/object_client.py b/tempest/services/object_storage/object_client.py
index fa43d94..33dba6e 100644
--- a/tempest/services/object_storage/object_client.py
+++ b/tempest/services/object_storage/object_client.py
@@ -149,25 +149,30 @@
self.expected_success(201, resp.status)
return resp, body
- def put_object_with_chunk(self, container, name, contents, chunk_size):
- """Put an object with Transfer-Encoding header"""
+ def put_object_with_chunk(self, container, name, contents):
+ """Put an object with Transfer-Encoding header
+
+ :param container: name of the container
+ :type container: string
+ :param name: name of the object
+ :type name: string
+ :param contents: object data
+ :type contents: iterable
+ """
headers = {'Transfer-Encoding': 'chunked'}
if self.token:
headers['X-Auth-Token'] = self.token
- conn = put_object_connection(self.base_url, container, name, contents,
- chunk_size, headers)
-
- resp = conn.getresponse()
- body = resp.read()
-
- resp_headers = {}
- for header, value in resp.getheaders():
- resp_headers[header.lower()] = value
+ url = "%s/%s" % (container, name)
+ resp, body = self.put(
+ url, headers=headers,
+ body=contents,
+ chunked=True
+ )
self._error_checker('PUT', None, headers, contents, resp, body)
self.expected_success(201, resp.status)
- return resp.status, resp.reason, resp_headers
+ return resp.status, resp.reason, resp
def create_object_continue(self, container, object_name,
data, metadata=None):
@@ -262,30 +267,7 @@
headers = dict(headers)
else:
headers = {}
- if hasattr(contents, 'read'):
- conn.putrequest('PUT', path)
- for header, value in six.iteritems(headers):
- conn.putheader(header, value)
- if 'Content-Length' not in headers:
- if 'Transfer-Encoding' not in headers:
- conn.putheader('Transfer-Encoding', 'chunked')
- conn.endheaders()
- chunk = contents.read(chunk_size)
- while chunk:
- conn.send('%x\r\n%s\r\n' % (len(chunk), chunk))
- chunk = contents.read(chunk_size)
- conn.send('0\r\n\r\n')
- else:
- conn.endheaders()
- left = headers['Content-Length']
- while left > 0:
- size = chunk_size
- if size > left:
- size = left
- chunk = contents.read(size)
- conn.send(chunk)
- left -= len(chunk)
- else:
- conn.request('PUT', path, contents, headers)
+
+ conn.request('PUT', path, contents, headers)
return conn
diff --git a/tempest/tests/lib/common/utils/test_data_utils.py b/tempest/tests/lib/common/utils/test_data_utils.py
index f9e1f44..399c4af 100644
--- a/tempest/tests/lib/common/utils/test_data_utils.py
+++ b/tempest/tests/lib/common/utils/test_data_utils.py
@@ -169,3 +169,10 @@
bad_mac = 99999999999999999999
self.assertRaises(TypeError, data_utils.get_ipv6_addr_by_EUI64,
cidr, bad_mac)
+
+ def test_chunkify(self):
+ data = "aaa"
+ chunks = data_utils.chunkify(data, 2)
+ self.assertEqual("aa", next(chunks))
+ self.assertEqual("a", next(chunks))
+ self.assertRaises(StopIteration, next, chunks)
diff --git a/tempest/tests/lib/fake_http.py b/tempest/tests/lib/fake_http.py
index 397c856..cfa4b93 100644
--- a/tempest/tests/lib/fake_http.py
+++ b/tempest/tests/lib/fake_http.py
@@ -21,7 +21,7 @@
self.return_type = return_type
def request(self, uri, method="GET", body=None, headers=None,
- redirections=5, connection_type=None):
+ redirections=5, connection_type=None, chunked=False):
if not self.return_type:
fake_headers = fake_http_response(headers)
return_obj = {