THRIFT-1857 Python 3 Support
Client: Python
Patch: Thomas Bartelmess, Eevee (Alex Munroe), helgridly, Christian Verkerk, Jeroen Vlek, Nobuaki Sukegawa
This closes #213 and closes #680
diff --git a/lib/py/src/transport/THttpClient.py b/lib/py/src/transport/THttpClient.py
index 5851fa2..5abd41c 100644
--- a/lib/py/src/transport/THttpClient.py
+++ b/lib/py/src/transport/THttpClient.py
@@ -17,17 +17,17 @@
# under the License.
#
-import httplib
+from io import BytesIO
import os
import socket
import sys
-import urllib
-import urlparse
import warnings
-from cStringIO import StringIO
+from six.moves import urllib
+from six.moves import http_client
-from TTransport import *
+from .TTransport import *
+import six
class THttpClient(TTransportBase):
@@ -52,31 +52,33 @@
self.path = path
self.scheme = 'http'
else:
- parsed = urlparse.urlparse(uri_or_host)
+ parsed = urllib.parse.urlparse(uri_or_host)
self.scheme = parsed.scheme
assert self.scheme in ('http', 'https')
if self.scheme == 'http':
- self.port = parsed.port or httplib.HTTP_PORT
+ self.port = parsed.port or http_client.HTTP_PORT
elif self.scheme == 'https':
- self.port = parsed.port or httplib.HTTPS_PORT
+ self.port = parsed.port or http_client.HTTPS_PORT
self.host = parsed.hostname
self.path = parsed.path
if parsed.query:
self.path += '?%s' % parsed.query
- self.__wbuf = StringIO()
+ self.__wbuf = BytesIO()
self.__http = None
+ self.__http_response = None
self.__timeout = None
self.__custom_headers = None
def open(self):
if self.scheme == 'http':
- self.__http = httplib.HTTP(self.host, self.port)
+ self.__http = http_client.HTTPConnection(self.host, self.port)
else:
- self.__http = httplib.HTTPS(self.host, self.port)
+ self.__http = http_client.HTTPSConnection(self.host, self.port)
def close(self):
self.__http.close()
self.__http = None
+ self.__http_response = None
def isOpen(self):
return self.__http is not None
@@ -94,7 +96,7 @@
self.__custom_headers = headers
def read(self, sz):
- return self.__http.file.read(sz)
+ return self.__http_response.read(sz)
def write(self, buf):
self.__wbuf.write(buf)
@@ -117,13 +119,12 @@
# Pull data out of buffer
data = self.__wbuf.getvalue()
- self.__wbuf = StringIO()
+ self.__wbuf = BytesIO()
# HTTP request
self.__http.putrequest('POST', self.path)
# Write headers
- self.__http.putheader('Host', self.host)
self.__http.putheader('Content-Type', 'application/x-thrift')
self.__http.putheader('Content-Length', str(len(data)))
@@ -131,11 +132,11 @@
user_agent = 'Python/THttpClient'
script = os.path.basename(sys.argv[0])
if script:
- user_agent = '%s (%s)' % (user_agent, urllib.quote(script))
+ user_agent = '%s (%s)' % (user_agent, urllib.parse.quote(script))
self.__http.putheader('User-Agent', user_agent)
if self.__custom_headers:
- for key, val in self.__custom_headers.iteritems():
+ for key, val in six.iteritems(self.__custom_headers):
self.__http.putheader(key, val)
self.__http.endheaders()
@@ -144,7 +145,10 @@
self.__http.send(data)
# Get reply to flush the request
- self.code, self.message, self.headers = self.__http.getreply()
+ self.__http_response = self.__http.getresponse()
+ self.code = self.__http_response.status
+ self.message = self.__http_response.reason
+ self.headers = self.__http_response.msg
# Decorate if we know how to timeout
if hasattr(socket, 'getdefaulttimeout'):
diff --git a/lib/py/src/transport/TSocket.py b/lib/py/src/transport/TSocket.py
index 7b564aa..cb204a4 100644
--- a/lib/py/src/transport/TSocket.py
+++ b/lib/py/src/transport/TSocket.py
@@ -22,7 +22,7 @@
import socket
import sys
-from TTransport import *
+from .TTransport import *
class TSocketBase(TTransportBase):
diff --git a/lib/py/src/transport/TTransport.py b/lib/py/src/transport/TTransport.py
index 5914aca..3fe289a 100644
--- a/lib/py/src/transport/TTransport.py
+++ b/lib/py/src/transport/TTransport.py
@@ -17,9 +17,9 @@
# under the License.
#
-from cStringIO import StringIO
from struct import pack, unpack
from thrift.Thrift import TException
+from ..compat import BufferIO
class TTransportException(TException):
@@ -52,7 +52,7 @@
pass
def readAll(self, sz):
- buff = ''
+ buff = b''
have = 0
while (have < sz):
chunk = self.read(sz - have)
@@ -138,8 +138,8 @@
def __init__(self, trans, rbuf_size=DEFAULT_BUFFER):
self.__trans = trans
- self.__wbuf = StringIO()
- self.__rbuf = StringIO("")
+ self.__wbuf = BufferIO()
+ self.__rbuf = BufferIO()
self.__rbuf_size = rbuf_size
def isOpen(self):
@@ -155,8 +155,7 @@
ret = self.__rbuf.read(sz)
if len(ret) != 0:
return ret
-
- self.__rbuf = StringIO(self.__trans.read(max(sz, self.__rbuf_size)))
+ self.__rbuf = BufferIO(self.__trans.read(max(sz, self.__rbuf_size)))
return self.__rbuf.read(sz)
def write(self, buf):
@@ -164,13 +163,14 @@
self.__wbuf.write(buf)
except Exception as e:
# on exception reset wbuf so it doesn't contain a partial function call
- self.__wbuf = StringIO()
+ self.__wbuf = BufferIO()
raise e
+ self.__wbuf.getvalue()
def flush(self):
out = self.__wbuf.getvalue()
# reset wbuf before write/flush to preserve state on underlying failure
- self.__wbuf = StringIO()
+ self.__wbuf = BufferIO()
self.__trans.write(out)
self.__trans.flush()
@@ -189,12 +189,12 @@
if len(retstring) < reqlen:
retstring += self.__trans.readAll(reqlen - len(retstring))
- self.__rbuf = StringIO(retstring)
+ self.__rbuf = BufferIO(retstring)
return self.__rbuf
class TMemoryBuffer(TTransportBase, CReadableTransport):
- """Wraps a cStringIO object as a TTransport.
+ """Wraps a cBytesIO object as a TTransport.
NOTE: Unlike the C++ version of this class, you cannot write to it
then immediately read from it. If you want to read from a
@@ -208,9 +208,9 @@
If value is set, this will be a transport for reading,
otherwise, it is for writing"""
if value is not None:
- self._buffer = StringIO(value)
+ self._buffer = BufferIO(value)
else:
- self._buffer = StringIO()
+ self._buffer = BufferIO()
def isOpen(self):
return not self._buffer.closed
@@ -256,8 +256,8 @@
def __init__(self, trans,):
self.__trans = trans
- self.__rbuf = StringIO()
- self.__wbuf = StringIO()
+ self.__rbuf = BufferIO()
+ self.__wbuf = BufferIO()
def isOpen(self):
return self.__trans.isOpen()
@@ -279,7 +279,7 @@
def readFrame(self):
buff = self.__trans.readAll(4)
sz, = unpack('!i', buff)
- self.__rbuf = StringIO(self.__trans.readAll(sz))
+ self.__rbuf = BufferIO(self.__trans.readAll(sz))
def write(self, buf):
self.__wbuf.write(buf)
@@ -288,7 +288,7 @@
wout = self.__wbuf.getvalue()
wsz = len(wout)
# reset wbuf before write/flush to preserve state on underlying failure
- self.__wbuf = StringIO()
+ self.__wbuf = BufferIO()
# N.B.: Doing this string concatenation is WAY cheaper than making
# two separate calls to the underlying socket object. Socket writes in
# Python turn out to be REALLY expensive, but it seems to do a pretty
@@ -309,7 +309,7 @@
while len(prefix) < reqlen:
self.readFrame()
prefix += self.__rbuf.getvalue()
- self.__rbuf = StringIO(prefix)
+ self.__rbuf = BufferIO(prefix)
return self.__rbuf
@@ -337,7 +337,7 @@
class TSaslClientTransport(TTransportBase, CReadableTransport):
"""
- SASL transport
+ SASL transport
"""
START = 1
@@ -363,8 +363,8 @@
self.transport = transport
self.sasl = SASLClient(host, service, mechanism, **sasl_kwargs)
- self.__wbuf = StringIO()
- self.__rbuf = StringIO()
+ self.__wbuf = BufferIO()
+ self.__rbuf = BufferIO()
def open(self):
if not self.transport.isOpen():
@@ -409,7 +409,7 @@
encoded = self.sasl.wrap(data)
self.transport.write(''.join((pack("!i", len(encoded)), encoded)))
self.transport.flush()
- self.__wbuf = StringIO()
+ self.__wbuf = BufferIO()
def read(self, sz):
ret = self.__rbuf.read(sz)
@@ -423,7 +423,7 @@
header = self.transport.readAll(4)
length, = unpack('!i', header)
encoded = self.transport.readAll(length)
- self.__rbuf = StringIO(self.sasl.unwrap(encoded))
+ self.__rbuf = BufferIO(self.sasl.unwrap(encoded))
def close(self):
self.sasl.dispose()
@@ -441,6 +441,6 @@
while len(prefix) < reqlen:
self._read_frame()
prefix += self.__rbuf.getvalue()
- self.__rbuf = StringIO(prefix)
+ self.__rbuf = BufferIO(prefix)
return self.__rbuf
diff --git a/lib/py/src/transport/TTwisted.py b/lib/py/src/transport/TTwisted.py
index 29bbd4c..cabe345 100644
--- a/lib/py/src/transport/TTwisted.py
+++ b/lib/py/src/transport/TTwisted.py
@@ -17,8 +17,8 @@
# under the License.
#
+from io import BytesIO
import struct
-from cStringIO import StringIO
from zope.interface import implements, Interface, Attribute
from twisted.internet.protocol import ServerFactory, ClientFactory, \
@@ -29,19 +29,20 @@
from twisted.web import server, resource, http
from thrift.transport import TTransport
+import six
class TMessageSenderTransport(TTransport.TTransportBase):
def __init__(self):
- self.__wbuf = StringIO()
+ self.__wbuf = BytesIO()
def write(self, buf):
self.__wbuf.write(buf)
def flush(self):
msg = self.__wbuf.getvalue()
- self.__wbuf = StringIO()
+ self.__wbuf = BytesIO()
return self.sendMessage(msg)
def sendMessage(self, message):
@@ -82,7 +83,7 @@
self.started.callback(self.client)
def connectionLost(self, reason=connectionDone):
- for k, v in self.client._reqs.iteritems():
+ for k, v in six.iteritems(self.client._reqs):
tex = TTransport.TTransportException(
type=TTransport.TTransportException.END_OF_FILE,
message='Connection closed')
diff --git a/lib/py/src/transport/TZlibTransport.py b/lib/py/src/transport/TZlibTransport.py
index a2f42a5..7fe5853 100644
--- a/lib/py/src/transport/TZlibTransport.py
+++ b/lib/py/src/transport/TZlibTransport.py
@@ -24,8 +24,8 @@
from __future__ import division
import zlib
-from cStringIO import StringIO
-from TTransport import TTransportBase, CReadableTransport
+from .TTransport import TTransportBase, CReadableTransport
+from ..compat import BufferIO
class TZlibTransportFactory(object):
@@ -88,8 +88,8 @@
"""
self.__trans = trans
self.compresslevel = compresslevel
- self.__rbuf = StringIO()
- self.__wbuf = StringIO()
+ self.__rbuf = BufferIO()
+ self.__wbuf = BufferIO()
self._init_zlib()
self._init_stats()
@@ -97,8 +97,8 @@
"""Internal method to initialize/reset the internal StringIO objects
for read and write buffers.
"""
- self.__rbuf = StringIO()
- self.__wbuf = StringIO()
+ self.__rbuf = BufferIO()
+ self.__wbuf = BufferIO()
def _init_stats(self):
"""Internal method to reset the internal statistics counters
@@ -203,7 +203,7 @@
self.bytes_in += len(zbuf)
self.bytes_in_comp += len(buf)
old = self.__rbuf.read()
- self.__rbuf = StringIO(old + buf)
+ self.__rbuf = BufferIO(old + buf)
if len(old) + len(buf) == 0:
return False
return True
@@ -228,7 +228,7 @@
ztail = self._zcomp_write.flush(zlib.Z_SYNC_FLUSH)
self.bytes_out_comp += len(ztail)
if (len(zbuf) + len(ztail)) > 0:
- self.__wbuf = StringIO()
+ self.__wbuf = BufferIO()
self.__trans.write(zbuf + ztail)
self.__trans.flush()
@@ -244,5 +244,5 @@
retstring += self.read(self.DEFAULT_BUFFSIZE)
while len(retstring) < reqlen:
retstring += self.read(reqlen - len(retstring))
- self.__rbuf = StringIO(retstring)
+ self.__rbuf = BufferIO(retstring)
return self.__rbuf