THRIFT-1857 Python 3 Support
Client: Python
Patch: Thomas Bartelmess, Eevee (Alex Munroe), helgridly, Christian Verkerk, Jeroen Vlek, Nobuaki Sukegawa
This closes #213 and closes #680
diff --git a/lib/py/setup.py b/lib/py/setup.py
index 7e0a964..090544c 100644
--- a/lib/py/setup.py
+++ b/lib/py/setup.py
@@ -19,6 +19,7 @@
# under the License.
#
+import platform
import sys
try:
from setuptools import setup, Extension
@@ -69,7 +70,7 @@
)
else:
extensions = dict()
-
+
setup(name = 'thrift',
version = '1.0.0-dev',
description = 'Python bindings for the Apache Thrift RPC system',
@@ -77,6 +78,7 @@
author_email = 'dev@thrift.apache.org',
url = 'http://thrift.apache.org',
license = 'Apache License 2.0',
+ install_requires=['six>=1.7.2'],
packages = [
'thrift',
'thrift.protocol',
@@ -90,15 +92,20 @@
'Intended Audience :: Developers',
'Programming Language :: Python',
'Programming Language :: Python :: 2',
+ 'Programming Language :: Python :: 3',
'Topic :: Software Development :: Libraries',
'Topic :: System :: Networking'
],
- use_2to3 = True,
**extensions
)
try:
- run_setup(True)
+ with_binary = False
+ # Don't even try to build the C module unless we're on CPython 2.x.
+ # TODO: fix it for CPython 3.x
+ if platform.python_implementation() == 'CPython' and sys.version_info < (3,):
+ with_binary = True
+ run_setup(with_binary)
except BuildFailed:
print()
print('*' * 80)
diff --git a/lib/py/src/TSCons.py b/lib/py/src/TSCons.py
index da8d283..ed2601a 100644
--- a/lib/py/src/TSCons.py
+++ b/lib/py/src/TSCons.py
@@ -19,6 +19,8 @@
from os import path
from SCons.Builder import Builder
+from six.moves import map
+from six.moves import zip
def scons_env(env, add=''):
diff --git a/lib/py/src/TSerialization.py b/lib/py/src/TSerialization.py
index 8a58d89..fbbe768 100644
--- a/lib/py/src/TSerialization.py
+++ b/lib/py/src/TSerialization.py
@@ -17,8 +17,8 @@
# under the License.
#
-from protocol import TBinaryProtocol
-from transport import TTransport
+from .protocol import TBinaryProtocol
+from .transport import TTransport
def serialize(thrift_object,
diff --git a/lib/py/src/TTornado.py b/lib/py/src/TTornado.py
index 65795ab..e3b4df7 100644
--- a/lib/py/src/TTornado.py
+++ b/lib/py/src/TTornado.py
@@ -20,11 +20,10 @@
from __future__ import absolute_import
import socket
import struct
-
import logging
logger = logging.getLogger(__name__)
-from thrift.transport.TTransport import TTransportException, TTransportBase, TMemoryBuffer
+from .transport.TTransport import TTransportException, TTransportBase, TMemoryBuffer
from io import BytesIO
from collections import deque
diff --git a/lib/py/src/compat.py b/lib/py/src/compat.py
new file mode 100644
index 0000000..b2f47dc
--- /dev/null
+++ b/lib/py/src/compat.py
@@ -0,0 +1,27 @@
+import sys
+
+if sys.version_info[0] == 2:
+
+ from cStringIO import StringIO as BufferIO
+
+ def binary_to_str(bin_val):
+ return bin_val
+
+ def str_to_binary(str_val):
+ return str_val
+
+else:
+
+ from io import BytesIO as BufferIO
+
+ def binary_to_str(bin_val):
+ try:
+ return bin_val.decode('utf8')
+ except:
+ return bin_val
+
+ def str_to_binary(str_val):
+ try:
+ return bytearray(str_val, 'utf8')
+ except:
+ return str_val
diff --git a/lib/py/src/protocol/TBase.py b/lib/py/src/protocol/TBase.py
index 6cbd5f3..118a679 100644
--- a/lib/py/src/protocol/TBase.py
+++ b/lib/py/src/protocol/TBase.py
@@ -17,7 +17,6 @@
# under the License.
#
-from thrift.Thrift import *
from thrift.protocol import TBinaryProtocol
from thrift.transport import TTransport
@@ -31,8 +30,7 @@
__slots__ = []
def __repr__(self):
- L = ['%s=%r' % (key, getattr(self, key))
- for key in self.__slots__]
+ L = ['%s=%r' % (key, getattr(self, key)) for key in self.__slots__]
return '%s(%s)' % (self.__class__.__name__, ', '.join(L))
def __eq__(self, other):
@@ -50,9 +48,9 @@
def read(self, iprot):
if (iprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and
- isinstance(iprot.trans, TTransport.CReadableTransport) and
- self.thrift_spec is not None and
- fastbinary is not None):
+ isinstance(iprot.trans, TTransport.CReadableTransport) and
+ self.thrift_spec is not None and
+ fastbinary is not None):
fastbinary.decode_binary(self,
iprot.trans,
(self.__class__, self.thrift_spec))
@@ -61,21 +59,13 @@
def write(self, oprot):
if (oprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and
- self.thrift_spec is not None and
- fastbinary is not None):
+ self.thrift_spec is not None and
+ fastbinary is not None):
oprot.trans.write(
fastbinary.encode_binary(self, (self.__class__, self.thrift_spec)))
return
oprot.writeStruct(self, self.thrift_spec)
-class TExceptionBase(Exception):
- # old style class so python2.4 can raise exceptions derived from this
- # This can't inherit from TBase because of that limitation.
+class TExceptionBase(TBase, Exception):
__slots__ = []
-
- __repr__ = TBase.__repr__.im_func
- __eq__ = TBase.__eq__.im_func
- __ne__ = TBase.__ne__.im_func
- read = TBase.read.im_func
- write = TBase.write.im_func
diff --git a/lib/py/src/protocol/TBinaryProtocol.py b/lib/py/src/protocol/TBinaryProtocol.py
index 6fdd08c..f92f558 100644
--- a/lib/py/src/protocol/TBinaryProtocol.py
+++ b/lib/py/src/protocol/TBinaryProtocol.py
@@ -17,7 +17,7 @@
# under the License.
#
-from TProtocol import *
+from .TProtocol import TType, TProtocolBase, TProtocolException
from struct import pack, unpack
@@ -118,7 +118,7 @@
buff = pack("!d", dub)
self.trans.write(buff)
- def writeString(self, str):
+ def writeBinary(self, str):
self.writeI32(len(str))
self.trans.write(str)
@@ -217,10 +217,10 @@
val, = unpack('!d', buff)
return val
- def readString(self):
+ def readBinary(self):
len = self.readI32()
- str = self.trans.readAll(len)
- return str
+ s = self.trans.readAll(len)
+ return s
class TBinaryProtocolFactory:
diff --git a/lib/py/src/protocol/TCompactProtocol.py b/lib/py/src/protocol/TCompactProtocol.py
index 7054ab0..b8d171e 100644
--- a/lib/py/src/protocol/TCompactProtocol.py
+++ b/lib/py/src/protocol/TCompactProtocol.py
@@ -17,9 +17,11 @@
# under the License.
#
-from TProtocol import *
+from .TProtocol import TType, TProtocolBase, TProtocolException, checkIntegerLimits
from struct import pack, unpack
+from ..compat import binary_to_str, str_to_binary
+
__all__ = ['TCompactProtocol', 'TCompactProtocolFactory']
CLEAR = 0
@@ -62,7 +64,7 @@
else:
out.append((n & 0xff) | 0x80)
n = n >> 7
- trans.write(''.join(map(chr, out)))
+ trans.write(bytearray(out))
def readVarint(trans):
@@ -141,7 +143,7 @@
self.__writeUByte(self.PROTOCOL_ID)
self.__writeUByte(self.VERSION | (type << self.TYPE_SHIFT_AMOUNT))
self.__writeVarint(seqid)
- self.__writeString(name)
+ self.__writeBinary(str_to_binary(name))
self.state = VALUE_WRITE
def writeMessageEnd(self):
@@ -254,10 +256,10 @@
def writeDouble(self, dub):
self.trans.write(pack('<d', dub))
- def __writeString(self, s):
+ def __writeBinary(self, s):
self.__writeSize(len(s))
self.trans.write(s)
- writeString = writer(__writeString)
+ writeBinary = writer(__writeBinary)
def readFieldBegin(self):
assert self.state == FIELD_READ, self.state
@@ -302,7 +304,7 @@
def __readSize(self):
result = self.__readVarint()
if result < 0:
- raise TException("Length < 0")
+ raise TProtocolException("Length < 0")
return result
def readMessageBegin(self):
@@ -310,15 +312,15 @@
proto_id = self.__readUByte()
if proto_id != self.PROTOCOL_ID:
raise TProtocolException(TProtocolException.BAD_VERSION,
- 'Bad protocol id in the message: %d' % proto_id)
+ 'Bad protocol id in the message: %d' % proto_id)
ver_type = self.__readUByte()
type = (ver_type >> self.TYPE_SHIFT_AMOUNT) & self.TYPE_BITS
version = ver_type & self.VERSION_MASK
if version != self.VERSION:
raise TProtocolException(TProtocolException.BAD_VERSION,
- 'Bad version: %d (expect %d)' % (version, self.VERSION))
+ 'Bad version: %d (expect %d)' % (version, self.VERSION))
seqid = self.__readVarint()
- name = self.__readString()
+ name = binary_to_str(self.__readBinary())
return (name, type, seqid)
def readMessageEnd(self):
@@ -388,10 +390,10 @@
val, = unpack('<d', buff)
return val
- def __readString(self):
+ def __readBinary(self):
len = self.__readSize()
return self.trans.readAll(len)
- readString = reader(__readString)
+ readBinary = reader(__readBinary)
def __getTType(self, byte):
return TTYPES[byte & 0x0f]
diff --git a/lib/py/src/protocol/TJSONProtocol.py b/lib/py/src/protocol/TJSONProtocol.py
index 7807a6c..3ed8bcb 100644
--- a/lib/py/src/protocol/TJSONProtocol.py
+++ b/lib/py/src/protocol/TJSONProtocol.py
@@ -17,11 +17,13 @@
# under the License.
#
-from TProtocol import TType, TProtocolBase, TProtocolException, \
- checkIntegerLimits
+from .TProtocol import TType, TProtocolBase, TProtocolException, checkIntegerLimits
import base64
-import json
import math
+import sys
+
+from ..compat import str_to_binary
+
__all__ = ['TJSONProtocol',
'TJSONProtocolFactory',
@@ -30,20 +32,39 @@
VERSION = 1
-COMMA = ','
-COLON = ':'
-LBRACE = '{'
-RBRACE = '}'
-LBRACKET = '['
-RBRACKET = ']'
-QUOTE = '"'
-BACKSLASH = '\\'
-ZERO = '0'
+COMMA = b','
+COLON = b':'
+LBRACE = b'{'
+RBRACE = b'}'
+LBRACKET = b'['
+RBRACKET = b']'
+QUOTE = b'"'
+BACKSLASH = b'\\'
+ZERO = b'0'
-ESCSEQ = '\\u00'
-ESCAPE_CHAR = '"\\bfnrt/'
-ESCAPE_CHAR_VALS = ['"', '\\', '\b', '\f', '\n', '\r', '\t', '/']
-NUMERIC_CHAR = '+-.0123456789Ee'
+ESCSEQ0 = ord('\\')
+ESCSEQ1 = ord('u')
+ESCAPE_CHAR_VALS = {
+ '"': '\\"',
+ '\\': '\\\\',
+ '\b': '\\b',
+ '\f': '\\f',
+ '\n': '\\n',
+ '\r': '\\r',
+ '\t': '\\t',
+ # '/': '\\/',
+}
+ESCAPE_CHARS = {
+ b'"': '"',
+ b'\\': '\\',
+ b'b': '\b',
+ b'f': '\f',
+ b'n': '\n',
+ b'r': '\r',
+ b't': '\t',
+ b'/': '/',
+}
+NUMERIC_CHAR = b'+-.0123456789Ee'
CTYPES = {TType.BOOL: 'tf',
TType.BYTE: 'i8',
@@ -70,7 +91,7 @@
def doIO(self, function):
pass
-
+
def write(self):
pass
@@ -85,7 +106,7 @@
class JSONListContext(JSONBaseContext):
-
+
def doIO(self, function):
if self.first is True:
self.first = False
@@ -100,7 +121,7 @@
class JSONPairContext(JSONBaseContext):
-
+
def __init__(self, protocol):
super(JSONPairContext, self).__init__(protocol)
self.colon = True
@@ -146,6 +167,7 @@
self.hasData = True
return self.data
+
class TJSONProtocolBase(TProtocolBase):
def __init__(self, trans):
@@ -174,14 +196,22 @@
def writeJSONString(self, string):
self.context.write()
- self.trans.write(json.dumps(string, ensure_ascii=False))
+ json_str = ['"']
+ for s in string:
+ escaped = ESCAPE_CHAR_VALS.get(s, s)
+ json_str.append(escaped)
+ json_str.append('"')
+ self.trans.write(str_to_binary(''.join(json_str)))
def writeJSONNumber(self, number, formatter='{}'):
self.context.write()
- jsNumber = formatter.format(number)
+ jsNumber = str(formatter.format(number)).encode('ascii')
if self.context.escapeNum():
- jsNumber = "%s%s%s" % (QUOTE, jsNumber, QUOTE)
- self.trans.write(jsNumber)
+ self.trans.write(QUOTE)
+ self.trans.write(jsNumber)
+ self.trans.write(QUOTE)
+ else:
+ self.trans.write(jsNumber)
def writeJSONBase64(self, binary):
self.context.write()
@@ -222,18 +252,23 @@
character = self.reader.read()
if character == QUOTE:
break
- if character == ESCSEQ[0]:
+ if ord(character) == ESCSEQ0:
character = self.reader.read()
- if character == ESCSEQ[1]:
- self.readJSONSyntaxChar(ZERO)
- self.readJSONSyntaxChar(ZERO)
- character = json.JSONDecoder().decode('"\u00%s"' % self.trans.read(2))
+ if ord(character) == ESCSEQ1:
+ character = chr(int(self.trans.read(4)))
else:
- off = ESCAPE_CHAR.find(character)
- if off == -1:
+ if character not in ESCAPE_CHARS:
raise TProtocolException(TProtocolException.INVALID_DATA,
"Expected control char")
- character = ESCAPE_CHAR_VALS[off]
+ character = ESCAPE_CHARS[character]
+ elif character in ESCAPE_CHAR_VALS:
+ raise TProtocolException(TProtocolException.INVALID_DATA,
+ "Unescaped control char")
+ elif sys.version_info[0] > 2:
+ utf8_bytes = bytearray([ord(character)])
+ while ord(self.reader.peek()) >= 0x80:
+ utf8_bytes.append(ord(self.reader.read()))
+ character = utf8_bytes.decode('utf8')
string.append(character)
return ''.join(string)
@@ -251,7 +286,7 @@
if self.isJSONNumeric(character) is False:
break
numeric.append(self.reader.read())
- return ''.join(numeric)
+ return b''.join(numeric).decode('ascii')
def readJSONInteger(self):
self.context.read()
@@ -267,12 +302,12 @@
def readJSONDouble(self):
self.context.read()
if self.reader.peek() == QUOTE:
- string = self.readJSONString(True)
+ string = self.readJSONString(True)
try:
double = float(string)
if (self.context.escapeNum is False and
- not math.isinf(double) and
- not math.isnan(double)):
+ not math.isinf(double) and
+ not math.isnan(double)):
raise TProtocolException(TProtocolException.INVALID_DATA,
"Numeric data unexpectedly quoted")
return double
@@ -430,12 +465,12 @@
def writeMapEnd(self):
self.writeJSONObjectEnd()
self.writeJSONArrayEnd()
-
+
def writeListBegin(self, etype, size):
self.writeJSONArrayStart()
self.writeJSONString(CTYPES[etype])
self.writeJSONNumber(size)
-
+
def writeListEnd(self):
self.writeJSONArrayEnd()
@@ -443,7 +478,7 @@
self.writeJSONArrayStart()
self.writeJSONString(CTYPES[etype])
self.writeJSONNumber(size)
-
+
def writeSetEnd(self):
self.writeJSONArrayEnd()
@@ -472,7 +507,7 @@
def writeString(self, string):
self.writeJSONString(string)
-
+
def writeBinary(self, binary):
self.writeJSONBase64(binary)
@@ -485,49 +520,49 @@
class TSimpleJSONProtocol(TJSONProtocolBase):
"""Simple, readable, write-only JSON protocol.
-
+
Useful for interacting with scripting languages.
"""
def readMessageBegin(self):
raise NotImplementedError()
-
+
def readMessageEnd(self):
raise NotImplementedError()
-
+
def readStructBegin(self):
raise NotImplementedError()
-
+
def readStructEnd(self):
raise NotImplementedError()
-
+
def writeMessageBegin(self, name, request_type, seqid):
self.resetWriteContext()
-
+
def writeMessageEnd(self):
pass
-
+
def writeStructBegin(self, name):
self.writeJSONObjectStart()
-
+
def writeStructEnd(self):
self.writeJSONObjectEnd()
-
+
def writeFieldBegin(self, name, ttype, fid):
self.writeJSONString(name)
-
+
def writeFieldEnd(self):
pass
-
+
def writeMapBegin(self, ktype, vtype, size):
self.writeJSONObjectStart()
-
+
def writeMapEnd(self):
self.writeJSONObjectEnd()
-
+
def _writeCollectionBegin(self, etype, size):
self.writeJSONArrayStart()
-
+
def _writeCollectionEnd(self):
self.writeJSONArrayEnd()
writeListBegin = _writeCollectionBegin
@@ -550,16 +585,16 @@
def writeI64(self, i64):
checkIntegerLimits(i64, 64)
self.writeJSONNumber(i64)
-
+
def writeBool(self, boolean):
self.writeJSONNumber(1 if boolean is True else 0)
def writeDouble(self, dbl):
self.writeJSONNumber(dbl)
-
+
def writeString(self, string):
self.writeJSONString(string)
-
+
def writeBinary(self, binary):
self.writeJSONBase64(binary)
diff --git a/lib/py/src/protocol/TProtocol.py b/lib/py/src/protocol/TProtocol.py
index 311a635..22339c0 100644
--- a/lib/py/src/protocol/TProtocol.py
+++ b/lib/py/src/protocol/TProtocol.py
@@ -17,7 +17,10 @@
# under the License.
#
-from thrift.Thrift import *
+from thrift.Thrift import TException, TType
+import six
+
+from ..compat import binary_to_str, str_to_binary
class TProtocolException(TException):
@@ -100,6 +103,9 @@
pass
def writeString(self, str_val):
+ self.writeBinary(str_to_binary(str_val))
+
+ def writeBinary(self, str_val):
pass
def readMessageBegin(self):
@@ -157,6 +163,9 @@
pass
def readString(self):
+ return binary_to_str(self.readBinary())
+
+ def readBinary(self):
pass
def skip(self, ttype):
@@ -187,18 +196,18 @@
self.readStructEnd()
elif ttype == TType.MAP:
(ktype, vtype, size) = self.readMapBegin()
- for i in xrange(size):
+ for i in range(size):
self.skip(ktype)
self.skip(vtype)
self.readMapEnd()
elif ttype == TType.SET:
(etype, size) = self.readSetBegin()
- for i in xrange(size):
+ for i in range(size):
self.skip(etype)
self.readSetEnd()
elif ttype == TType.LIST:
(etype, size) = self.readListBegin()
- for i in xrange(size):
+ for i in range(size):
self.skip(etype)
self.readListEnd()
@@ -246,13 +255,13 @@
(list_type, list_len) = self.readListBegin()
if tspec is None:
# list values are simple types
- for idx in xrange(list_len):
+ for idx in range(list_len):
results.append(reader())
else:
# this is like an inlined readFieldByTType
container_reader = self._TTYPE_HANDLERS[list_type][0]
val_reader = getattr(self, container_reader)
- for idx in xrange(list_len):
+ for idx in range(list_len):
val = val_reader(tspec)
results.append(val)
self.readListEnd()
@@ -266,12 +275,12 @@
(set_type, set_len) = self.readSetBegin()
if tspec is None:
# set members are simple types
- for idx in xrange(set_len):
+ for idx in range(set_len):
results.add(reader())
else:
container_reader = self._TTYPE_HANDLERS[set_type][0]
val_reader = getattr(self, container_reader)
- for idx in xrange(set_len):
+ for idx in range(set_len):
results.add(val_reader(tspec))
self.readSetEnd()
return results
@@ -292,7 +301,7 @@
key_reader = getattr(self, self._TTYPE_HANDLERS[key_ttype][0])
val_reader = getattr(self, self._TTYPE_HANDLERS[val_ttype][0])
# list values are simple types
- for idx in xrange(map_len):
+ for idx in range(map_len):
if key_spec is None:
k_val = key_reader()
else:
@@ -363,7 +372,7 @@
k_writer = getattr(self, ktype_name)
v_writer = getattr(self, vtype_name)
self.writeMapBegin(k_type, v_type, len(val))
- for m_key, m_val in val.iteritems():
+ for m_key, m_val in six.iteritems(val):
if not k_is_container:
k_writer(m_key)
else:
@@ -402,6 +411,7 @@
else:
writer(val)
+
def checkIntegerLimits(i, bits):
if bits == 8 and (i < -128 or i > 127):
raise TProtocolException(TProtocolException.INVALID_DATA,
@@ -416,6 +426,7 @@
raise TProtocolException(TProtocolException.INVALID_DATA,
"i64 requires -9223372036854775808 <= number <= 9223372036854775807")
+
class TProtocolFactory:
def getProtocol(self, trans):
pass
diff --git a/lib/py/src/server/THttpServer.py b/lib/py/src/server/THttpServer.py
index 6f92173..bf3b0e3 100644
--- a/lib/py/src/server/THttpServer.py
+++ b/lib/py/src/server/THttpServer.py
@@ -17,7 +17,7 @@
# under the License.
#
-import BaseHTTPServer
+from six.moves import BaseHTTPServer
from thrift.server import TServer
from thrift.transport import TTransport
diff --git a/lib/py/src/server/TNonblockingServer.py b/lib/py/src/server/TNonblockingServer.py
index 39486cd..79748b7 100644
--- a/lib/py/src/server/TNonblockingServer.py
+++ b/lib/py/src/server/TNonblockingServer.py
@@ -26,13 +26,14 @@
"""
import threading
import socket
-import Queue
import select
import struct
import logging
logger = logging.getLogger(__name__)
+from six.moves import queue
+
from thrift.transport import TTransport
from thrift.protocol.TBinaryProtocol import TBinaryProtocolFactory
@@ -57,7 +58,7 @@
callback(True, otrans.getvalue())
except Exception:
logger.exception("Exception while processing request")
- callback(False, '')
+ callback(False, b'')
WAIT_LEN = 0
WAIT_MESSAGE = 1
@@ -104,7 +105,7 @@
self.socket.setblocking(False)
self.status = WAIT_LEN
self.len = 0
- self.message = ''
+ self.message = b''
self.lock = threading.Lock()
self.wake_up = wake_up
@@ -132,7 +133,7 @@
logger.error("empty frame, it's really strange")
self.close()
else:
- self.message = ''
+ self.message = b''
self.status = WAIT_MESSAGE
@socket_exception
@@ -162,7 +163,7 @@
sent = self.socket.send(self.message)
if sent == len(self.message):
self.status = WAIT_LEN
- self.message = ''
+ self.message = b''
self.len = 0
else:
self.message = self.message[sent:]
@@ -185,10 +186,10 @@
self.close()
self.wake_up()
return
- self.len = ''
+ self.len = 0
if len(message) == 0:
# it was a oneway request, do not write answer
- self.message = ''
+ self.message = b''
self.status = WAIT_LEN
else:
self.message = struct.pack('!i', len(message)) + message
@@ -236,7 +237,7 @@
self.out_protocol = outputProtocolFactory or self.in_protocol
self.threads = int(threads)
self.clients = {}
- self.tasks = Queue.Queue()
+ self.tasks = queue.Queue()
self._read, self._write = socket.socketpair()
self.prepared = False
self._stop = False
@@ -252,7 +253,7 @@
if self.prepared:
return
self.socket.listen()
- for _ in xrange(self.threads):
+ for _ in range(self.threads):
thread = Worker(self.tasks)
thread.setDaemon(True)
thread.start()
@@ -269,7 +270,7 @@
In this case, we can just write anything to the second socket from
socketpair.
"""
- self._write.send('1')
+ self._write.send(b'1')
def stop(self):
"""Stop the server.
@@ -290,7 +291,7 @@
"""Does select on open connections."""
readable = [self.socket.handle.fileno(), self._read.fileno()]
writable = []
- for i, connection in self.clients.items():
+ for i, connection in list(self.clients.items()):
if connection.is_readable():
readable.append(connection.fileno())
if connection.is_writeable():
@@ -332,7 +333,7 @@
def close(self):
"""Closes the server."""
- for _ in xrange(self.threads):
+ for _ in range(self.threads):
self.tasks.put([None, None, None, None, None])
self.socket.close()
self.prepared = False
diff --git a/lib/py/src/server/TProcessPoolServer.py b/lib/py/src/server/TProcessPoolServer.py
index ae7fe1c..b2c2308 100644
--- a/lib/py/src/server/TProcessPoolServer.py
+++ b/lib/py/src/server/TProcessPoolServer.py
@@ -23,7 +23,7 @@
from multiprocessing import Process, Value, Condition, reduction
-from TServer import TServer
+from .TServer import TServer
from thrift.transport.TTransport import TTransportException
diff --git a/lib/py/src/server/TServer.py b/lib/py/src/server/TServer.py
index 8c58e39..130f730 100644
--- a/lib/py/src/server/TServer.py
+++ b/lib/py/src/server/TServer.py
@@ -17,7 +17,7 @@
# under the License.
#
-import Queue
+from six.moves import queue
import os
import sys
import threading
@@ -139,7 +139,7 @@
def __init__(self, *args, **kwargs):
TServer.__init__(self, *args)
- self.clients = Queue.Queue()
+ self.clients = queue.Queue()
self.threads = 10
self.daemon = kwargs.get("daemon", False)
@@ -250,7 +250,7 @@
try:
while True:
self.processor.process(iprot, oprot)
- except TTransport.TTransportException as tx:
+ except TTransport.TTransportException:
pass
except Exception as e:
logger.exception(e)
@@ -261,7 +261,7 @@
os._exit(ecode)
- except TTransport.TTransportException as tx:
+ except TTransport.TTransportException:
pass
except Exception as x:
logger.exception(x)
diff --git a/lib/py/src/transport/THttpClient.py b/lib/py/src/transport/THttpClient.py
index 5851fa2..5abd41c 100644
--- a/lib/py/src/transport/THttpClient.py
+++ b/lib/py/src/transport/THttpClient.py
@@ -17,17 +17,17 @@
# under the License.
#
-import httplib
+from io import BytesIO
import os
import socket
import sys
-import urllib
-import urlparse
import warnings
-from cStringIO import StringIO
+from six.moves import urllib
+from six.moves import http_client
-from TTransport import *
+from .TTransport import *
+import six
class THttpClient(TTransportBase):
@@ -52,31 +52,33 @@
self.path = path
self.scheme = 'http'
else:
- parsed = urlparse.urlparse(uri_or_host)
+ parsed = urllib.parse.urlparse(uri_or_host)
self.scheme = parsed.scheme
assert self.scheme in ('http', 'https')
if self.scheme == 'http':
- self.port = parsed.port or httplib.HTTP_PORT
+ self.port = parsed.port or http_client.HTTP_PORT
elif self.scheme == 'https':
- self.port = parsed.port or httplib.HTTPS_PORT
+ self.port = parsed.port or http_client.HTTPS_PORT
self.host = parsed.hostname
self.path = parsed.path
if parsed.query:
self.path += '?%s' % parsed.query
- self.__wbuf = StringIO()
+ self.__wbuf = BytesIO()
self.__http = None
+ self.__http_response = None
self.__timeout = None
self.__custom_headers = None
def open(self):
if self.scheme == 'http':
- self.__http = httplib.HTTP(self.host, self.port)
+ self.__http = http_client.HTTPConnection(self.host, self.port)
else:
- self.__http = httplib.HTTPS(self.host, self.port)
+ self.__http = http_client.HTTPSConnection(self.host, self.port)
def close(self):
self.__http.close()
self.__http = None
+ self.__http_response = None
def isOpen(self):
return self.__http is not None
@@ -94,7 +96,7 @@
self.__custom_headers = headers
def read(self, sz):
- return self.__http.file.read(sz)
+ return self.__http_response.read(sz)
def write(self, buf):
self.__wbuf.write(buf)
@@ -117,13 +119,12 @@
# Pull data out of buffer
data = self.__wbuf.getvalue()
- self.__wbuf = StringIO()
+ self.__wbuf = BytesIO()
# HTTP request
self.__http.putrequest('POST', self.path)
# Write headers
- self.__http.putheader('Host', self.host)
self.__http.putheader('Content-Type', 'application/x-thrift')
self.__http.putheader('Content-Length', str(len(data)))
@@ -131,11 +132,11 @@
user_agent = 'Python/THttpClient'
script = os.path.basename(sys.argv[0])
if script:
- user_agent = '%s (%s)' % (user_agent, urllib.quote(script))
+ user_agent = '%s (%s)' % (user_agent, urllib.parse.quote(script))
self.__http.putheader('User-Agent', user_agent)
if self.__custom_headers:
- for key, val in self.__custom_headers.iteritems():
+ for key, val in six.iteritems(self.__custom_headers):
self.__http.putheader(key, val)
self.__http.endheaders()
@@ -144,7 +145,10 @@
self.__http.send(data)
# Get reply to flush the request
- self.code, self.message, self.headers = self.__http.getreply()
+ self.__http_response = self.__http.getresponse()
+ self.code = self.__http_response.status
+ self.message = self.__http_response.reason
+ self.headers = self.__http_response.msg
# Decorate if we know how to timeout
if hasattr(socket, 'getdefaulttimeout'):
diff --git a/lib/py/src/transport/TSocket.py b/lib/py/src/transport/TSocket.py
index 7b564aa..cb204a4 100644
--- a/lib/py/src/transport/TSocket.py
+++ b/lib/py/src/transport/TSocket.py
@@ -22,7 +22,7 @@
import socket
import sys
-from TTransport import *
+from .TTransport import *
class TSocketBase(TTransportBase):
diff --git a/lib/py/src/transport/TTransport.py b/lib/py/src/transport/TTransport.py
index 5914aca..3fe289a 100644
--- a/lib/py/src/transport/TTransport.py
+++ b/lib/py/src/transport/TTransport.py
@@ -17,9 +17,9 @@
# under the License.
#
-from cStringIO import StringIO
from struct import pack, unpack
from thrift.Thrift import TException
+from ..compat import BufferIO
class TTransportException(TException):
@@ -52,7 +52,7 @@
pass
def readAll(self, sz):
- buff = ''
+ buff = b''
have = 0
while (have < sz):
chunk = self.read(sz - have)
@@ -138,8 +138,8 @@
def __init__(self, trans, rbuf_size=DEFAULT_BUFFER):
self.__trans = trans
- self.__wbuf = StringIO()
- self.__rbuf = StringIO("")
+ self.__wbuf = BufferIO()
+ self.__rbuf = BufferIO()
self.__rbuf_size = rbuf_size
def isOpen(self):
@@ -155,8 +155,7 @@
ret = self.__rbuf.read(sz)
if len(ret) != 0:
return ret
-
- self.__rbuf = StringIO(self.__trans.read(max(sz, self.__rbuf_size)))
+ self.__rbuf = BufferIO(self.__trans.read(max(sz, self.__rbuf_size)))
return self.__rbuf.read(sz)
def write(self, buf):
@@ -164,13 +163,14 @@
self.__wbuf.write(buf)
except Exception as e:
# on exception reset wbuf so it doesn't contain a partial function call
- self.__wbuf = StringIO()
+ self.__wbuf = BufferIO()
raise e
+ self.__wbuf.getvalue()
def flush(self):
out = self.__wbuf.getvalue()
# reset wbuf before write/flush to preserve state on underlying failure
- self.__wbuf = StringIO()
+ self.__wbuf = BufferIO()
self.__trans.write(out)
self.__trans.flush()
@@ -189,12 +189,12 @@
if len(retstring) < reqlen:
retstring += self.__trans.readAll(reqlen - len(retstring))
- self.__rbuf = StringIO(retstring)
+ self.__rbuf = BufferIO(retstring)
return self.__rbuf
class TMemoryBuffer(TTransportBase, CReadableTransport):
- """Wraps a cStringIO object as a TTransport.
+ """Wraps a cBytesIO object as a TTransport.
NOTE: Unlike the C++ version of this class, you cannot write to it
then immediately read from it. If you want to read from a
@@ -208,9 +208,9 @@
If value is set, this will be a transport for reading,
otherwise, it is for writing"""
if value is not None:
- self._buffer = StringIO(value)
+ self._buffer = BufferIO(value)
else:
- self._buffer = StringIO()
+ self._buffer = BufferIO()
def isOpen(self):
return not self._buffer.closed
@@ -256,8 +256,8 @@
def __init__(self, trans,):
self.__trans = trans
- self.__rbuf = StringIO()
- self.__wbuf = StringIO()
+ self.__rbuf = BufferIO()
+ self.__wbuf = BufferIO()
def isOpen(self):
return self.__trans.isOpen()
@@ -279,7 +279,7 @@
def readFrame(self):
buff = self.__trans.readAll(4)
sz, = unpack('!i', buff)
- self.__rbuf = StringIO(self.__trans.readAll(sz))
+ self.__rbuf = BufferIO(self.__trans.readAll(sz))
def write(self, buf):
self.__wbuf.write(buf)
@@ -288,7 +288,7 @@
wout = self.__wbuf.getvalue()
wsz = len(wout)
# reset wbuf before write/flush to preserve state on underlying failure
- self.__wbuf = StringIO()
+ self.__wbuf = BufferIO()
# N.B.: Doing this string concatenation is WAY cheaper than making
# two separate calls to the underlying socket object. Socket writes in
# Python turn out to be REALLY expensive, but it seems to do a pretty
@@ -309,7 +309,7 @@
while len(prefix) < reqlen:
self.readFrame()
prefix += self.__rbuf.getvalue()
- self.__rbuf = StringIO(prefix)
+ self.__rbuf = BufferIO(prefix)
return self.__rbuf
@@ -337,7 +337,7 @@
class TSaslClientTransport(TTransportBase, CReadableTransport):
"""
- SASL transport
+ SASL transport
"""
START = 1
@@ -363,8 +363,8 @@
self.transport = transport
self.sasl = SASLClient(host, service, mechanism, **sasl_kwargs)
- self.__wbuf = StringIO()
- self.__rbuf = StringIO()
+ self.__wbuf = BufferIO()
+ self.__rbuf = BufferIO()
def open(self):
if not self.transport.isOpen():
@@ -409,7 +409,7 @@
encoded = self.sasl.wrap(data)
self.transport.write(''.join((pack("!i", len(encoded)), encoded)))
self.transport.flush()
- self.__wbuf = StringIO()
+ self.__wbuf = BufferIO()
def read(self, sz):
ret = self.__rbuf.read(sz)
@@ -423,7 +423,7 @@
header = self.transport.readAll(4)
length, = unpack('!i', header)
encoded = self.transport.readAll(length)
- self.__rbuf = StringIO(self.sasl.unwrap(encoded))
+ self.__rbuf = BufferIO(self.sasl.unwrap(encoded))
def close(self):
self.sasl.dispose()
@@ -441,6 +441,6 @@
while len(prefix) < reqlen:
self._read_frame()
prefix += self.__rbuf.getvalue()
- self.__rbuf = StringIO(prefix)
+ self.__rbuf = BufferIO(prefix)
return self.__rbuf
diff --git a/lib/py/src/transport/TTwisted.py b/lib/py/src/transport/TTwisted.py
index 29bbd4c..cabe345 100644
--- a/lib/py/src/transport/TTwisted.py
+++ b/lib/py/src/transport/TTwisted.py
@@ -17,8 +17,8 @@
# under the License.
#
+from io import BytesIO
import struct
-from cStringIO import StringIO
from zope.interface import implements, Interface, Attribute
from twisted.internet.protocol import ServerFactory, ClientFactory, \
@@ -29,19 +29,20 @@
from twisted.web import server, resource, http
from thrift.transport import TTransport
+import six
class TMessageSenderTransport(TTransport.TTransportBase):
def __init__(self):
- self.__wbuf = StringIO()
+ self.__wbuf = BytesIO()
def write(self, buf):
self.__wbuf.write(buf)
def flush(self):
msg = self.__wbuf.getvalue()
- self.__wbuf = StringIO()
+ self.__wbuf = BytesIO()
return self.sendMessage(msg)
def sendMessage(self, message):
@@ -82,7 +83,7 @@
self.started.callback(self.client)
def connectionLost(self, reason=connectionDone):
- for k, v in self.client._reqs.iteritems():
+ for k, v in six.iteritems(self.client._reqs):
tex = TTransport.TTransportException(
type=TTransport.TTransportException.END_OF_FILE,
message='Connection closed')
diff --git a/lib/py/src/transport/TZlibTransport.py b/lib/py/src/transport/TZlibTransport.py
index a2f42a5..7fe5853 100644
--- a/lib/py/src/transport/TZlibTransport.py
+++ b/lib/py/src/transport/TZlibTransport.py
@@ -24,8 +24,8 @@
from __future__ import division
import zlib
-from cStringIO import StringIO
-from TTransport import TTransportBase, CReadableTransport
+from .TTransport import TTransportBase, CReadableTransport
+from ..compat import BufferIO
class TZlibTransportFactory(object):
@@ -88,8 +88,8 @@
"""
self.__trans = trans
self.compresslevel = compresslevel
- self.__rbuf = StringIO()
- self.__wbuf = StringIO()
+ self.__rbuf = BufferIO()
+ self.__wbuf = BufferIO()
self._init_zlib()
self._init_stats()
@@ -97,8 +97,8 @@
"""Internal method to initialize/reset the internal StringIO objects
for read and write buffers.
"""
- self.__rbuf = StringIO()
- self.__wbuf = StringIO()
+ self.__rbuf = BufferIO()
+ self.__wbuf = BufferIO()
def _init_stats(self):
"""Internal method to reset the internal statistics counters
@@ -203,7 +203,7 @@
self.bytes_in += len(zbuf)
self.bytes_in_comp += len(buf)
old = self.__rbuf.read()
- self.__rbuf = StringIO(old + buf)
+ self.__rbuf = BufferIO(old + buf)
if len(old) + len(buf) == 0:
return False
return True
@@ -228,7 +228,7 @@
ztail = self._zcomp_write.flush(zlib.Z_SYNC_FLUSH)
self.bytes_out_comp += len(ztail)
if (len(zbuf) + len(ztail)) > 0:
- self.__wbuf = StringIO()
+ self.__wbuf = BufferIO()
self.__trans.write(zbuf + ztail)
self.__trans.flush()
@@ -244,5 +244,5 @@
retstring += self.read(self.DEFAULT_BUFFSIZE)
while len(retstring) < reqlen:
retstring += self.read(reqlen - len(retstring))
- self.__rbuf = StringIO(retstring)
+ self.__rbuf = BufferIO(retstring)
return self.__rbuf