THRIFT-5923: UUID python
Client: py
Patch: CJCombrink
This closes #3330
diff --git a/lib/py/setup.py b/lib/py/setup.py
index 2dd2a77..456fd6d 100644
--- a/lib/py/setup.py
+++ b/lib/py/setup.py
@@ -85,6 +85,14 @@
'src/ext/binary.cpp',
'src/ext/compact.cpp',
],
+ depends=[
+ 'src/ext/binary.h',
+ 'src/ext/compact.h',
+ 'src/ext/endian.h',
+ 'src/ext/protocol.h',
+ 'src/ext/protocol.tcc',
+ 'src/ext/types.h',
+ ],
include_dirs=include_dirs,
)
],
@@ -138,6 +146,8 @@
try:
with_binary = True
run_setup(with_binary)
+ sys.exit(0)
+
except BuildFailed:
print()
print('*' * 80)
@@ -146,4 +156,16 @@
print('*' * 80)
print()
+# Retry but without the binary
+try:
run_setup(False)
+ sys.exit(0)
+
+except BuildFailed:
+ print()
+ print('*' * 80)
+ print("An error occurred while trying to compile without the C extension enabled")
+ print("Build failed")
+ print('*' * 80)
+ print()
+ sys.exit(1)
diff --git a/lib/py/src/Thrift.py b/lib/py/src/Thrift.py
index 81fe8cf..7b28c90 100644
--- a/lib/py/src/Thrift.py
+++ b/lib/py/src/Thrift.py
@@ -34,8 +34,7 @@
MAP = 13
SET = 14
LIST = 15
- UTF8 = 16
- UTF16 = 17
+ UUID = 16
_VALUES_TO_NAMES = (
'STOP',
@@ -54,8 +53,7 @@
'MAP',
'SET',
'LIST',
- 'UTF8',
- 'UTF16',
+ 'UUID',
)
diff --git a/lib/py/src/ext/binary.h b/lib/py/src/ext/binary.h
index 960b0d0..dd7750b 100644
--- a/lib/py/src/ext/binary.h
+++ b/lib/py/src/ext/binary.h
@@ -88,6 +88,10 @@
return encodeValue(value, parsedspec.type, parsedspec.typeargs);
}
+ void writeUuid(char* value) {
+ writeBuffer(value, 16);
+ }
+
void writeFieldStop() { writeByte(static_cast<uint8_t>(T_STOP)); }
bool readBool(bool& val) {
@@ -159,6 +163,13 @@
return len;
}
+ int32_t readUuid(char** buf) {
+ if (!readBytes(buf, 16)) {
+ return -1;
+ }
+ return 16;
+ }
+
int32_t readListBegin(TType& etype) {
int32_t len;
uint8_t b = 0;
@@ -206,6 +217,7 @@
}
SKIPBYTES(len);
}
+ bool skipUuid() { SKIPBYTES(16); }
#undef SKIPBYTES
private:
diff --git a/lib/py/src/ext/compact.cpp b/lib/py/src/ext/compact.cpp
index ae89f2a..8d13d3d 100644
--- a/lib/py/src/ext/compact.cpp
+++ b/lib/py/src/ext/compact.cpp
@@ -24,23 +24,26 @@
namespace thrift {
namespace py {
+/** Mapping of Compact type to Thrift Type according.
+ * This list must match the TType enum in TEnum.h */
const uint8_t CompactProtocol::TTypeToCType[] = {
- CT_STOP, // T_STOP
- 0, // unused
- CT_BOOLEAN_TRUE, // T_BOOL
- CT_BYTE, // T_BYTE
- CT_DOUBLE, // T_DOUBLE
- 0, // unused
- CT_I16, // T_I16
- 0, // unused
- CT_I32, // T_I32
- 0, // unused
- CT_I64, // T_I64
- CT_BINARY, // T_STRING
- CT_STRUCT, // T_STRUCT
- CT_MAP, // T_MAP
- CT_SET, // T_SET
- CT_LIST, // T_LIST
+/* 0 */ CT_STOP, // T_STOP
+/* 1 */ 0, // unused
+/* 2 */ CT_BOOLEAN_TRUE, // T_BOOL
+/* 3 */ CT_BYTE, // T_BYTE
+/* 4 */ CT_DOUBLE, // T_DOUBLE
+/* 5 */ 0, // unused
+/* 6 */ CT_I16, // T_I16
+/* 7 */ 0, // unused
+/* 8 */ CT_I32, // T_I32
+/* 9 */ 0, // unused
+/* 10 */ CT_I64, // T_I64
+/* 11 */ CT_BINARY, // T_STRING
+/* 12 */ CT_STRUCT, // T_STRUCT
+/* 13 */ CT_MAP, // T_MAP
+/* 14 */ CT_SET, // T_SET
+/* 15 */ CT_LIST, // T_LIST
+/* 16 */ CT_UUID, // T_UUID
};
bool CompactProtocol::readFieldBegin(TType& type, int16_t& tag) {
@@ -98,6 +101,8 @@
return T_MAP;
case CT_STRUCT:
return T_STRUCT;
+ case CT_UUID:
+ return T_UUID;
default:
PyErr_Format(PyExc_TypeError, "don't know what type: %d", type);
return static_cast<TType>(-1);
diff --git a/lib/py/src/ext/compact.h b/lib/py/src/ext/compact.h
index 8f72b09..0d8946b 100644
--- a/lib/py/src/ext/compact.h
+++ b/lib/py/src/ext/compact.h
@@ -104,6 +104,10 @@
void writeFieldStop() { writeByte(0); }
+ void writeUuid(char* value) {
+ writeBuffer(value, 16);
+ }
+
bool readBool(bool& val) {
if (readBool_.exists) {
readBool_.exists = false;
@@ -231,6 +235,13 @@
}
bool readFieldBegin(TType& type, int16_t& tag);
+ bool readUuid(char** buf) {
+ if (!readBytes(buf, 16)) {
+ return false;
+ }
+ return true;
+ }
+
bool skipBool() {
bool val;
return readBool(val);
@@ -263,6 +274,9 @@
}
SKIPBYTES(len);
}
+ bool skipUuid() {
+ SKIPBYTES(16);
+ }
#undef SKIPBYTES
private:
@@ -279,7 +293,8 @@
CT_LIST = 0x09,
CT_SET = 0x0A,
CT_MAP = 0x0B,
- CT_STRUCT = 0x0C
+ CT_STRUCT = 0x0C,
+ CT_UUID = 0x0D,
};
static const uint8_t TTypeToCType[];
@@ -288,7 +303,7 @@
int toCompactType(TType type) {
int i = static_cast<int>(type);
- return i < 16 ? TTypeToCType[i] : -1;
+ return i <= 16 ? TTypeToCType[i] : -1;
}
uint32_t toZigZag(int32_t val) { return (val >> 31) ^ (val << 1); }
diff --git a/lib/py/src/ext/module.cpp b/lib/py/src/ext/module.cpp
index a1b0e56..e2b540e 100644
--- a/lib/py/src/ext/module.cpp
+++ b/lib/py/src/ext/module.cpp
@@ -38,6 +38,8 @@
PyObject* INTERN_STRING(TFrozenDict);
PyObject* INTERN_STRING(cstringio_buf);
PyObject* INTERN_STRING(cstringio_refill);
+PyObject* INTERN_STRING(UUID);
+PyObject* INTERN_STRING(bytes);
static PyObject* INTERN_STRING(string_length_limit);
static PyObject* INTERN_STRING(container_length_limit);
static PyObject* INTERN_STRING(trans);
@@ -186,6 +188,8 @@
INIT_INTERN_STRING(string_length_limit);
INIT_INTERN_STRING(container_length_limit);
INIT_INTERN_STRING(trans);
+ INIT_INTERN_STRING(UUID);
+ INIT_INTERN_STRING(bytes);
#undef INIT_INTERN_STRING
PyObject* module =
diff --git a/lib/py/src/ext/protocol.tcc b/lib/py/src/ext/protocol.tcc
index aad5a3c..b517c38 100644
--- a/lib/py/src/ext/protocol.tcc
+++ b/lib/py/src/ext/protocol.tcc
@@ -542,10 +542,27 @@
return true;
}
+ case T_UUID: {
+ ScopedPyObject instval(PyObject_GetAttr(value, INTERN_STRING(bytes)));
+ if (!instval) {
+ return false;
+ }
+
+ Py_ssize_t size;
+ char* buffer;
+ if (PyBytes_AsStringAndSize(instval.get(), &buffer, &size) < 0) {
+ return false;
+ }
+ if (size != 16) {
+ PyErr_SetString(PyExc_TypeError, "uuid.bytes must be exactly 16 bytes long");
+ return false;
+ }
+ impl()->writeUuid(buffer);
+ return true;
+ }
+
case T_STOP:
case T_VOID:
- case T_UTF16:
- case T_UTF8:
case T_U64:
default:
PyErr_Format(PyExc_TypeError, "Unexpected TType for encodeValue: %d", type);
@@ -625,11 +642,12 @@
}
return true;
}
+ case T_UUID: {
+ return impl()->skipUuid();
+ }
case T_STOP:
case T_VOID:
- case T_UTF16:
- case T_UTF8:
case T_U64:
default:
PyErr_Format(PyExc_TypeError, "Unexpected TType for skip: %d", type);
@@ -816,10 +834,36 @@
return readStruct(Py_None, parsedargs.klass, parsedargs.spec);
}
+ case T_UUID: {
+ char* buf = nullptr;
+ if(!impl()->readUuid(&buf)) {
+ return nullptr;
+ }
+
+ if(!UuidModule) {
+ UuidModule = PyImport_ImportModule("uuid");
+ if (!UuidModule)
+ return nullptr;
+ }
+
+ ScopedPyObject cls(PyObject_GetAttr(UuidModule, INTERN_STRING(UUID)));
+ if (!cls) {
+ return nullptr;
+ }
+
+ ScopedPyObject pyBytes(PyBytes_FromStringAndSize(buf, 16));
+ if (!pyBytes) {
+ return nullptr;
+ }
+
+ ScopedPyObject args(PyTuple_New(0));
+ ScopedPyObject kwargs(Py_BuildValue("{O:O}", INTERN_STRING(bytes), pyBytes.get()));
+ ScopedPyObject ret(PyObject_Call(cls.get(), args.get(), kwargs.get()));
+ return ret.release();
+ }
+
case T_STOP:
case T_VOID:
- case T_UTF16:
- case T_UTF8:
case T_U64:
default:
PyErr_Format(PyExc_TypeError, "Unexpected TType for decodeValue: %d", type);
diff --git a/lib/py/src/ext/types.cpp b/lib/py/src/ext/types.cpp
index 0c20e56..a7ccd0a 100644
--- a/lib/py/src/ext/types.cpp
+++ b/lib/py/src/ext/types.cpp
@@ -26,6 +26,7 @@
namespace py {
PyObject* ThriftModule = nullptr;
+PyObject* UuidModule = nullptr;
#if PY_MAJOR_VERSION < 3
char refill_signature[] = {'s', '#', 'i'};
diff --git a/lib/py/src/ext/types.h b/lib/py/src/ext/types.h
index 9b45dd0..2848b28 100644
--- a/lib/py/src/ext/types.h
+++ b/lib/py/src/ext/types.h
@@ -48,6 +48,8 @@
extern PyObject* INTERN_STRING(TFrozenDict);
extern PyObject* INTERN_STRING(cstringio_buf);
extern PyObject* INTERN_STRING(cstringio_refill);
+extern PyObject* INTERN_STRING(UUID);
+extern PyObject* INTERN_STRING(bytes);
}
namespace apache {
@@ -55,6 +57,7 @@
namespace py {
extern PyObject* ThriftModule;
+extern PyObject* UuidModule;
// Stolen out of TProtocol.h.
// It would be a huge pain to have both get this from one place.
@@ -76,8 +79,7 @@
T_MAP = 13,
T_SET = 14,
T_LIST = 15,
- T_UTF8 = 16,
- T_UTF16 = 17
+ T_UUID = 16,
};
// replace with unique_ptr when we're OK with C++11
diff --git a/lib/py/src/protocol/TBinaryProtocol.py b/lib/py/src/protocol/TBinaryProtocol.py
index af64ec1..b73e3c9 100644
--- a/lib/py/src/protocol/TBinaryProtocol.py
+++ b/lib/py/src/protocol/TBinaryProtocol.py
@@ -18,6 +18,7 @@
#
from struct import pack, unpack
+import uuid
from .TProtocol import TType, TProtocolBase, TProtocolException, TProtocolFactory
@@ -131,6 +132,9 @@
self.writeI32(len(str))
self.trans.write(str)
+ def writeUuid(self, uuid):
+ self.trans.write(uuid.bytes)
+
def readMessageBegin(self):
sz = self.readI32()
if sz < 0:
@@ -235,6 +239,11 @@
s = self.trans.readAll(size)
return s
+ def readUuid(self):
+ buff = self.trans.readAll(16)
+ val = uuid.UUID(bytes=buff)
+ return val
+
class TBinaryProtocolFactory(TProtocolFactory):
def __init__(self, strictRead=False, strictWrite=True, **kwargs):
diff --git a/lib/py/src/protocol/TCompactProtocol.py b/lib/py/src/protocol/TCompactProtocol.py
index a3527cd..b58095a 100644
--- a/lib/py/src/protocol/TCompactProtocol.py
+++ b/lib/py/src/protocol/TCompactProtocol.py
@@ -19,6 +19,7 @@
from .TProtocol import TType, TProtocolBase, TProtocolException, TProtocolFactory, checkIntegerLimits
from struct import pack, unpack
+import uuid
__all__ = ['TCompactProtocol', 'TCompactProtocolFactory']
@@ -80,6 +81,7 @@
shift += 7
+# As per TCompactProtocol.tcc
class CompactType(object):
STOP = 0x00
TRUE = 0x01
@@ -94,6 +96,7 @@
SET = 0x0A
MAP = 0x0B
STRUCT = 0x0C
+ UUID = 0x0D
CTYPES = {
@@ -109,6 +112,7 @@
TType.LIST: CompactType.LIST,
TType.SET: CompactType.SET,
TType.MAP: CompactType.MAP,
+ TType.UUID: CompactType.UUID,
}
TTYPES = {}
@@ -276,6 +280,10 @@
def writeDouble(self, dub):
self.trans.write(pack('<d', dub))
+ @writer
+ def writeUuid(self, uuid):
+ self.trans.write(uuid.bytes)
+
def __writeBinary(self, s):
self.__writeSize(len(s))
self.trans.write(s)
@@ -416,6 +424,12 @@
val, = unpack('<d', buff)
return val
+ @reader
+ def readUuid(self):
+ buff = self.trans.readAll(16)
+ val = uuid.UUID(bytes=buff)
+ return val
+
def __readBinary(self):
size = self.__readSize()
self._check_string_length(size)
diff --git a/lib/py/src/protocol/TJSONProtocol.py b/lib/py/src/protocol/TJSONProtocol.py
index 004a40a..6782eeb 100644
--- a/lib/py/src/protocol/TJSONProtocol.py
+++ b/lib/py/src/protocol/TJSONProtocol.py
@@ -21,6 +21,7 @@
TProtocolFactory, checkIntegerLimits)
import base64
import math
+import uuid
__all__ = ['TJSONProtocol',
@@ -64,6 +65,7 @@
}
NUMERIC_CHAR = b'+-.0123456789Ee'
+# Type names as TJSONProtocol.cpp
CTYPES = {
TType.BOOL: 'tf',
TType.BYTE: 'i8',
@@ -76,6 +78,7 @@
TType.LIST: 'lst',
TType.SET: 'set',
TType.MAP: 'map',
+ TType.UUID: 'uid',
}
JTYPES = {}
@@ -480,6 +483,11 @@
def readBinary(self):
return self.readJSONBase64()
+ def readUuid(self):
+ buff = self.readJSONString(False)
+ val = uuid.UUID(buff)
+ return val
+
def writeMessageBegin(self, name, request_type, seqid):
self.resetWriteContext()
self.writeJSONArrayStart()
@@ -565,6 +573,9 @@
def writeBinary(self, binary):
self.writeJSONBase64(binary)
+ def writeUuid(self, uuid):
+ self.writeJSONString(str(uuid))
+
class TJSONProtocolFactory(TProtocolFactory):
def getProtocol(self, trans):
@@ -659,6 +670,9 @@
def writeBinary(self, binary):
self.writeJSONBase64(binary)
+ def writeUuid(self, uuid):
+ self.writeJSONString(str(uuid))
+
class TSimpleJSONProtocolFactory(TProtocolFactory):
diff --git a/lib/py/src/protocol/TProtocol.py b/lib/py/src/protocol/TProtocol.py
index a0d1452..975cbf5 100644
--- a/lib/py/src/protocol/TProtocol.py
+++ b/lib/py/src/protocol/TProtocol.py
@@ -117,7 +117,10 @@
def writeString(self, str_val):
self.writeBinary(bytes(str_val, 'utf-8'))
- def writeBinary(self, str_val):
+ def writeBinary(self, uuid):
+ pass
+
+ def writeUuid(self, str_val):
pass
def readMessageBegin(self):
@@ -180,6 +183,9 @@
def readBinary(self):
pass
+ def readUuid(self):
+ pass
+
def skip(self, ttype):
if ttype == TType.BOOL:
self.readBool()
@@ -220,6 +226,8 @@
for i in range(size):
self.skip(etype)
self.readListEnd()
+ elif ttype == TType.UUID:
+ self.readUuid()
else:
raise TProtocolException(
TProtocolException.INVALID_DATA,
@@ -243,8 +251,7 @@
('readContainerMap', 'writeContainerMap', True), # 13 TType.MAP
('readContainerSet', 'writeContainerSet', True), # 14 TType.SET
('readContainerList', 'writeContainerList', True), # 15 TType.LIST
- (None, None, False), # 16 TType.UTF8 # TODO: handle utf8 types?
- (None, None, False) # 17 TType.UTF16 # TODO: handle utf16 types?
+ ('readUuid', 'writeUuid', False), # 16 TType.UUID
)
def _ttype_handlers(self, ttype, spec):
diff --git a/lib/py/test/thrift_TBinaryProtocol.py b/lib/py/test/thrift_TBinaryProtocol.py
index c4777ad..e84bfe1 100644
--- a/lib/py/test/thrift_TBinaryProtocol.py
+++ b/lib/py/test/thrift_TBinaryProtocol.py
@@ -18,6 +18,7 @@
#
import unittest
+import uuid
import _import_local_thrift # noqa
from thrift.protocol.TBinaryProtocol import TBinaryProtocol
@@ -52,6 +53,9 @@
if type.capitalize() == 'Bool':
protocol.writeBool(data)
+ if type.capitalize() == 'Uuid':
+ protocol.writeUuid(data)
+
transport.flush()
data_r = buf.getvalue()
buf = TTransport.TMemoryBuffer(data_r)
@@ -81,9 +85,12 @@
if type.capitalize() == 'Bool':
return protocol.readBool()
+ if type.capitalize() == 'Uuid':
+ return protocol.readUuid()
+
def testField(type, data):
- TType = {"Bool": 2, "Byte": 3, "Binary": 5, "I16": 6, "I32": 8, "I64": 10, "Double": 11, "String": 12}
+ TType = {"Bool": 2, "Byte": 3, "Binary": 5, "I16": 6, "I32": 8, "I64": 10, "Double": 11, "String": 12, "Uuid": 13}
buf = TTransport.TMemoryBuffer()
transport = TTransport.TBufferedTransportFactory().getTransport(buf)
protocol = TBinaryProtocol(transport)
@@ -113,6 +120,9 @@
if type.capitalize() == 'Bool':
protocol.writeBool(data)
+ if type.capitalize() == 'Uuid':
+ protocol.writeUuid(data)
+
protocol.writeFieldEnd()
protocol.writeStructEnd()
@@ -148,6 +158,9 @@
if type.capitalize() == 'Bool':
return protocol.readBool()
+ if type.capitalize() == 'Uuid':
+ return protocol.readUuid()
+
protocol.readFieldEnd()
protocol.readStructEnd()
@@ -245,6 +258,8 @@
self.assertEqual(True, testField('Bool', True))
self.assertEqual(3.1415926, testNaked("Double", 3.1415926))
self.assertEqual("hello thrift", testNaked("String", "hello thrift"))
+ self.assertEqual(uuid.UUID('{00010203-0405-0607-0809-0a0b0c0d0e0f}'), testNaked("Uuid", uuid.UUID('{00010203-0405-0607-0809-0a0b0c0d0e0f}')))
+ self.assertEqual(uuid.UUID('{00010203-0405-0607-0809-0a0b0c0d0e0f}'), testField("Uuid", uuid.UUID('{00010203-0405-0607-0809-0a0b0c0d0e0f}')))
TMessageType = {"T_CALL": 1, "T_REPLY": 2, "T_EXCEPTION": 3, "T_ONEWAY": 4}
test_data = [("short message name", TMessageType['T_CALL'], 0),
diff --git a/lib/py/test/thrift_TCompactProtocol.py b/lib/py/test/thrift_TCompactProtocol.py
index 1d6af8e..3435702 100644
--- a/lib/py/test/thrift_TCompactProtocol.py
+++ b/lib/py/test/thrift_TCompactProtocol.py
@@ -21,6 +21,7 @@
from thrift.protocol import TCompactProtocol
from thrift.transport import TTransport
import unittest
+import uuid
CLEAR = 0
FIELD_WRITE = 1
@@ -70,6 +71,10 @@
protocol.state = CONTAINER_WRITE
protocol.writeBool(True)
+ if type.capitalize() == 'Uuid':
+ protocol.state = CONTAINER_WRITE
+ protocol.writeUuid(data)
+
transport.flush()
data_r = buf.getvalue()
buf = TTransport.TMemoryBuffer(data_r)
@@ -107,9 +112,13 @@
protocol.state = CONTAINER_READ
return protocol.readBool()
+ if type.capitalize() == 'Uuid':
+ protocol.state = CONTAINER_READ
+ return protocol.readUuid()
+
def testField(type, data):
- TType = {"Bool": 2, "Byte": 3, "Binary": 5, "I16": 6, "I32": 8, "I64": 10, "Double": 11, "String": 12}
+ TType = {"Bool": 2, "Byte": 3, "Binary": 5, "I16": 6, "I32": 8, "I64": 10, "Double": 11, "String": 12, "Uuid": 13}
buf = TTransport.TMemoryBuffer()
transport = TTransport.TBufferedTransportFactory().getTransport(buf)
protocol = TCompactProtocol.TCompactProtocol(transport)
@@ -139,6 +148,9 @@
elif type.capitalize() == 'Bool':
protocol.writeBool(data)
+ if type.capitalize() == 'Uuid':
+ protocol.writeUuid(data)
+
protocol.writeFieldEnd()
protocol.writeStructEnd()
@@ -174,6 +186,9 @@
elif type.capitalize() == 'Bool':
return protocol.readBool()
+ if type.capitalize() == 'Uuid':
+ return protocol.readUuid()
+
protocol.readFieldEnd()
protocol.readStructEnd()
@@ -268,6 +283,9 @@
self.assertEqual(True, testField('Bool', True))
self.assertEqual(3.14159261, testField('Double', 3.14159261))
self.assertEqual("hello thrift", testField('String', "hello thrift"))
+ self.assertEqual(uuid.UUID('{00010203-0405-0607-0809-0a0b0c0d0e0f}'), testNaked("Uuid", uuid.UUID('{00010203-0405-0607-0809-0a0b0c0d0e0f}')))
+ self.assertEqual(uuid.UUID('{00010203-0405-0607-0809-0a0b0c0d0e0f}'), testField("Uuid", uuid.UUID('{00010203-0405-0607-0809-0a0b0c0d0e0f}')))
+
TMessage = {"T_CALL": 1, "T_REPLY": 2, "T_EXCEPTION": 3, "T_ONEWAY": 4}
test_data = [("short message name", TMessage["T_CALL"], 0),
("1", TMessage["T_REPLY"], 12345),