Thrift now a TLP - INFRA-3116
git-svn-id: https://svn.apache.org/repos/asf/thrift/branches/0.1.x@1028168 13f79535-47bb-0310-9956-ffa450edef68
diff --git a/lib/py/Makefile.am b/lib/py/Makefile.am
new file mode 100644
index 0000000..6d3bbea
--- /dev/null
+++ b/lib/py/Makefile.am
@@ -0,0 +1,36 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+DESTDIR ?= /
+EXTRA_DIST = setup.py src
+
+all-local:
+ $(PYTHON) setup.py build
+
+# We're ignoring prefix here because site-packages seems to be
+# the equivalent of /usr/local/lib in Python land.
+# Old version (can't put inline because it's not portable).
+#$(PYTHON) setup.py install --prefix=$(prefix) --root=$(DESTDIR) $(PYTHON_SETUPUTIL_ARGS)
+install-exec-hook:
+ $(PYTHON) setup.py install --root=$(DESTDIR) --prefix=$(PY_PREFIX) $(PYTHON_SETUPUTIL_ARGS)
+
+clean-local:
+ $(RM) -r build
+
+check-local: all
diff --git a/lib/py/README b/lib/py/README
new file mode 100644
index 0000000..29b8c73
--- /dev/null
+++ b/lib/py/README
@@ -0,0 +1,35 @@
+Thrift Python Software Library
+
+License
+=======
+
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+
+Using Thrift with Python
+========================
+
+Thrift is provided as a set of Python packages. The top level package is
+thrift, and there are subpackages for the protocol, transport, and server
+code. Each package contains modules using standard Thrift naming conventions
+(i.e. TProtocol, TTransport) and implementations in corresponding modules
+(i.e. TSocket). There is also a subpackage reflection, which contains
+the generated code for the reflection structures.
+
+The Python libraries can be installed manually using the provided setup.py
+file, or automatically using the install hook provided via autoconf/automake.
+To use the latter, become superuser and do make install.
diff --git a/lib/py/setup.py b/lib/py/setup.py
new file mode 100644
index 0000000..7483711
--- /dev/null
+++ b/lib/py/setup.py
@@ -0,0 +1,43 @@
+#!/usr/bin/env python
+
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+from distutils.core import setup, Extension
+
+fastbinarymod = Extension('thrift.protocol.fastbinary',
+ sources = ['src/protocol/fastbinary.c'],
+ )
+
+setup(name = 'Thrift',
+ version = '1.0',
+ description = 'Thrift Python Libraries',
+ author = ['Mark Slee'],
+ author_email = ['mcslee@facebook.com'],
+ url = 'http://code.facebook.com/thrift',
+ packages = [
+ 'thrift',
+ 'thrift.protocol',
+ 'thrift.transport',
+ 'thrift.server',
+ ],
+ package_dir = {'thrift' : 'src'},
+ ext_modules = [fastbinarymod],
+ )
+
diff --git a/lib/py/src/TSCons.py b/lib/py/src/TSCons.py
new file mode 100644
index 0000000..2404625
--- /dev/null
+++ b/lib/py/src/TSCons.py
@@ -0,0 +1,33 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+from os import path
+from SCons.Builder import Builder
+
+def scons_env(env, add=''):
+ opath = path.dirname(path.abspath('$TARGET'))
+ lstr = 'thrift --gen cpp -o ' + opath + ' ' + add + ' $SOURCE'
+ cppbuild = Builder(action = lstr)
+ env.Append(BUILDERS = {'ThriftCpp' : cppbuild})
+
+def gen_cpp(env, dir, file):
+ scons_env(env)
+ suffixes = ['_types.h', '_types.cpp']
+ targets = map(lambda s: 'gen-cpp/' + file + s, suffixes)
+ return env.ThriftCpp(targets, dir+file+'.thrift')
diff --git a/lib/py/src/Thrift.py b/lib/py/src/Thrift.py
new file mode 100644
index 0000000..21d7aa4
--- /dev/null
+++ b/lib/py/src/Thrift.py
@@ -0,0 +1,123 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+class TType:
+ STOP = 0
+ VOID = 1
+ BOOL = 2
+ BYTE = 3
+ I08 = 3
+ DOUBLE = 4
+ I16 = 6
+ I32 = 8
+ I64 = 10
+ STRING = 11
+ UTF7 = 11
+ STRUCT = 12
+ MAP = 13
+ SET = 14
+ LIST = 15
+ UTF8 = 16
+ UTF16 = 17
+
+class TMessageType:
+ CALL = 1
+ REPLY = 2
+ EXCEPTION = 3
+ ONEWAY = 4
+
+class TProcessor:
+
+ """Base class for procsessor, which works on two streams."""
+
+ def process(iprot, oprot):
+ pass
+
+class TException(Exception):
+
+ """Base class for all thrift exceptions."""
+
+ def __init__(self, message=None):
+ Exception.__init__(self, message)
+ self.message = message
+
+class TApplicationException(TException):
+
+ """Application level thrift exceptions."""
+
+ UNKNOWN = 0
+ UNKNOWN_METHOD = 1
+ INVALID_MESSAGE_TYPE = 2
+ WRONG_METHOD_NAME = 3
+ BAD_SEQUENCE_ID = 4
+ MISSING_RESULT = 5
+
+ def __init__(self, type=UNKNOWN, message=None):
+ TException.__init__(self, message)
+ self.type = type
+
+ def __str__(self):
+ if self.message:
+ return self.message
+ elif self.type == UNKNOWN_METHOD:
+ return 'Unknown method'
+ elif self.type == INVALID_MESSAGE_TYPE:
+ return 'Invalid message type'
+ elif self.type == WRONG_METHOD_NAME:
+ return 'Wrong method name'
+ elif self.type == BAD_SEQUENCE_ID:
+ return 'Bad sequence ID'
+ elif self.type == MISSING_RESULT:
+ return 'Missing result'
+ else:
+ return 'Default (unknown) TApplicationException'
+
+ def read(self, iprot):
+ iprot.readStructBegin()
+ while True:
+ (fname, ftype, fid) = iprot.readFieldBegin()
+ if ftype == TType.STOP:
+ break
+ if fid == 1:
+ if ftype == TType.STRING:
+ self.message = iprot.readString();
+ else:
+ iprot.skip(ftype)
+ elif fid == 2:
+ if ftype == TType.I32:
+ self.type = iprot.readI32();
+ else:
+ iprot.skip(ftype)
+ else:
+ iprot.skip(ftype)
+ iprot.readFieldEnd()
+ iprot.readStructEnd()
+
+ def write(self, oprot):
+ oprot.writeStructBegin('TApplicationException')
+ if self.message != None:
+ oprot.writeFieldBegin('message', TType.STRING, 1)
+ oprot.writeString(self.message)
+ oprot.writeFieldEnd()
+ if self.type != None:
+ oprot.writeFieldBegin('type', TType.I32, 2)
+ oprot.writeI32(self.type)
+ oprot.writeFieldEnd()
+ oprot.writeFieldStop()
+ oprot.writeStructEnd()
diff --git a/lib/py/src/__init__.py b/lib/py/src/__init__.py
new file mode 100644
index 0000000..48d659c
--- /dev/null
+++ b/lib/py/src/__init__.py
@@ -0,0 +1,20 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+__all__ = ['Thrift', 'TSCons']
diff --git a/lib/py/src/protocol/TBinaryProtocol.py b/lib/py/src/protocol/TBinaryProtocol.py
new file mode 100644
index 0000000..db1a7a4
--- /dev/null
+++ b/lib/py/src/protocol/TBinaryProtocol.py
@@ -0,0 +1,259 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+from TProtocol import *
+from struct import pack, unpack
+
+class TBinaryProtocol(TProtocolBase):
+
+ """Binary implementation of the Thrift protocol driver."""
+
+ # NastyHaxx. Python 2.4+ on 32-bit machines forces hex constants to be
+ # positive, converting this into a long. If we hardcode the int value
+ # instead it'll stay in 32 bit-land.
+
+ # VERSION_MASK = 0xffff0000
+ VERSION_MASK = -65536
+
+ # VERSION_1 = 0x80010000
+ VERSION_1 = -2147418112
+
+ TYPE_MASK = 0x000000ff
+
+ def __init__(self, trans, strictRead=False, strictWrite=True):
+ TProtocolBase.__init__(self, trans)
+ self.strictRead = strictRead
+ self.strictWrite = strictWrite
+
+ def writeMessageBegin(self, name, type, seqid):
+ if self.strictWrite:
+ self.writeI32(TBinaryProtocol.VERSION_1 | type)
+ self.writeString(name)
+ self.writeI32(seqid)
+ else:
+ self.writeString(name)
+ self.writeByte(type)
+ self.writeI32(seqid)
+
+ def writeMessageEnd(self):
+ pass
+
+ def writeStructBegin(self, name):
+ pass
+
+ def writeStructEnd(self):
+ pass
+
+ def writeFieldBegin(self, name, type, id):
+ self.writeByte(type)
+ self.writeI16(id)
+
+ def writeFieldEnd(self):
+ pass
+
+ def writeFieldStop(self):
+ self.writeByte(TType.STOP);
+
+ def writeMapBegin(self, ktype, vtype, size):
+ self.writeByte(ktype)
+ self.writeByte(vtype)
+ self.writeI32(size)
+
+ def writeMapEnd(self):
+ pass
+
+ def writeListBegin(self, etype, size):
+ self.writeByte(etype)
+ self.writeI32(size)
+
+ def writeListEnd(self):
+ pass
+
+ def writeSetBegin(self, etype, size):
+ self.writeByte(etype)
+ self.writeI32(size)
+
+ def writeSetEnd(self):
+ pass
+
+ def writeBool(self, bool):
+ if bool:
+ self.writeByte(1)
+ else:
+ self.writeByte(0)
+
+ def writeByte(self, byte):
+ buff = pack("!b", byte)
+ self.trans.write(buff)
+
+ def writeI16(self, i16):
+ buff = pack("!h", i16)
+ self.trans.write(buff)
+
+ def writeI32(self, i32):
+ buff = pack("!i", i32)
+ self.trans.write(buff)
+
+ def writeI64(self, i64):
+ buff = pack("!q", i64)
+ self.trans.write(buff)
+
+ def writeDouble(self, dub):
+ buff = pack("!d", dub)
+ self.trans.write(buff)
+
+ def writeString(self, str):
+ self.writeI32(len(str))
+ self.trans.write(str)
+
+ def readMessageBegin(self):
+ sz = self.readI32()
+ if sz < 0:
+ version = sz & TBinaryProtocol.VERSION_MASK
+ if version != TBinaryProtocol.VERSION_1:
+ raise TProtocolException(TProtocolException.BAD_VERSION, 'Bad version in readMessageBegin: %d' % (sz))
+ type = sz & TBinaryProtocol.TYPE_MASK
+ name = self.readString()
+ seqid = self.readI32()
+ else:
+ if self.strictRead:
+ raise TProtocolException(TProtocolException.BAD_VERSION, 'No protocol version header')
+ name = self.trans.readAll(sz)
+ type = self.readByte()
+ seqid = self.readI32()
+ return (name, type, seqid)
+
+ def readMessageEnd(self):
+ pass
+
+ def readStructBegin(self):
+ pass
+
+ def readStructEnd(self):
+ pass
+
+ def readFieldBegin(self):
+ type = self.readByte()
+ if type == TType.STOP:
+ return (None, type, 0)
+ id = self.readI16()
+ return (None, type, id)
+
+ def readFieldEnd(self):
+ pass
+
+ def readMapBegin(self):
+ ktype = self.readByte()
+ vtype = self.readByte()
+ size = self.readI32()
+ return (ktype, vtype, size)
+
+ def readMapEnd(self):
+ pass
+
+ def readListBegin(self):
+ etype = self.readByte()
+ size = self.readI32()
+ return (etype, size)
+
+ def readListEnd(self):
+ pass
+
+ def readSetBegin(self):
+ etype = self.readByte()
+ size = self.readI32()
+ return (etype, size)
+
+ def readSetEnd(self):
+ pass
+
+ def readBool(self):
+ byte = self.readByte()
+ if byte == 0:
+ return False
+ return True
+
+ def readByte(self):
+ buff = self.trans.readAll(1)
+ val, = unpack('!b', buff)
+ return val
+
+ def readI16(self):
+ buff = self.trans.readAll(2)
+ val, = unpack('!h', buff)
+ return val
+
+ def readI32(self):
+ buff = self.trans.readAll(4)
+ val, = unpack('!i', buff)
+ return val
+
+ def readI64(self):
+ buff = self.trans.readAll(8)
+ val, = unpack('!q', buff)
+ return val
+
+ def readDouble(self):
+ buff = self.trans.readAll(8)
+ val, = unpack('!d', buff)
+ return val
+
+ def readString(self):
+ len = self.readI32()
+ str = self.trans.readAll(len)
+ return str
+
+
+class TBinaryProtocolFactory:
+ def __init__(self, strictRead=False, strictWrite=True):
+ self.strictRead = strictRead
+ self.strictWrite = strictWrite
+
+ def getProtocol(self, trans):
+ prot = TBinaryProtocol(trans, self.strictRead, self.strictWrite)
+ return prot
+
+
+class TBinaryProtocolAccelerated(TBinaryProtocol):
+
+ """C-Accelerated version of TBinaryProtocol.
+
+ This class does not override any of TBinaryProtocol's methods,
+ but the generated code recognizes it directly and will call into
+ our C module to do the encoding, bypassing this object entirely.
+ We inherit from TBinaryProtocol so that the normal TBinaryProtocol
+ encoding can happen if the fastbinary module doesn't work for some
+ reason. (TODO(dreiss): Make this happen sanely in more cases.)
+
+ In order to take advantage of the C module, just use
+ TBinaryProtocolAccelerated instead of TBinaryProtocol.
+
+ NOTE: This code was contributed by an external developer.
+ The internal Thrift team has reviewed and tested it,
+ but we cannot guarantee that it is production-ready.
+ Please feel free to report bugs and/or success stories
+ to the public mailing list.
+ """
+
+ pass
+
+
+class TBinaryProtocolAcceleratedFactory:
+ def getProtocol(self, trans):
+ return TBinaryProtocolAccelerated(trans)
diff --git a/lib/py/src/protocol/TProtocol.py b/lib/py/src/protocol/TProtocol.py
new file mode 100644
index 0000000..be3cb14
--- /dev/null
+++ b/lib/py/src/protocol/TProtocol.py
@@ -0,0 +1,205 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+from thrift.Thrift import *
+
+class TProtocolException(TException):
+
+ """Custom Protocol Exception class"""
+
+ UNKNOWN = 0
+ INVALID_DATA = 1
+ NEGATIVE_SIZE = 2
+ SIZE_LIMIT = 3
+ BAD_VERSION = 4
+
+ def __init__(self, type=UNKNOWN, message=None):
+ TException.__init__(self, message)
+ self.type = type
+
+class TProtocolBase:
+
+ """Base class for Thrift protocol driver."""
+
+ def __init__(self, trans):
+ self.trans = trans
+
+ def writeMessageBegin(self, name, type, seqid):
+ pass
+
+ def writeMessageEnd(self):
+ pass
+
+ def writeStructBegin(self, name):
+ pass
+
+ def writeStructEnd(self):
+ pass
+
+ def writeFieldBegin(self, name, type, id):
+ pass
+
+ def writeFieldEnd(self):
+ pass
+
+ def writeFieldStop(self):
+ pass
+
+ def writeMapBegin(self, ktype, vtype, size):
+ pass
+
+ def writeMapEnd(self):
+ pass
+
+ def writeListBegin(self, etype, size):
+ pass
+
+ def writeListEnd(self):
+ pass
+
+ def writeSetBegin(self, etype, size):
+ pass
+
+ def writeSetEnd(self):
+ pass
+
+ def writeBool(self, bool):
+ pass
+
+ def writeByte(self, byte):
+ pass
+
+ def writeI16(self, i16):
+ pass
+
+ def writeI32(self, i32):
+ pass
+
+ def writeI64(self, i64):
+ pass
+
+ def writeDouble(self, dub):
+ pass
+
+ def writeString(self, str):
+ pass
+
+ def readMessageBegin(self):
+ pass
+
+ def readMessageEnd(self):
+ pass
+
+ def readStructBegin(self):
+ pass
+
+ def readStructEnd(self):
+ pass
+
+ def readFieldBegin(self):
+ pass
+
+ def readFieldEnd(self):
+ pass
+
+ def readMapBegin(self):
+ pass
+
+ def readMapEnd(self):
+ pass
+
+ def readListBegin(self):
+ pass
+
+ def readListEnd(self):
+ pass
+
+ def readSetBegin(self):
+ pass
+
+ def readSetEnd(self):
+ pass
+
+ def readBool(self):
+ pass
+
+ def readByte(self):
+ pass
+
+ def readI16(self):
+ pass
+
+ def readI32(self):
+ pass
+
+ def readI64(self):
+ pass
+
+ def readDouble(self):
+ pass
+
+ def readString(self):
+ pass
+
+ def skip(self, type):
+ if type == TType.STOP:
+ return
+ elif type == TType.BOOL:
+ self.readBool()
+ elif type == TType.BYTE:
+ self.readByte()
+ elif type == TType.I16:
+ self.readI16()
+ elif type == TType.I32:
+ self.readI32()
+ elif type == TType.I64:
+ self.readI64()
+ elif type == TType.DOUBLE:
+ self.readDouble()
+ elif type == TType.STRING:
+ self.readString()
+ elif type == TType.STRUCT:
+ name = self.readStructBegin()
+ while True:
+ (name, type, id) = self.readFieldBegin()
+ if type == TType.STOP:
+ break
+ self.skip(type)
+ self.readFieldEnd()
+ self.readStructEnd()
+ elif type == TType.MAP:
+ (ktype, vtype, size) = self.readMapBegin()
+ for i in range(size):
+ self.skip(ktype)
+ self.skip(vtype)
+ self.readMapEnd()
+ elif type == TType.SET:
+ (etype, size) = self.readSetBegin()
+ for i in range(size):
+ self.skip(etype)
+ self.readSetEnd()
+ elif type == TType.LIST:
+ (etype, size) = self.readListBegin()
+ for i in range(size):
+ self.skip(etype)
+ self.readListEnd()
+
+class TProtocolFactory:
+ def getProtocol(self, trans):
+ pass
diff --git a/lib/py/src/protocol/__init__.py b/lib/py/src/protocol/__init__.py
new file mode 100644
index 0000000..01bfe18
--- /dev/null
+++ b/lib/py/src/protocol/__init__.py
@@ -0,0 +1,20 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+__all__ = ['TProtocol', 'TBinaryProtocol', 'fastbinary']
diff --git a/lib/py/src/protocol/fastbinary.c b/lib/py/src/protocol/fastbinary.c
new file mode 100644
index 0000000..67b215a
--- /dev/null
+++ b/lib/py/src/protocol/fastbinary.c
@@ -0,0 +1,1203 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include <Python.h>
+#include "cStringIO.h"
+#include <stdbool.h>
+#include <stdint.h>
+#include <netinet/in.h>
+
+/* Fix endianness issues on Solaris */
+#if defined (__SVR4) && defined (__sun)
+ #if defined(__i386) && !defined(__i386__)
+ #define __i386__
+ #endif
+
+ #ifndef BIG_ENDIAN
+ #define BIG_ENDIAN (4321)
+ #endif
+ #ifndef LITTLE_ENDIAN
+ #define LITTLE_ENDIAN (1234)
+ #endif
+
+ /* I386 is LE, even on Solaris */
+ #if !defined(BYTE_ORDER) && defined(__i386__)
+ #define BYTE_ORDER LITTLE_ENDIAN
+ #endif
+#endif
+
+// TODO(dreiss): defval appears to be unused. Look into removing it.
+// TODO(dreiss): Make parse_spec_args recursive, and cache the output
+// permanently in the object. (Malloc and orphan.)
+// TODO(dreiss): Why do we need cStringIO for reading, why not just char*?
+// Can cStringIO let us work with a BufferedTransport?
+// TODO(dreiss): Don't ignore the rv from cwrite (maybe).
+
+/* ====== BEGIN UTILITIES ====== */
+
+#define INIT_OUTBUF_SIZE 128
+
+// Stolen out of TProtocol.h.
+// It would be a huge pain to have both get this from one place.
+typedef enum TType {
+ T_STOP = 0,
+ T_VOID = 1,
+ T_BOOL = 2,
+ T_BYTE = 3,
+ T_I08 = 3,
+ T_I16 = 6,
+ T_I32 = 8,
+ T_U64 = 9,
+ T_I64 = 10,
+ T_DOUBLE = 4,
+ T_STRING = 11,
+ T_UTF7 = 11,
+ T_STRUCT = 12,
+ T_MAP = 13,
+ T_SET = 14,
+ T_LIST = 15,
+ T_UTF8 = 16,
+ T_UTF16 = 17
+} TType;
+
+#ifndef __BYTE_ORDER
+# if defined(BYTE_ORDER) && defined(LITTLE_ENDIAN) && defined(BIG_ENDIAN)
+# define __BYTE_ORDER BYTE_ORDER
+# define __LITTLE_ENDIAN LITTLE_ENDIAN
+# define __BIG_ENDIAN BIG_ENDIAN
+# else
+# error "Cannot determine endianness"
+# endif
+#endif
+
+// Same comment as the enum. Sorry.
+#if __BYTE_ORDER == __BIG_ENDIAN
+# define ntohll(n) (n)
+# define htonll(n) (n)
+#elif __BYTE_ORDER == __LITTLE_ENDIAN
+# if defined(__GNUC__) && defined(__GLIBC__)
+# include <byteswap.h>
+# define ntohll(n) bswap_64(n)
+# define htonll(n) bswap_64(n)
+# else /* GNUC & GLIBC */
+# define ntohll(n) ( (((unsigned long long)ntohl(n)) << 32) + ntohl(n >> 32) )
+# define htonll(n) ( (((unsigned long long)htonl(n)) << 32) + htonl(n >> 32) )
+# endif /* GNUC & GLIBC */
+#else /* __BYTE_ORDER */
+# error "Can't define htonll or ntohll!"
+#endif
+
+// Doing a benchmark shows that interning actually makes a difference, amazingly.
+#define INTERN_STRING(value) _intern_ ## value
+
+#define INT_CONV_ERROR_OCCURRED(v) ( ((v) == -1) && PyErr_Occurred() )
+#define CHECK_RANGE(v, min, max) ( ((v) <= (max)) && ((v) >= (min)) )
+
+// Py_ssize_t was not defined before Python 2.5
+#if (PY_VERSION_HEX < 0x02050000)
+typedef int Py_ssize_t;
+#endif
+
+/**
+ * A cache of the spec_args for a set or list,
+ * so we don't have to keep calling PyTuple_GET_ITEM.
+ */
+typedef struct {
+ TType element_type;
+ PyObject* typeargs;
+} SetListTypeArgs;
+
+/**
+ * A cache of the spec_args for a map,
+ * so we don't have to keep calling PyTuple_GET_ITEM.
+ */
+typedef struct {
+ TType ktag;
+ TType vtag;
+ PyObject* ktypeargs;
+ PyObject* vtypeargs;
+} MapTypeArgs;
+
+/**
+ * A cache of the spec_args for a struct,
+ * so we don't have to keep calling PyTuple_GET_ITEM.
+ */
+typedef struct {
+ PyObject* klass;
+ PyObject* spec;
+} StructTypeArgs;
+
+/**
+ * A cache of the item spec from a struct specification,
+ * so we don't have to keep calling PyTuple_GET_ITEM.
+ */
+typedef struct {
+ int tag;
+ TType type;
+ PyObject* attrname;
+ PyObject* typeargs;
+ PyObject* defval;
+} StructItemSpec;
+
+/**
+ * A cache of the two key attributes of a CReadableTransport,
+ * so we don't have to keep calling PyObject_GetAttr.
+ */
+typedef struct {
+ PyObject* stringiobuf;
+ PyObject* refill_callable;
+} DecodeBuffer;
+
+/** Pointer to interned string to speed up attribute lookup. */
+static PyObject* INTERN_STRING(cstringio_buf);
+/** Pointer to interned string to speed up attribute lookup. */
+static PyObject* INTERN_STRING(cstringio_refill);
+
+static inline bool
+check_ssize_t_32(Py_ssize_t len) {
+ // error from getting the int
+ if (INT_CONV_ERROR_OCCURRED(len)) {
+ return false;
+ }
+ if (!CHECK_RANGE(len, 0, INT32_MAX)) {
+ PyErr_SetString(PyExc_OverflowError, "string size out of range");
+ return false;
+ }
+ return true;
+}
+
+static inline bool
+parse_pyint(PyObject* o, int32_t* ret, int32_t min, int32_t max) {
+ long val = PyInt_AsLong(o);
+
+ if (INT_CONV_ERROR_OCCURRED(val)) {
+ return false;
+ }
+ if (!CHECK_RANGE(val, min, max)) {
+ PyErr_SetString(PyExc_OverflowError, "int out of range");
+ return false;
+ }
+
+ *ret = (int32_t) val;
+ return true;
+}
+
+
+/* --- FUNCTIONS TO PARSE STRUCT SPECIFICATOINS --- */
+
+static bool
+parse_set_list_args(SetListTypeArgs* dest, PyObject* typeargs) {
+ if (PyTuple_Size(typeargs) != 2) {
+ PyErr_SetString(PyExc_TypeError, "expecting tuple of size 2 for list/set type args");
+ return false;
+ }
+
+ dest->element_type = PyInt_AsLong(PyTuple_GET_ITEM(typeargs, 0));
+ if (INT_CONV_ERROR_OCCURRED(dest->element_type)) {
+ return false;
+ }
+
+ dest->typeargs = PyTuple_GET_ITEM(typeargs, 1);
+
+ return true;
+}
+
+static bool
+parse_map_args(MapTypeArgs* dest, PyObject* typeargs) {
+ if (PyTuple_Size(typeargs) != 4) {
+ PyErr_SetString(PyExc_TypeError, "expecting 4 arguments for typeargs to map");
+ return false;
+ }
+
+ dest->ktag = PyInt_AsLong(PyTuple_GET_ITEM(typeargs, 0));
+ if (INT_CONV_ERROR_OCCURRED(dest->ktag)) {
+ return false;
+ }
+
+ dest->vtag = PyInt_AsLong(PyTuple_GET_ITEM(typeargs, 2));
+ if (INT_CONV_ERROR_OCCURRED(dest->vtag)) {
+ return false;
+ }
+
+ dest->ktypeargs = PyTuple_GET_ITEM(typeargs, 1);
+ dest->vtypeargs = PyTuple_GET_ITEM(typeargs, 3);
+
+ return true;
+}
+
+static bool
+parse_struct_args(StructTypeArgs* dest, PyObject* typeargs) {
+ if (PyTuple_Size(typeargs) != 2) {
+ PyErr_SetString(PyExc_TypeError, "expecting tuple of size 2 for struct args");
+ return false;
+ }
+
+ dest->klass = PyTuple_GET_ITEM(typeargs, 0);
+ dest->spec = PyTuple_GET_ITEM(typeargs, 1);
+
+ return true;
+}
+
+static int
+parse_struct_item_spec(StructItemSpec* dest, PyObject* spec_tuple) {
+
+ // i'd like to use ParseArgs here, but it seems to be a bottleneck.
+ if (PyTuple_Size(spec_tuple) != 5) {
+ PyErr_SetString(PyExc_TypeError, "expecting 5 arguments for spec tuple");
+ return false;
+ }
+
+ dest->tag = PyInt_AsLong(PyTuple_GET_ITEM(spec_tuple, 0));
+ if (INT_CONV_ERROR_OCCURRED(dest->tag)) {
+ return false;
+ }
+
+ dest->type = PyInt_AsLong(PyTuple_GET_ITEM(spec_tuple, 1));
+ if (INT_CONV_ERROR_OCCURRED(dest->type)) {
+ return false;
+ }
+
+ dest->attrname = PyTuple_GET_ITEM(spec_tuple, 2);
+ dest->typeargs = PyTuple_GET_ITEM(spec_tuple, 3);
+ dest->defval = PyTuple_GET_ITEM(spec_tuple, 4);
+ return true;
+}
+
+/* ====== END UTILITIES ====== */
+
+
+/* ====== BEGIN WRITING FUNCTIONS ====== */
+
+/* --- LOW-LEVEL WRITING FUNCTIONS --- */
+
+static void writeByte(PyObject* outbuf, int8_t val) {
+ int8_t net = val;
+ PycStringIO->cwrite(outbuf, (char*)&net, sizeof(int8_t));
+}
+
+static void writeI16(PyObject* outbuf, int16_t val) {
+ int16_t net = (int16_t)htons(val);
+ PycStringIO->cwrite(outbuf, (char*)&net, sizeof(int16_t));
+}
+
+static void writeI32(PyObject* outbuf, int32_t val) {
+ int32_t net = (int32_t)htonl(val);
+ PycStringIO->cwrite(outbuf, (char*)&net, sizeof(int32_t));
+}
+
+static void writeI64(PyObject* outbuf, int64_t val) {
+ int64_t net = (int64_t)htonll(val);
+ PycStringIO->cwrite(outbuf, (char*)&net, sizeof(int64_t));
+}
+
+static void writeDouble(PyObject* outbuf, double dub) {
+ // Unfortunately, bitwise_cast doesn't work in C. Bad C!
+ union {
+ double f;
+ int64_t t;
+ } transfer;
+ transfer.f = dub;
+ writeI64(outbuf, transfer.t);
+}
+
+
+/* --- MAIN RECURSIVE OUTPUT FUCNTION -- */
+
+static int
+output_val(PyObject* output, PyObject* value, TType type, PyObject* typeargs) {
+ /*
+ * Refcounting Strategy:
+ *
+ * We assume that elements of the thrift_spec tuple are not going to be
+ * mutated, so we don't ref count those at all. Other than that, we try to
+ * keep a reference to all the user-created objects while we work with them.
+ * output_val assumes that a reference is already held. The *caller* is
+ * responsible for handling references
+ */
+
+ switch (type) {
+
+ case T_BOOL: {
+ int v = PyObject_IsTrue(value);
+ if (v == -1) {
+ return false;
+ }
+
+ writeByte(output, (int8_t) v);
+ break;
+ }
+ case T_I08: {
+ int32_t val;
+
+ if (!parse_pyint(value, &val, INT8_MIN, INT8_MAX)) {
+ return false;
+ }
+
+ writeByte(output, (int8_t) val);
+ break;
+ }
+ case T_I16: {
+ int32_t val;
+
+ if (!parse_pyint(value, &val, INT16_MIN, INT16_MAX)) {
+ return false;
+ }
+
+ writeI16(output, (int16_t) val);
+ break;
+ }
+ case T_I32: {
+ int32_t val;
+
+ if (!parse_pyint(value, &val, INT32_MIN, INT32_MAX)) {
+ return false;
+ }
+
+ writeI32(output, val);
+ break;
+ }
+ case T_I64: {
+ int64_t nval = PyLong_AsLongLong(value);
+
+ if (INT_CONV_ERROR_OCCURRED(nval)) {
+ return false;
+ }
+
+ if (!CHECK_RANGE(nval, INT64_MIN, INT64_MAX)) {
+ PyErr_SetString(PyExc_OverflowError, "int out of range");
+ return false;
+ }
+
+ writeI64(output, nval);
+ break;
+ }
+
+ case T_DOUBLE: {
+ double nval = PyFloat_AsDouble(value);
+ if (nval == -1.0 && PyErr_Occurred()) {
+ return false;
+ }
+
+ writeDouble(output, nval);
+ break;
+ }
+
+ case T_STRING: {
+ Py_ssize_t len = PyString_Size(value);
+
+ if (!check_ssize_t_32(len)) {
+ return false;
+ }
+
+ writeI32(output, (int32_t) len);
+ PycStringIO->cwrite(output, PyString_AsString(value), (int32_t) len);
+ break;
+ }
+
+ case T_LIST:
+ case T_SET: {
+ Py_ssize_t len;
+ SetListTypeArgs parsedargs;
+ PyObject *item;
+ PyObject *iterator;
+
+ if (!parse_set_list_args(&parsedargs, typeargs)) {
+ return false;
+ }
+
+ len = PyObject_Length(value);
+
+ if (!check_ssize_t_32(len)) {
+ return false;
+ }
+
+ writeByte(output, parsedargs.element_type);
+ writeI32(output, (int32_t) len);
+
+ iterator = PyObject_GetIter(value);
+ if (iterator == NULL) {
+ return false;
+ }
+
+ while ((item = PyIter_Next(iterator))) {
+ if (!output_val(output, item, parsedargs.element_type, parsedargs.typeargs)) {
+ Py_DECREF(item);
+ Py_DECREF(iterator);
+ return false;
+ }
+ Py_DECREF(item);
+ }
+
+ Py_DECREF(iterator);
+
+ if (PyErr_Occurred()) {
+ return false;
+ }
+
+ break;
+ }
+
+ case T_MAP: {
+ PyObject *k, *v;
+ Py_ssize_t pos = 0;
+ Py_ssize_t len;
+
+ MapTypeArgs parsedargs;
+
+ len = PyDict_Size(value);
+ if (!check_ssize_t_32(len)) {
+ return false;
+ }
+
+ if (!parse_map_args(&parsedargs, typeargs)) {
+ return false;
+ }
+
+ writeByte(output, parsedargs.ktag);
+ writeByte(output, parsedargs.vtag);
+ writeI32(output, len);
+
+ // TODO(bmaurer): should support any mapping, not just dicts
+ while (PyDict_Next(value, &pos, &k, &v)) {
+ // TODO(dreiss): Think hard about whether these INCREFs actually
+ // turn any unsafe scenarios into safe scenarios.
+ Py_INCREF(k);
+ Py_INCREF(v);
+
+ if (!output_val(output, k, parsedargs.ktag, parsedargs.ktypeargs)
+ || !output_val(output, v, parsedargs.vtag, parsedargs.vtypeargs)) {
+ Py_DECREF(k);
+ Py_DECREF(v);
+ return false;
+ }
+ Py_DECREF(k);
+ Py_DECREF(v);
+ }
+ break;
+ }
+
+ // TODO(dreiss): Consider breaking this out as a function
+ // the way we did for decode_struct.
+ case T_STRUCT: {
+ StructTypeArgs parsedargs;
+ Py_ssize_t nspec;
+ Py_ssize_t i;
+
+ if (!parse_struct_args(&parsedargs, typeargs)) {
+ return false;
+ }
+
+ nspec = PyTuple_Size(parsedargs.spec);
+
+ if (nspec == -1) {
+ return false;
+ }
+
+ for (i = 0; i < nspec; i++) {
+ StructItemSpec parsedspec;
+ PyObject* spec_tuple;
+ PyObject* instval = NULL;
+
+ spec_tuple = PyTuple_GET_ITEM(parsedargs.spec, i);
+ if (spec_tuple == Py_None) {
+ continue;
+ }
+
+ if (!parse_struct_item_spec (&parsedspec, spec_tuple)) {
+ return false;
+ }
+
+ instval = PyObject_GetAttr(value, parsedspec.attrname);
+
+ if (!instval) {
+ return false;
+ }
+
+ if (instval == Py_None) {
+ Py_DECREF(instval);
+ continue;
+ }
+
+ writeByte(output, (int8_t) parsedspec.type);
+ writeI16(output, parsedspec.tag);
+
+ if (!output_val(output, instval, parsedspec.type, parsedspec.typeargs)) {
+ Py_DECREF(instval);
+ return false;
+ }
+
+ Py_DECREF(instval);
+ }
+
+ writeByte(output, (int8_t)T_STOP);
+ break;
+ }
+
+ case T_STOP:
+ case T_VOID:
+ case T_UTF16:
+ case T_UTF8:
+ case T_U64:
+ default:
+ PyErr_SetString(PyExc_TypeError, "Unexpected TType");
+ return false;
+
+ }
+
+ return true;
+}
+
+
+/* --- TOP-LEVEL WRAPPER FOR OUTPUT -- */
+
+static PyObject *
+encode_binary(PyObject *self, PyObject *args) {
+ PyObject* enc_obj;
+ PyObject* type_args;
+ PyObject* buf;
+ PyObject* ret = NULL;
+
+ if (!PyArg_ParseTuple(args, "OO", &enc_obj, &type_args)) {
+ return NULL;
+ }
+
+ buf = PycStringIO->NewOutput(INIT_OUTBUF_SIZE);
+ if (output_val(buf, enc_obj, T_STRUCT, type_args)) {
+ ret = PycStringIO->cgetvalue(buf);
+ }
+
+ Py_DECREF(buf);
+ return ret;
+}
+
+/* ====== END WRITING FUNCTIONS ====== */
+
+
+/* ====== BEGIN READING FUNCTIONS ====== */
+
+/* --- LOW-LEVEL READING FUNCTIONS --- */
+
+static void
+free_decodebuf(DecodeBuffer* d) {
+ Py_XDECREF(d->stringiobuf);
+ Py_XDECREF(d->refill_callable);
+}
+
+static bool
+decode_buffer_from_obj(DecodeBuffer* dest, PyObject* obj) {
+ dest->stringiobuf = PyObject_GetAttr(obj, INTERN_STRING(cstringio_buf));
+ if (!dest->stringiobuf) {
+ return false;
+ }
+
+ if (!PycStringIO_InputCheck(dest->stringiobuf)) {
+ free_decodebuf(dest);
+ PyErr_SetString(PyExc_TypeError, "expecting stringio input");
+ return false;
+ }
+
+ dest->refill_callable = PyObject_GetAttr(obj, INTERN_STRING(cstringio_refill));
+
+ if(!dest->refill_callable) {
+ free_decodebuf(dest);
+ return false;
+ }
+
+ if (!PyCallable_Check(dest->refill_callable)) {
+ free_decodebuf(dest);
+ PyErr_SetString(PyExc_TypeError, "expecting callable");
+ return false;
+ }
+
+ return true;
+}
+
+static bool readBytes(DecodeBuffer* input, char** output, int len) {
+ int read;
+
+ // TODO(dreiss): Don't fear the malloc. Think about taking a copy of
+ // the partial read instead of forcing the transport
+ // to prepend it to its buffer.
+
+ read = PycStringIO->cread(input->stringiobuf, output, len);
+
+ if (read == len) {
+ return true;
+ } else if (read == -1) {
+ return false;
+ } else {
+ PyObject* newiobuf;
+
+ // using building functions as this is a rare codepath
+ newiobuf = PyObject_CallFunction(
+ input->refill_callable, "s#i", *output, read, len, NULL);
+ if (newiobuf == NULL) {
+ return false;
+ }
+
+ // must do this *AFTER* the call so that we don't deref the io buffer
+ Py_CLEAR(input->stringiobuf);
+ input->stringiobuf = newiobuf;
+
+ read = PycStringIO->cread(input->stringiobuf, output, len);
+
+ if (read == len) {
+ return true;
+ } else if (read == -1) {
+ return false;
+ } else {
+ // TODO(dreiss): This could be a valid code path for big binary blobs.
+ PyErr_SetString(PyExc_TypeError,
+ "refill claimed to have refilled the buffer, but didn't!!");
+ return false;
+ }
+ }
+}
+
+static int8_t readByte(DecodeBuffer* input) {
+ char* buf;
+ if (!readBytes(input, &buf, sizeof(int8_t))) {
+ return -1;
+ }
+
+ return *(int8_t*) buf;
+}
+
+static int16_t readI16(DecodeBuffer* input) {
+ char* buf;
+ if (!readBytes(input, &buf, sizeof(int16_t))) {
+ return -1;
+ }
+
+ return (int16_t) ntohs(*(int16_t*) buf);
+}
+
+static int32_t readI32(DecodeBuffer* input) {
+ char* buf;
+ if (!readBytes(input, &buf, sizeof(int32_t))) {
+ return -1;
+ }
+ return (int32_t) ntohl(*(int32_t*) buf);
+}
+
+
+static int64_t readI64(DecodeBuffer* input) {
+ char* buf;
+ if (!readBytes(input, &buf, sizeof(int64_t))) {
+ return -1;
+ }
+
+ return (int64_t) ntohll(*(int64_t*) buf);
+}
+
+static double readDouble(DecodeBuffer* input) {
+ union {
+ int64_t f;
+ double t;
+ } transfer;
+
+ transfer.f = readI64(input);
+ if (transfer.f == -1) {
+ return -1;
+ }
+ return transfer.t;
+}
+
+static bool
+checkTypeByte(DecodeBuffer* input, TType expected) {
+ TType got = readByte(input);
+ if (INT_CONV_ERROR_OCCURRED(got)) {
+ return false;
+ }
+
+ if (expected != got) {
+ PyErr_SetString(PyExc_TypeError, "got wrong ttype while reading field");
+ return false;
+ }
+ return true;
+}
+
+static bool
+skip(DecodeBuffer* input, TType type) {
+#define SKIPBYTES(n) \
+ do { \
+ if (!readBytes(input, &dummy_buf, (n))) { \
+ return false; \
+ } \
+ } while(0)
+
+ char* dummy_buf;
+
+ switch (type) {
+
+ case T_BOOL:
+ case T_I08: SKIPBYTES(1); break;
+ case T_I16: SKIPBYTES(2); break;
+ case T_I32: SKIPBYTES(4); break;
+ case T_I64:
+ case T_DOUBLE: SKIPBYTES(8); break;
+
+ case T_STRING: {
+ // TODO(dreiss): Find out if these check_ssize_t32s are really necessary.
+ int len = readI32(input);
+ if (!check_ssize_t_32(len)) {
+ return false;
+ }
+ SKIPBYTES(len);
+ break;
+ }
+
+ case T_LIST:
+ case T_SET: {
+ TType etype;
+ int len, i;
+
+ etype = readByte(input);
+ if (etype == -1) {
+ return false;
+ }
+
+ len = readI32(input);
+ if (!check_ssize_t_32(len)) {
+ return false;
+ }
+
+ for (i = 0; i < len; i++) {
+ if (!skip(input, etype)) {
+ return false;
+ }
+ }
+ break;
+ }
+
+ case T_MAP: {
+ TType ktype, vtype;
+ int len, i;
+
+ ktype = readByte(input);
+ if (ktype == -1) {
+ return false;
+ }
+
+ vtype = readByte(input);
+ if (vtype == -1) {
+ return false;
+ }
+
+ len = readI32(input);
+ if (!check_ssize_t_32(len)) {
+ return false;
+ }
+
+ for (i = 0; i < len; i++) {
+ if (!(skip(input, ktype) && skip(input, vtype))) {
+ return false;
+ }
+ }
+ break;
+ }
+
+ case T_STRUCT: {
+ while (true) {
+ TType type;
+
+ type = readByte(input);
+ if (type == -1) {
+ return false;
+ }
+
+ if (type == T_STOP)
+ break;
+
+ SKIPBYTES(2); // tag
+ if (!skip(input, type)) {
+ return false;
+ }
+ }
+ break;
+ }
+
+ case T_STOP:
+ case T_VOID:
+ case T_UTF16:
+ case T_UTF8:
+ case T_U64:
+ default:
+ PyErr_SetString(PyExc_TypeError, "Unexpected TType");
+ return false;
+
+ }
+
+ return true;
+
+#undef SKIPBYTES
+}
+
+
+/* --- HELPER FUNCTION FOR DECODE_VAL --- */
+
+static PyObject*
+decode_val(DecodeBuffer* input, TType type, PyObject* typeargs);
+
+static bool
+decode_struct(DecodeBuffer* input, PyObject* output, PyObject* spec_seq) {
+ int spec_seq_len = PyTuple_Size(spec_seq);
+ if (spec_seq_len == -1) {
+ return false;
+ }
+
+ while (true) {
+ TType type;
+ int16_t tag;
+ PyObject* item_spec;
+ PyObject* fieldval = NULL;
+ StructItemSpec parsedspec;
+
+ type = readByte(input);
+ if (type == -1) {
+ return false;
+ }
+ if (type == T_STOP) {
+ break;
+ }
+ tag = readI16(input);
+ if (INT_CONV_ERROR_OCCURRED(tag)) {
+ return false;
+ }
+ if (tag >= 0 && tag < spec_seq_len) {
+ item_spec = PyTuple_GET_ITEM(spec_seq, tag);
+ } else {
+ item_spec = Py_None;
+ }
+
+ if (item_spec == Py_None) {
+ if (!skip(input, type)) {
+ return false;
+ } else {
+ continue;
+ }
+ }
+
+ if (!parse_struct_item_spec(&parsedspec, item_spec)) {
+ return false;
+ }
+ if (parsedspec.type != type) {
+ if (!skip(input, type)) {
+ PyErr_SetString(PyExc_TypeError, "struct field had wrong type while reading and can't be skipped");
+ return false;
+ } else {
+ continue;
+ }
+ }
+
+ fieldval = decode_val(input, parsedspec.type, parsedspec.typeargs);
+ if (fieldval == NULL) {
+ return false;
+ }
+
+ if (PyObject_SetAttr(output, parsedspec.attrname, fieldval) == -1) {
+ Py_DECREF(fieldval);
+ return false;
+ }
+ Py_DECREF(fieldval);
+ }
+ return true;
+}
+
+
+/* --- MAIN RECURSIVE INPUT FUCNTION --- */
+
+// Returns a new reference.
+static PyObject*
+decode_val(DecodeBuffer* input, TType type, PyObject* typeargs) {
+ switch (type) {
+
+ case T_BOOL: {
+ int8_t v = readByte(input);
+ if (INT_CONV_ERROR_OCCURRED(v)) {
+ return NULL;
+ }
+
+ switch (v) {
+ case 0: Py_RETURN_FALSE;
+ case 1: Py_RETURN_TRUE;
+ // Don't laugh. This is a potentially serious issue.
+ default: PyErr_SetString(PyExc_TypeError, "boolean out of range"); return NULL;
+ }
+ break;
+ }
+ case T_I08: {
+ int8_t v = readByte(input);
+ if (INT_CONV_ERROR_OCCURRED(v)) {
+ return NULL;
+ }
+
+ return PyInt_FromLong(v);
+ }
+ case T_I16: {
+ int16_t v = readI16(input);
+ if (INT_CONV_ERROR_OCCURRED(v)) {
+ return NULL;
+ }
+ return PyInt_FromLong(v);
+ }
+ case T_I32: {
+ int32_t v = readI32(input);
+ if (INT_CONV_ERROR_OCCURRED(v)) {
+ return NULL;
+ }
+ return PyInt_FromLong(v);
+ }
+
+ case T_I64: {
+ int64_t v = readI64(input);
+ if (INT_CONV_ERROR_OCCURRED(v)) {
+ return NULL;
+ }
+ // TODO(dreiss): Find out if we can take this fastpath always when
+ // sizeof(long) == sizeof(long long).
+ if (CHECK_RANGE(v, LONG_MIN, LONG_MAX)) {
+ return PyInt_FromLong((long) v);
+ }
+
+ return PyLong_FromLongLong(v);
+ }
+
+ case T_DOUBLE: {
+ double v = readDouble(input);
+ if (v == -1.0 && PyErr_Occurred()) {
+ return false;
+ }
+ return PyFloat_FromDouble(v);
+ }
+
+ case T_STRING: {
+ Py_ssize_t len = readI32(input);
+ char* buf;
+ if (!readBytes(input, &buf, len)) {
+ return NULL;
+ }
+
+ return PyString_FromStringAndSize(buf, len);
+ }
+
+ case T_LIST:
+ case T_SET: {
+ SetListTypeArgs parsedargs;
+ int32_t len;
+ PyObject* ret = NULL;
+ int i;
+
+ if (!parse_set_list_args(&parsedargs, typeargs)) {
+ return NULL;
+ }
+
+ if (!checkTypeByte(input, parsedargs.element_type)) {
+ return NULL;
+ }
+
+ len = readI32(input);
+ if (!check_ssize_t_32(len)) {
+ return NULL;
+ }
+
+ ret = PyList_New(len);
+ if (!ret) {
+ return NULL;
+ }
+
+ for (i = 0; i < len; i++) {
+ PyObject* item = decode_val(input, parsedargs.element_type, parsedargs.typeargs);
+ if (!item) {
+ Py_DECREF(ret);
+ return NULL;
+ }
+ PyList_SET_ITEM(ret, i, item);
+ }
+
+ // TODO(dreiss): Consider biting the bullet and making two separate cases
+ // for list and set, avoiding this post facto conversion.
+ if (type == T_SET) {
+ PyObject* setret;
+#if (PY_VERSION_HEX < 0x02050000)
+ // hack needed for older versions
+ setret = PyObject_CallFunctionObjArgs((PyObject*)&PySet_Type, ret, NULL);
+#else
+ // official version
+ setret = PySet_New(ret);
+#endif
+ Py_DECREF(ret);
+ return setret;
+ }
+ return ret;
+ }
+
+ case T_MAP: {
+ int32_t len;
+ int i;
+ MapTypeArgs parsedargs;
+ PyObject* ret = NULL;
+
+ if (!parse_map_args(&parsedargs, typeargs)) {
+ return NULL;
+ }
+
+ if (!checkTypeByte(input, parsedargs.ktag)) {
+ return NULL;
+ }
+ if (!checkTypeByte(input, parsedargs.vtag)) {
+ return NULL;
+ }
+
+ len = readI32(input);
+ if (!check_ssize_t_32(len)) {
+ return false;
+ }
+
+ ret = PyDict_New();
+ if (!ret) {
+ goto error;
+ }
+
+ for (i = 0; i < len; i++) {
+ PyObject* k = NULL;
+ PyObject* v = NULL;
+ k = decode_val(input, parsedargs.ktag, parsedargs.ktypeargs);
+ if (k == NULL) {
+ goto loop_error;
+ }
+ v = decode_val(input, parsedargs.vtag, parsedargs.vtypeargs);
+ if (v == NULL) {
+ goto loop_error;
+ }
+ if (PyDict_SetItem(ret, k, v) == -1) {
+ goto loop_error;
+ }
+
+ Py_DECREF(k);
+ Py_DECREF(v);
+ continue;
+
+ // Yuck! Destructors, anyone?
+ loop_error:
+ Py_XDECREF(k);
+ Py_XDECREF(v);
+ goto error;
+ }
+
+ return ret;
+
+ error:
+ Py_XDECREF(ret);
+ return NULL;
+ }
+
+ case T_STRUCT: {
+ StructTypeArgs parsedargs;
+ if (!parse_struct_args(&parsedargs, typeargs)) {
+ return NULL;
+ }
+
+ PyObject* ret = PyObject_CallObject(parsedargs.klass, NULL);
+ if (!ret) {
+ return NULL;
+ }
+
+ if (!decode_struct(input, ret, parsedargs.spec)) {
+ Py_DECREF(ret);
+ return NULL;
+ }
+
+ return ret;
+ }
+
+ case T_STOP:
+ case T_VOID:
+ case T_UTF16:
+ case T_UTF8:
+ case T_U64:
+ default:
+ PyErr_SetString(PyExc_TypeError, "Unexpected TType");
+ return NULL;
+ }
+}
+
+
+/* --- TOP-LEVEL WRAPPER FOR INPUT -- */
+
+static PyObject*
+decode_binary(PyObject *self, PyObject *args) {
+ PyObject* output_obj = NULL;
+ PyObject* transport = NULL;
+ PyObject* typeargs = NULL;
+ StructTypeArgs parsedargs;
+ DecodeBuffer input = {};
+
+ if (!PyArg_ParseTuple(args, "OOO", &output_obj, &transport, &typeargs)) {
+ return NULL;
+ }
+
+ if (!parse_struct_args(&parsedargs, typeargs)) {
+ return NULL;
+ }
+
+ if (!decode_buffer_from_obj(&input, transport)) {
+ return NULL;
+ }
+
+ if (!decode_struct(&input, output_obj, parsedargs.spec)) {
+ free_decodebuf(&input);
+ return NULL;
+ }
+
+ free_decodebuf(&input);
+
+ Py_RETURN_NONE;
+}
+
+/* ====== END READING FUNCTIONS ====== */
+
+
+/* -- PYTHON MODULE SETUP STUFF --- */
+
+static PyMethodDef ThriftFastBinaryMethods[] = {
+
+ {"encode_binary", encode_binary, METH_VARARGS, ""},
+ {"decode_binary", decode_binary, METH_VARARGS, ""},
+
+ {NULL, NULL, 0, NULL} /* Sentinel */
+};
+
+PyMODINIT_FUNC
+initfastbinary(void) {
+#define INIT_INTERN_STRING(value) \
+ do { \
+ INTERN_STRING(value) = PyString_InternFromString(#value); \
+ if(!INTERN_STRING(value)) return; \
+ } while(0)
+
+ INIT_INTERN_STRING(cstringio_buf);
+ INIT_INTERN_STRING(cstringio_refill);
+#undef INIT_INTERN_STRING
+
+ PycString_IMPORT;
+ if (PycStringIO == NULL) return;
+
+ (void) Py_InitModule("thrift.protocol.fastbinary", ThriftFastBinaryMethods);
+}
diff --git a/lib/py/src/server/THttpServer.py b/lib/py/src/server/THttpServer.py
new file mode 100644
index 0000000..21fc314
--- /dev/null
+++ b/lib/py/src/server/THttpServer.py
@@ -0,0 +1,63 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+import BaseHTTPServer
+
+from thrift.server import TServer
+from thrift.transport import TTransport
+
+class THttpServer(TServer.TServer):
+ """A simple HTTP-based Thrift server
+
+ This class is not very performant, but it is useful (for example) for
+ acting as a mock version of an Apache-based PHP Thrift endpoint."""
+
+ def __init__(self, processor, server_address,
+ inputProtocolFactory, outputProtocolFactory = None):
+ """Set up protocol factories and HTTP server.
+
+ See BaseHTTPServer for server_address.
+ See TServer for protocol factories."""
+
+ if outputProtocolFactory is None:
+ outputProtocolFactory = inputProtocolFactory
+
+ TServer.TServer.__init__(self, processor, None, None, None,
+ inputProtocolFactory, outputProtocolFactory)
+
+ thttpserver = self
+
+ class RequestHander(BaseHTTPServer.BaseHTTPRequestHandler):
+ def do_POST(self):
+ # Don't care about the request path.
+ self.send_response(200)
+ self.send_header("content-type", "application/x-thrift")
+ self.end_headers()
+
+ itrans = TTransport.TFileObjectTransport(self.rfile)
+ otrans = TTransport.TFileObjectTransport(self.wfile)
+ iprot = thttpserver.inputProtocolFactory.getProtocol(itrans)
+ oprot = thttpserver.outputProtocolFactory.getProtocol(otrans)
+ thttpserver.processor.process(iprot, oprot)
+ otrans.flush()
+
+ self.httpd = BaseHTTPServer.HTTPServer(server_address, RequestHander)
+
+ def serve(self):
+ self.httpd.serve_forever()
diff --git a/lib/py/src/server/TNonblockingServer.py b/lib/py/src/server/TNonblockingServer.py
new file mode 100644
index 0000000..deec708
--- /dev/null
+++ b/lib/py/src/server/TNonblockingServer.py
@@ -0,0 +1,309 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+"""Implementation of non-blocking server.
+
+The main idea of the server is reciving and sending requests
+only from main thread.
+
+It also makes thread pool server in tasks terms, not connections.
+"""
+import threading
+import socket
+import Queue
+import select
+import struct
+import logging
+
+from thrift.transport import TTransport
+from thrift.protocol.TBinaryProtocol import TBinaryProtocolFactory
+
+__all__ = ['TNonblockingServer']
+
+class Worker(threading.Thread):
+ """Worker is a small helper to process incoming connection."""
+ def __init__(self, queue):
+ threading.Thread.__init__(self)
+ self.queue = queue
+
+ def run(self):
+ """Process queries from task queue, stop if processor is None."""
+ while True:
+ try:
+ processor, iprot, oprot, otrans, callback = self.queue.get()
+ if processor is None:
+ break
+ processor.process(iprot, oprot)
+ callback(True, otrans.getvalue())
+ except Exception:
+ logging.exception("Exception while processing request")
+ callback(False, '')
+
+WAIT_LEN = 0
+WAIT_MESSAGE = 1
+WAIT_PROCESS = 2
+SEND_ANSWER = 3
+CLOSED = 4
+
+def locked(func):
+ "Decorator which locks self.lock."
+ def nested(self, *args, **kwargs):
+ self.lock.acquire()
+ try:
+ return func(self, *args, **kwargs)
+ finally:
+ self.lock.release()
+ return nested
+
+def socket_exception(func):
+ "Decorator close object on socket.error."
+ def read(self, *args, **kwargs):
+ try:
+ return func(self, *args, **kwargs)
+ except socket.error:
+ self.close()
+ return read
+
+class Connection:
+ """Basic class is represented connection.
+
+ It can be in state:
+ WAIT_LEN --- connection is reading request len.
+ WAIT_MESSAGE --- connection is reading request.
+ WAIT_PROCESS --- connection has just read whole request and
+ waits for call ready routine.
+ SEND_ANSWER --- connection is sending answer string (including length
+ of answer).
+ CLOSED --- socket was closed and connection should be deleted.
+ """
+ def __init__(self, new_socket, wake_up):
+ self.socket = new_socket
+ self.socket.setblocking(False)
+ self.status = WAIT_LEN
+ self.len = 0
+ self.message = ''
+ self.lock = threading.Lock()
+ self.wake_up = wake_up
+
+ def _read_len(self):
+ """Reads length of request.
+
+ It's really paranoic routine and it may be replaced by
+ self.socket.recv(4)."""
+ read = self.socket.recv(4 - len(self.message))
+ if len(read) == 0:
+ # if we read 0 bytes and self.message is empty, it means client close
+ # connection
+ if len(self.message) != 0:
+ logging.error("can't read frame size from socket")
+ self.close()
+ return
+ self.message += read
+ if len(self.message) == 4:
+ self.len, = struct.unpack('!i', self.message)
+ if self.len < 0:
+ logging.error("negative frame size, it seems client"\
+ " doesn't use FramedTransport")
+ self.close()
+ elif self.len == 0:
+ logging.error("empty frame, it's really strange")
+ self.close()
+ else:
+ self.message = ''
+ self.status = WAIT_MESSAGE
+
+ @socket_exception
+ def read(self):
+ """Reads data from stream and switch state."""
+ assert self.status in (WAIT_LEN, WAIT_MESSAGE)
+ if self.status == WAIT_LEN:
+ self._read_len()
+ # go back to the main loop here for simplicity instead of
+ # falling through, even though there is a good chance that
+ # the message is already available
+ elif self.status == WAIT_MESSAGE:
+ read = self.socket.recv(self.len - len(self.message))
+ if len(read) == 0:
+ logging.error("can't read frame from socket (get %d of %d bytes)" %
+ (len(self.message), self.len))
+ self.close()
+ return
+ self.message += read
+ if len(self.message) == self.len:
+ self.status = WAIT_PROCESS
+
+ @socket_exception
+ def write(self):
+ """Writes data from socket and switch state."""
+ assert self.status == SEND_ANSWER
+ sent = self.socket.send(self.message)
+ if sent == len(self.message):
+ self.status = WAIT_LEN
+ self.message = ''
+ self.len = 0
+ else:
+ self.message = self.message[sent:]
+
+ @locked
+ def ready(self, all_ok, message):
+ """Callback function for switching state and waking up main thread.
+
+ This function is the only function witch can be called asynchronous.
+
+ The ready can switch Connection to three states:
+ WAIT_LEN if request was oneway.
+ SEND_ANSWER if request was processed in normal way.
+ CLOSED if request throws unexpected exception.
+
+ The one wakes up main thread.
+ """
+ assert self.status == WAIT_PROCESS
+ if not all_ok:
+ self.close()
+ self.wake_up()
+ return
+ self.len = ''
+ self.message = struct.pack('!i', len(message)) + message
+ if len(message) == 0:
+ # it was a oneway request, do not write answer
+ self.status = WAIT_LEN
+ else:
+ self.status = SEND_ANSWER
+ self.wake_up()
+
+ @locked
+ def is_writeable(self):
+ "Returns True if connection should be added to write list of select."
+ return self.status == SEND_ANSWER
+
+ # it's not necessary, but...
+ @locked
+ def is_readable(self):
+ "Returns True if connection should be added to read list of select."
+ return self.status in (WAIT_LEN, WAIT_MESSAGE)
+
+ @locked
+ def is_closed(self):
+ "Returns True if connection is closed."
+ return self.status == CLOSED
+
+ def fileno(self):
+ "Returns the file descriptor of the associated socket."
+ return self.socket.fileno()
+
+ def close(self):
+ "Closes connection"
+ self.status = CLOSED
+ self.socket.close()
+
+class TNonblockingServer:
+ """Non-blocking server."""
+ def __init__(self, processor, lsocket, inputProtocolFactory=None,
+ outputProtocolFactory=None, threads=10):
+ self.processor = processor
+ self.socket = lsocket
+ self.in_protocol = inputProtocolFactory or TBinaryProtocolFactory()
+ self.out_protocol = outputProtocolFactory or self.in_protocol
+ self.threads = int(threads)
+ self.clients = {}
+ self.tasks = Queue.Queue()
+ self._read, self._write = socket.socketpair()
+ self.prepared = False
+
+ def setNumThreads(self, num):
+ """Set the number of worker threads that should be created."""
+ # implement ThreadPool interface
+ assert not self.prepared, "You can't change number of threads for working server"
+ self.threads = num
+
+ def prepare(self):
+ """Prepares server for serve requests."""
+ self.socket.listen()
+ for _ in xrange(self.threads):
+ thread = Worker(self.tasks)
+ thread.setDaemon(True)
+ thread.start()
+ self.prepared = True
+
+ def wake_up(self):
+ """Wake up main thread.
+
+ The server usualy waits in select call in we should terminate one.
+ The simplest way is using socketpair.
+
+ Select always wait to read from the first socket of socketpair.
+
+ In this case, we can just write anything to the second socket from
+ socketpair."""
+ self._write.send('1')
+
+ def _select(self):
+ """Does select on open connections."""
+ readable = [self.socket.handle.fileno(), self._read.fileno()]
+ writable = []
+ for i, connection in self.clients.items():
+ if connection.is_readable():
+ readable.append(connection.fileno())
+ if connection.is_writeable():
+ writable.append(connection.fileno())
+ if connection.is_closed():
+ del self.clients[i]
+ return select.select(readable, writable, readable)
+
+ def handle(self):
+ """Handle requests.
+
+ WARNING! You must call prepare BEFORE calling handle.
+ """
+ assert self.prepared, "You have to call prepare before handle"
+ rset, wset, xset = self._select()
+ for readable in rset:
+ if readable == self._read.fileno():
+ # don't care i just need to clean readable flag
+ self._read.recv(1024)
+ elif readable == self.socket.handle.fileno():
+ client = self.socket.accept().handle
+ self.clients[client.fileno()] = Connection(client, self.wake_up)
+ else:
+ connection = self.clients[readable]
+ connection.read()
+ if connection.status == WAIT_PROCESS:
+ itransport = TTransport.TMemoryBuffer(connection.message)
+ otransport = TTransport.TMemoryBuffer()
+ iprot = self.in_protocol.getProtocol(itransport)
+ oprot = self.out_protocol.getProtocol(otransport)
+ self.tasks.put([self.processor, iprot, oprot,
+ otransport, connection.ready])
+ for writeable in wset:
+ self.clients[writeable].write()
+ for oob in xset:
+ self.clients[oob].close()
+ del self.clients[oob]
+
+ def close(self):
+ """Closes the server."""
+ for _ in xrange(self.threads):
+ self.tasks.put([None, None, None, None, None])
+ self.socket.close()
+ self.prepared = False
+
+ def serve(self):
+ """Serve forever."""
+ self.prepare()
+ while True:
+ self.handle()
diff --git a/lib/py/src/server/TServer.py b/lib/py/src/server/TServer.py
new file mode 100644
index 0000000..6152911
--- /dev/null
+++ b/lib/py/src/server/TServer.py
@@ -0,0 +1,270 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+import logging
+import sys
+import os
+import traceback
+import threading
+import Queue
+
+from thrift.Thrift import TProcessor
+from thrift.transport import TTransport
+from thrift.protocol import TBinaryProtocol
+
+class TServer:
+
+ """Base interface for a server, which must have a serve method."""
+
+ """ 3 constructors for all servers:
+ 1) (processor, serverTransport)
+ 2) (processor, serverTransport, transportFactory, protocolFactory)
+ 3) (processor, serverTransport,
+ inputTransportFactory, outputTransportFactory,
+ inputProtocolFactory, outputProtocolFactory)"""
+ def __init__(self, *args):
+ if (len(args) == 2):
+ self.__initArgs__(args[0], args[1],
+ TTransport.TTransportFactoryBase(),
+ TTransport.TTransportFactoryBase(),
+ TBinaryProtocol.TBinaryProtocolFactory(),
+ TBinaryProtocol.TBinaryProtocolFactory())
+ elif (len(args) == 4):
+ self.__initArgs__(args[0], args[1], args[2], args[2], args[3], args[3])
+ elif (len(args) == 6):
+ self.__initArgs__(args[0], args[1], args[2], args[3], args[4], args[5])
+
+ def __initArgs__(self, processor, serverTransport,
+ inputTransportFactory, outputTransportFactory,
+ inputProtocolFactory, outputProtocolFactory):
+ self.processor = processor
+ self.serverTransport = serverTransport
+ self.inputTransportFactory = inputTransportFactory
+ self.outputTransportFactory = outputTransportFactory
+ self.inputProtocolFactory = inputProtocolFactory
+ self.outputProtocolFactory = outputProtocolFactory
+
+ def serve(self):
+ pass
+
+class TSimpleServer(TServer):
+
+ """Simple single-threaded server that just pumps around one transport."""
+
+ def __init__(self, *args):
+ TServer.__init__(self, *args)
+
+ def serve(self):
+ self.serverTransport.listen()
+ while True:
+ client = self.serverTransport.accept()
+ itrans = self.inputTransportFactory.getTransport(client)
+ otrans = self.outputTransportFactory.getTransport(client)
+ iprot = self.inputProtocolFactory.getProtocol(itrans)
+ oprot = self.outputProtocolFactory.getProtocol(otrans)
+ try:
+ while True:
+ self.processor.process(iprot, oprot)
+ except TTransport.TTransportException, tx:
+ pass
+ except Exception, x:
+ logging.exception(x)
+
+ itrans.close()
+ otrans.close()
+
+class TThreadedServer(TServer):
+
+ """Threaded server that spawns a new thread per each connection."""
+
+ def __init__(self, *args):
+ TServer.__init__(self, *args)
+
+ def serve(self):
+ self.serverTransport.listen()
+ while True:
+ try:
+ client = self.serverTransport.accept()
+ t = threading.Thread(target = self.handle, args=(client,))
+ t.start()
+ except KeyboardInterrupt:
+ raise
+ except Exception, x:
+ logging.exception(x)
+
+ def handle(self, client):
+ itrans = self.inputTransportFactory.getTransport(client)
+ otrans = self.outputTransportFactory.getTransport(client)
+ iprot = self.inputProtocolFactory.getProtocol(itrans)
+ oprot = self.outputProtocolFactory.getProtocol(otrans)
+ try:
+ while True:
+ self.processor.process(iprot, oprot)
+ except TTransport.TTransportException, tx:
+ pass
+ except Exception, x:
+ logging.exception(x)
+
+ itrans.close()
+ otrans.close()
+
+class TThreadPoolServer(TServer):
+
+ """Server with a fixed size pool of threads which service requests."""
+
+ def __init__(self, *args):
+ TServer.__init__(self, *args)
+ self.clients = Queue.Queue()
+ self.threads = 10
+
+ def setNumThreads(self, num):
+ """Set the number of worker threads that should be created"""
+ self.threads = num
+
+ def serveThread(self):
+ """Loop around getting clients from the shared queue and process them."""
+ while True:
+ try:
+ client = self.clients.get()
+ self.serveClient(client)
+ except Exception, x:
+ logging.exception(x)
+
+ def serveClient(self, client):
+ """Process input/output from a client for as long as possible"""
+ itrans = self.inputTransportFactory.getTransport(client)
+ otrans = self.outputTransportFactory.getTransport(client)
+ iprot = self.inputProtocolFactory.getProtocol(itrans)
+ oprot = self.outputProtocolFactory.getProtocol(otrans)
+ try:
+ while True:
+ self.processor.process(iprot, oprot)
+ except TTransport.TTransportException, tx:
+ pass
+ except Exception, x:
+ logging.exception(x)
+
+ itrans.close()
+ otrans.close()
+
+ def serve(self):
+ """Start a fixed number of worker threads and put client into a queue"""
+ for i in range(self.threads):
+ try:
+ t = threading.Thread(target = self.serveThread)
+ t.start()
+ except Exception, x:
+ logging.exception(x)
+
+ # Pump the socket for clients
+ self.serverTransport.listen()
+ while True:
+ try:
+ client = self.serverTransport.accept()
+ self.clients.put(client)
+ except Exception, x:
+ logging.exception(x)
+
+
+class TForkingServer(TServer):
+
+ """A Thrift server that forks a new process for each request"""
+ """
+ This is more scalable than the threaded server as it does not cause
+ GIL contention.
+
+ Note that this has different semantics from the threading server.
+ Specifically, updates to shared variables will no longer be shared.
+ It will also not work on windows.
+
+ This code is heavily inspired by SocketServer.ForkingMixIn in the
+ Python stdlib.
+ """
+
+ def __init__(self, *args):
+ TServer.__init__(self, *args)
+ self.children = []
+
+ def serve(self):
+ def try_close(file):
+ try:
+ file.close()
+ except IOError, e:
+ logging.warning(e, exc_info=True)
+
+
+ self.serverTransport.listen()
+ while True:
+ client = self.serverTransport.accept()
+ try:
+ pid = os.fork()
+
+ if pid: # parent
+ # add before collect, otherwise you race w/ waitpid
+ self.children.append(pid)
+ self.collect_children()
+
+ # Parent must close socket or the connection may not get
+ # closed promptly
+ itrans = self.inputTransportFactory.getTransport(client)
+ otrans = self.outputTransportFactory.getTransport(client)
+ try_close(itrans)
+ try_close(otrans)
+ else:
+ itrans = self.inputTransportFactory.getTransport(client)
+ otrans = self.outputTransportFactory.getTransport(client)
+
+ iprot = self.inputProtocolFactory.getProtocol(itrans)
+ oprot = self.outputProtocolFactory.getProtocol(otrans)
+
+ ecode = 0
+ try:
+ try:
+ while True:
+ self.processor.process(iprot, oprot)
+ except TTransport.TTransportException, tx:
+ pass
+ except Exception, e:
+ logging.exception(e)
+ ecode = 1
+ finally:
+ try_close(itrans)
+ try_close(otrans)
+
+ os._exit(ecode)
+
+ except TTransport.TTransportException, tx:
+ pass
+ except Exception, x:
+ logging.exception(x)
+
+
+ def collect_children(self):
+ while self.children:
+ try:
+ pid, status = os.waitpid(0, os.WNOHANG)
+ except os.error:
+ pid = None
+
+ if pid:
+ self.children.remove(pid)
+ else:
+ break
+
+
diff --git a/lib/py/src/server/__init__.py b/lib/py/src/server/__init__.py
new file mode 100644
index 0000000..1bf6e25
--- /dev/null
+++ b/lib/py/src/server/__init__.py
@@ -0,0 +1,20 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+__all__ = ['TServer', 'TNonblockingServer']
diff --git a/lib/py/src/transport/THttpClient.py b/lib/py/src/transport/THttpClient.py
new file mode 100644
index 0000000..5086032
--- /dev/null
+++ b/lib/py/src/transport/THttpClient.py
@@ -0,0 +1,100 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+from TTransport import *
+from cStringIO import StringIO
+
+import urlparse
+import httplib
+import warnings
+
+class THttpClient(TTransportBase):
+
+ """Http implementation of TTransport base."""
+
+ def __init__(self, uri_or_host, port=None, path=None):
+ """THttpClient supports two different types constructor parameters.
+
+ THttpClient(host, port, path) - deprecated
+ THttpClient(uri)
+
+ Only the second supports https."""
+
+ if port is not None:
+ warnings.warn("Please use the THttpClient('http://host:port/path') syntax", DeprecationWarning, stacklevel=2)
+ self.host = uri_or_host
+ self.port = port
+ assert path
+ self.path = path
+ self.scheme = 'http'
+ else:
+ parsed = urlparse.urlparse(uri_or_host)
+ self.scheme = parsed.scheme
+ assert self.scheme in ('http', 'https')
+ if self.scheme == 'http':
+ self.port = parsed.port or httplib.HTTP_PORT
+ elif self.scheme == 'https':
+ self.port = parsed.port or httplib.HTTPS_PORT
+ self.host = parsed.hostname
+ self.path = parsed.path
+ self.__wbuf = StringIO()
+ self.__http = None
+
+ def open(self):
+ if self.scheme == 'http':
+ self.__http = httplib.HTTP(self.host, self.port)
+ else:
+ self.__http = httplib.HTTPS(self.host, self.port)
+
+ def close(self):
+ self.__http.close()
+ self.__http = None
+
+ def isOpen(self):
+ return self.__http != None
+
+ def read(self, sz):
+ return self.__http.file.read(sz)
+
+ def write(self, buf):
+ self.__wbuf.write(buf)
+
+ def flush(self):
+ if self.isOpen():
+ self.close()
+ self.open();
+
+ # Pull data out of buffer
+ data = self.__wbuf.getvalue()
+ self.__wbuf = StringIO()
+
+ # HTTP request
+ self.__http.putrequest('POST', self.path)
+
+ # Write headers
+ self.__http.putheader('Host', self.host)
+ self.__http.putheader('Content-Type', 'application/x-thrift')
+ self.__http.putheader('Content-Length', str(len(data)))
+ self.__http.endheaders()
+
+ # Write payload
+ self.__http.send(data)
+
+ # Get reply to flush the request
+ self.code, self.message, self.headers = self.__http.getreply()
diff --git a/lib/py/src/transport/TSocket.py b/lib/py/src/transport/TSocket.py
new file mode 100644
index 0000000..4645a02
--- /dev/null
+++ b/lib/py/src/transport/TSocket.py
@@ -0,0 +1,147 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+from TTransport import *
+import os
+import errno
+import socket
+
+class TSocketBase(TTransportBase):
+ def _resolveAddr(self):
+ if self._unix_socket is not None:
+ return [(socket.AF_UNIX, socket.SOCK_STREAM, None, None, self._unix_socket)]
+ else:
+ return socket.getaddrinfo(self.host, self.port, socket.AF_UNSPEC, socket.SOCK_STREAM, 0, socket.AI_PASSIVE | socket.AI_ADDRCONFIG)
+
+ def close(self):
+ if self.handle:
+ self.handle.close()
+ self.handle = None
+
+class TSocket(TSocketBase):
+ """Socket implementation of TTransport base."""
+
+ def __init__(self, host='localhost', port=9090, unix_socket=None):
+ """Initialize a TSocket
+
+ @param host(str) The host to connect to.
+ @param port(int) The (TCP) port to connect to.
+ @param unix_socket(str) The filename of a unix socket to connect to.
+ (host and port will be ignored.)
+ """
+
+ self.host = host
+ self.port = port
+ self.handle = None
+ self._unix_socket = unix_socket
+ self._timeout = None
+
+ def setHandle(self, h):
+ self.handle = h
+
+ def isOpen(self):
+ return self.handle != None
+
+ def setTimeout(self, ms):
+ if ms is None:
+ self._timeout = None
+ else:
+ self._timeout = ms/1000.0
+
+ if (self.handle != None):
+ self.handle.settimeout(self._timeout)
+
+ def open(self):
+ try:
+ res0 = self._resolveAddr()
+ for res in res0:
+ self.handle = socket.socket(res[0], res[1])
+ self.handle.settimeout(self._timeout)
+ try:
+ self.handle.connect(res[4])
+ except socket.error, e:
+ if res is not res0[-1]:
+ continue
+ else:
+ raise e
+ break
+ except socket.error, e:
+ if self._unix_socket:
+ message = 'Could not connect to socket %s' % self._unix_socket
+ else:
+ message = 'Could not connect to %s:%d' % (self.host, self.port)
+ raise TTransportException(TTransportException.NOT_OPEN, message)
+
+ def read(self, sz):
+ buff = self.handle.recv(sz)
+ if len(buff) == 0:
+ raise TTransportException('TSocket read 0 bytes')
+ return buff
+
+ def write(self, buff):
+ sent = 0
+ have = len(buff)
+ while sent < have:
+ plus = self.handle.send(buff)
+ if plus == 0:
+ raise TTransportException('TSocket sent 0 bytes')
+ sent += plus
+ buff = buff[plus:]
+
+ def flush(self):
+ pass
+
+class TServerSocket(TSocketBase, TServerTransportBase):
+ """Socket implementation of TServerTransport base."""
+
+ def __init__(self, port=9090, unix_socket=None):
+ self.host = None
+ self.port = port
+ self._unix_socket = unix_socket
+ self.handle = None
+
+ def listen(self):
+ res0 = self._resolveAddr()
+ for res in res0:
+ if res[0] is socket.AF_INET6 or res is res0[-1]:
+ break
+
+ # We need remove the old unix socket if the file exists and
+ # nobody is listening on it.
+ if self._unix_socket:
+ tmp = socket.socket(res[0], res[1])
+ try:
+ tmp.connect(res[4])
+ except socket.error, err:
+ eno, message = err.args
+ if eno == errno.ECONNREFUSED:
+ os.unlink(res[4])
+
+ self.handle = socket.socket(res[0], res[1])
+ self.handle.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
+ if hasattr(self.handle, 'set_timeout'):
+ self.handle.set_timeout(None)
+ self.handle.bind(res[4])
+ self.handle.listen(128)
+
+ def accept(self):
+ client, addr = self.handle.accept()
+ result = TSocket()
+ result.setHandle(client)
+ return result
diff --git a/lib/py/src/transport/TTransport.py b/lib/py/src/transport/TTransport.py
new file mode 100644
index 0000000..32553bd
--- /dev/null
+++ b/lib/py/src/transport/TTransport.py
@@ -0,0 +1,326 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+from cStringIO import StringIO
+from struct import pack,unpack
+from thrift.Thrift import TException
+
+class TTransportException(TException):
+
+ """Custom Transport Exception class"""
+
+ UNKNOWN = 0
+ NOT_OPEN = 1
+ ALREADY_OPEN = 2
+ TIMED_OUT = 3
+ END_OF_FILE = 4
+
+ def __init__(self, type=UNKNOWN, message=None):
+ TException.__init__(self, message)
+ self.type = type
+
+class TTransportBase:
+
+ """Base class for Thrift transport layer."""
+
+ def isOpen(self):
+ pass
+
+ def open(self):
+ pass
+
+ def close(self):
+ pass
+
+ def read(self, sz):
+ pass
+
+ def readAll(self, sz):
+ buff = ''
+ have = 0
+ while (have < sz):
+ chunk = self.read(sz-have)
+ have += len(chunk)
+ buff += chunk
+
+ if len(chunk) == 0:
+ raise EOFError()
+
+ return buff
+
+ def write(self, buf):
+ pass
+
+ def flush(self):
+ pass
+
+# This class should be thought of as an interface.
+class CReadableTransport:
+ """base class for transports that are readable from C"""
+
+ # TODO(dreiss): Think about changing this interface to allow us to use
+ # a (Python, not c) StringIO instead, because it allows
+ # you to write after reading.
+
+ # NOTE: This is a classic class, so properties will NOT work
+ # correctly for setting.
+ @property
+ def cstringio_buf(self):
+ """A cStringIO buffer that contains the current chunk we are reading."""
+ pass
+
+ def cstringio_refill(self, partialread, reqlen):
+ """Refills cstringio_buf.
+
+ Returns the currently used buffer (which can but need not be the same as
+ the old cstringio_buf). partialread is what the C code has read from the
+ buffer, and should be inserted into the buffer before any more reads. The
+ return value must be a new, not borrowed reference. Something along the
+ lines of self._buf should be fine.
+
+ If reqlen bytes can't be read, throw EOFError.
+ """
+ pass
+
+class TServerTransportBase:
+
+ """Base class for Thrift server transports."""
+
+ def listen(self):
+ pass
+
+ def accept(self):
+ pass
+
+ def close(self):
+ pass
+
+class TTransportFactoryBase:
+
+ """Base class for a Transport Factory"""
+
+ def getTransport(self, trans):
+ return trans
+
+class TBufferedTransportFactory:
+
+ """Factory transport that builds buffered transports"""
+
+ def getTransport(self, trans):
+ buffered = TBufferedTransport(trans)
+ return buffered
+
+
+class TBufferedTransport(TTransportBase,CReadableTransport):
+
+ """Class that wraps another transport and buffers its I/O."""
+
+ DEFAULT_BUFFER = 4096
+
+ def __init__(self, trans):
+ self.__trans = trans
+ self.__wbuf = StringIO()
+ self.__rbuf = StringIO("")
+
+ def isOpen(self):
+ return self.__trans.isOpen()
+
+ def open(self):
+ return self.__trans.open()
+
+ def close(self):
+ return self.__trans.close()
+
+ def read(self, sz):
+ ret = self.__rbuf.read(sz)
+ if len(ret) != 0:
+ return ret
+
+ self.__rbuf = StringIO(self.__trans.read(max(sz, self.DEFAULT_BUFFER)))
+ return self.__rbuf.read(sz)
+
+ def write(self, buf):
+ self.__wbuf.write(buf)
+
+ def flush(self):
+ out = self.__wbuf.getvalue()
+ # reset wbuf before write/flush to preserve state on underlying failure
+ self.__wbuf = StringIO()
+ self.__trans.write(out)
+ self.__trans.flush()
+
+ # Implement the CReadableTransport interface.
+ @property
+ def cstringio_buf(self):
+ return self.__rbuf
+
+ def cstringio_refill(self, partialread, reqlen):
+ retstring = partialread
+ if reqlen < self.DEFAULT_BUFFER:
+ # try to make a read of as much as we can.
+ retstring += self.__trans.read(self.DEFAULT_BUFFER)
+
+ # but make sure we do read reqlen bytes.
+ if len(retstring) < reqlen:
+ retstring += self.__trans.readAll(reqlen - len(retstring))
+
+ self.__rbuf = StringIO(retstring)
+ return self.__rbuf
+
+class TMemoryBuffer(TTransportBase, CReadableTransport):
+ """Wraps a cStringIO object as a TTransport.
+
+ NOTE: Unlike the C++ version of this class, you cannot write to it
+ then immediately read from it. If you want to read from a
+ TMemoryBuffer, you must either pass a string to the constructor.
+ TODO(dreiss): Make this work like the C++ version.
+ """
+
+ def __init__(self, value=None):
+ """value -- a value to read from for stringio
+
+ If value is set, this will be a transport for reading,
+ otherwise, it is for writing"""
+ if value is not None:
+ self._buffer = StringIO(value)
+ else:
+ self._buffer = StringIO()
+
+ def isOpen(self):
+ return not self._buffer.closed
+
+ def open(self):
+ pass
+
+ def close(self):
+ self._buffer.close()
+
+ def read(self, sz):
+ return self._buffer.read(sz)
+
+ def write(self, buf):
+ self._buffer.write(buf)
+
+ def flush(self):
+ pass
+
+ def getvalue(self):
+ return self._buffer.getvalue()
+
+ # Implement the CReadableTransport interface.
+ @property
+ def cstringio_buf(self):
+ return self._buffer
+
+ def cstringio_refill(self, partialread, reqlen):
+ # only one shot at reading...
+ raise EOFError()
+
+class TFramedTransportFactory:
+
+ """Factory transport that builds framed transports"""
+
+ def getTransport(self, trans):
+ framed = TFramedTransport(trans)
+ return framed
+
+
+class TFramedTransport(TTransportBase, CReadableTransport):
+
+ """Class that wraps another transport and frames its I/O when writing."""
+
+ def __init__(self, trans,):
+ self.__trans = trans
+ self.__rbuf = StringIO()
+ self.__wbuf = StringIO()
+
+ def isOpen(self):
+ return self.__trans.isOpen()
+
+ def open(self):
+ return self.__trans.open()
+
+ def close(self):
+ return self.__trans.close()
+
+ def read(self, sz):
+ ret = self.__rbuf.read(sz)
+ if len(ret) != 0:
+ return ret
+
+ self.readFrame()
+ return self.__rbuf.read(sz)
+
+ def readFrame(self):
+ buff = self.__trans.readAll(4)
+ sz, = unpack('!i', buff)
+ self.__rbuf = StringIO(self.__trans.readAll(sz))
+
+ def write(self, buf):
+ self.__wbuf.write(buf)
+
+ def flush(self):
+ wout = self.__wbuf.getvalue()
+ wsz = len(wout)
+ # reset wbuf before write/flush to preserve state on underlying failure
+ self.__wbuf = StringIO()
+ # N.B.: Doing this string concatenation is WAY cheaper than making
+ # two separate calls to the underlying socket object. Socket writes in
+ # Python turn out to be REALLY expensive, but it seems to do a pretty
+ # good job of managing string buffer operations without excessive copies
+ buf = pack("!i", wsz) + wout
+ self.__trans.write(buf)
+ self.__trans.flush()
+
+ # Implement the CReadableTransport interface.
+ @property
+ def cstringio_buf(self):
+ return self.__rbuf
+
+ def cstringio_refill(self, prefix, reqlen):
+ # self.__rbuf will already be empty here because fastbinary doesn't
+ # ask for a refill until the previous buffer is empty. Therefore,
+ # we can start reading new frames immediately.
+ while len(prefix) < reqlen:
+ readFrame()
+ prefix += self.__rbuf.getvalue()
+ self.__rbuf = StringIO(prefix)
+ return self.__rbuf
+
+
+class TFileObjectTransport(TTransportBase):
+ """Wraps a file-like object to make it work as a Thrift transport."""
+
+ def __init__(self, fileobj):
+ self.fileobj = fileobj
+
+ def isOpen(self):
+ return True
+
+ def close(self):
+ self.fileobj.close()
+
+ def read(self, sz):
+ return self.fileobj.read(sz)
+
+ def write(self, buf):
+ self.fileobj.write(buf)
+
+ def flush(self):
+ self.fileobj.flush()
diff --git a/lib/py/src/transport/TTwisted.py b/lib/py/src/transport/TTwisted.py
new file mode 100644
index 0000000..b5c2147
--- /dev/null
+++ b/lib/py/src/transport/TTwisted.py
@@ -0,0 +1,177 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+from zope.interface import implements, Interface, Attribute
+from twisted.internet.protocol import Protocol, ServerFactory, ClientFactory, \
+ connectionDone
+from twisted.internet import defer
+from twisted.protocols import basic
+from twisted.python import log
+
+
+from thrift.transport import TTransport
+from cStringIO import StringIO
+
+
+class TMessageSenderTransport(TTransport.TTransportBase):
+
+ def __init__(self):
+ self.__wbuf = StringIO()
+
+ def write(self, buf):
+ self.__wbuf.write(buf)
+
+ def flush(self):
+ msg = self.__wbuf.getvalue()
+ self.__wbuf = StringIO()
+ self.sendMessage(msg)
+
+ def sendMessage(self, message):
+ raise NotImplementedError
+
+
+class TCallbackTransport(TMessageSenderTransport):
+
+ def __init__(self, func):
+ TMessageSenderTransport.__init__(self)
+ self.func = func
+
+ def sendMessage(self, message):
+ self.func(message)
+
+
+class ThriftClientProtocol(basic.Int32StringReceiver):
+
+ def __init__(self, client_class, iprot_factory, oprot_factory=None):
+ self._client_class = client_class
+ self._iprot_factory = iprot_factory
+ if oprot_factory is None:
+ self._oprot_factory = iprot_factory
+ else:
+ self._oprot_factory = oprot_factory
+
+ self.recv_map = {}
+ self.started = defer.Deferred()
+
+ def dispatch(self, msg):
+ self.sendString(msg)
+
+ def connectionMade(self):
+ tmo = TCallbackTransport(self.dispatch)
+ self.client = self._client_class(tmo, self._oprot_factory)
+ self.started.callback(self.client)
+
+ def connectionLost(self, reason=connectionDone):
+ for k,v in self.client._reqs.iteritems():
+ tex = TTransport.TTransportException(
+ type=TTransport.TTransportException.END_OF_FILE,
+ message='Connection closed')
+ v.errback(tex)
+
+ def stringReceived(self, frame):
+ tr = TTransport.TMemoryBuffer(frame)
+ iprot = self._iprot_factory.getProtocol(tr)
+ (fname, mtype, rseqid) = iprot.readMessageBegin()
+
+ try:
+ method = self.recv_map[fname]
+ except KeyError:
+ method = getattr(self.client, 'recv_' + fname)
+ self.recv_map[fname] = method
+
+ method(iprot, mtype, rseqid)
+
+
+class ThriftServerProtocol(basic.Int32StringReceiver):
+
+ def dispatch(self, msg):
+ self.sendString(msg)
+
+ def processError(self, error):
+ self.transport.loseConnection()
+
+ def processOk(self, _, tmo):
+ msg = tmo.getvalue()
+
+ if len(msg) > 0:
+ self.dispatch(msg)
+
+ def stringReceived(self, frame):
+ tmi = TTransport.TMemoryBuffer(frame)
+ tmo = TTransport.TMemoryBuffer()
+
+ iprot = self.factory.iprot_factory.getProtocol(tmi)
+ oprot = self.factory.oprot_factory.getProtocol(tmo)
+
+ d = self.factory.processor.process(iprot, oprot)
+ d.addCallbacks(self.processOk, self.processError,
+ callbackArgs=(tmo,))
+
+
+class IThriftServerFactory(Interface):
+
+ processor = Attribute("Thrift processor")
+
+ iprot_factory = Attribute("Input protocol factory")
+
+ oprot_factory = Attribute("Output protocol factory")
+
+
+class IThriftClientFactory(Interface):
+
+ client_class = Attribute("Thrift client class")
+
+ iprot_factory = Attribute("Input protocol factory")
+
+ oprot_factory = Attribute("Output protocol factory")
+
+
+class ThriftServerFactory(ServerFactory):
+
+ implements(IThriftServerFactory)
+
+ protocol = ThriftServerProtocol
+
+ def __init__(self, processor, iprot_factory, oprot_factory=None):
+ self.processor = processor
+ self.iprot_factory = iprot_factory
+ if oprot_factory is None:
+ self.oprot_factory = iprot_factory
+ else:
+ self.oprot_factory = oprot_factory
+
+
+class ThriftClientFactory(ClientFactory):
+
+ implements(IThriftClientFactory)
+
+ protocol = ThriftClientProtocol
+
+ def __init__(self, client_class, iprot_factory, oprot_factory=None):
+ self.client_class = client_class
+ self.iprot_factory = iprot_factory
+ if oprot_factory is None:
+ self.oprot_factory = iprot_factory
+ else:
+ self.oprot_factory = oprot_factory
+
+ def buildProtocol(self, addr):
+ p = self.protocol(self.client_class, self.iprot_factory,
+ self.oprot_factory)
+ p.factory = self
+ return p
diff --git a/lib/py/src/transport/__init__.py b/lib/py/src/transport/__init__.py
new file mode 100644
index 0000000..02c6048
--- /dev/null
+++ b/lib/py/src/transport/__init__.py
@@ -0,0 +1,20 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+__all__ = ['TTransport', 'TSocket', 'THttpClient']