Thrift: Native-code Binary Protocol encoder.
Summary:
Merging a patch from Ben Maurer.
This adds a python extension (i.e., a C module) that
encodes Python thrift structs into the standard binary protocol
much faster than our generated Python code.
Also added by-value equality comparison to thrift structs
(to help with testing).
Cleaned up some trailing whitespace too.
Reviewed By: mcslee, dreiss
Test Plan:
Recompiled Thrift.
Thrifted a bunch of IDLs and compared the generated Python output.
Looked at the extension module a lot.
test/FastBinaryTest.py
Revert Plan: ok
git-svn-id: https://svn.apache.org/repos/asf/incubator/thrift/trunk@665224 13f79535-47bb-0310-9956-ffa450edef68
diff --git a/test/FastbinaryTest.py b/test/FastbinaryTest.py
new file mode 100755
index 0000000..0918002
--- /dev/null
+++ b/test/FastbinaryTest.py
@@ -0,0 +1,190 @@
+#!/usr/bin/env python
+r"""
+thrift -py DebugProtoTest.thrift
+./FastbinaryTest.py
+"""
+
+# TODO(dreiss): Test error cases. Check for memory leaks.
+
+import sys
+sys.path.append('./gen-py')
+
+import math
+from DebugProtoTest import Srv
+from DebugProtoTest.ttypes import *
+from thrift.transport import TTransport
+from thrift.protocol import TBinaryProtocol
+
+import timeit
+from cStringIO import StringIO
+from copy import deepcopy
+from pprint import pprint
+
+class TDevNullTransport(TTransport.TTransportBase):
+ def __init__(self):
+ pass
+ def isOpen(self):
+ return True
+
+ooe1 = OneOfEach()
+ooe1.im_true = True;
+ooe1.im_false = False;
+ooe1.a_bite = 0xd6;
+ooe1.integer16 = 27000;
+ooe1.integer32 = 1<<24;
+ooe1.integer64 = 6000 * 1000 * 1000;
+ooe1.double_precision = math.pi;
+ooe1.some_characters = "Debug THIS!";
+ooe1.zomg_unicode = "\xd7\n\a\t";
+
+ooe2 = OneOfEach();
+ooe2.integer16 = 16;
+ooe2.integer32 = 32;
+ooe2.integer64 = 64;
+ooe2.double_precision = (math.sqrt(5)+1)/2;
+ooe2.some_characters = ":R (me going \"rrrr\")";
+ooe2.zomg_unicode = "\xd3\x80\xe2\x85\xae\xce\x9d\x20"\
+ "\xd0\x9d\xce\xbf\xe2\x85\xbf\xd0\xbe"\
+ "\xc9\xa1\xd0\xb3\xd0\xb0\xcf\x81\xe2\x84\x8e"\
+ "\x20\xce\x91\x74\x74\xce\xb1\xe2\x85\xbd\xce\xba"\
+ "\xc7\x83\xe2\x80\xbc";
+
+hm = HolyMoley({"big":[], "contain":set(), "bonks":{}})
+hm.big.append(ooe1)
+hm.big.append(ooe2)
+hm.big[0].a_bite = 0x22;
+hm.big[1].a_bite = 0x22;
+
+hm.contain.add(("and a one", "and a two"))
+hm.contain.add(("then a one, two", "three!", "FOUR!"))
+hm.contain.add(())
+
+hm.bonks["nothing"] = [];
+hm.bonks["something"] = [
+ Bonk({"type":1, "message":"Wait."}),
+ Bonk({"type":2, "message":"What?"}),
+]
+hm.bonks["poe"] = [
+ Bonk({"type":3, "message":"quoth"}),
+ Bonk({"type":4, "message":"the raven"}),
+ Bonk({"type":5, "message":"nevermore"}),
+]
+
+rs = RandomStuff()
+rs.a = 1
+rs.b = 2
+rs.c = 3
+rs.myintlist = range(20)
+rs.maps = {1:Wrapper({"foo":Empty()}),2:Wrapper({"foo":Empty()})}
+rs.bigint = 124523452435L
+rs.triple = 3.14
+
+my_zero = Srv.Janky_result({"arg":5})
+my_nega = Srv.Janky_args({"success":6})
+
+def checkWrite(o):
+ trans_fast = TTransport.TMemoryBuffer()
+ trans_slow = TTransport.TMemoryBuffer()
+ prot_fast = TBinaryProtocol.TBinaryProtocolAccelerated(trans_fast)
+ prot_slow = TBinaryProtocol.TBinaryProtocol(trans_slow)
+
+ o.write(prot_fast)
+ o.write(prot_slow)
+ ORIG = trans_slow.getvalue()
+ MINE = trans_fast.getvalue()
+ if ORIG != MINE:
+ print "mine: %s\norig: %s" % (repr(MINE), repr(ORIG))
+
+def checkRead(o):
+ prot = TBinaryProtocol.TBinaryProtocol(TTransport.TMemoryBuffer())
+ o.write(prot)
+ prot = TBinaryProtocol.TBinaryProtocolAccelerated(
+ TTransport.TMemoryBuffer(
+ prot.trans.getvalue()))
+ c = o.__class__()
+ c.read(prot)
+ if c != o:
+ print "copy: "
+ pprint(eval(repr(c)))
+ print "orig: "
+ pprint(eval(repr(o)))
+
+
+def doTest():
+ checkWrite(hm)
+ no_set = deepcopy(hm)
+ no_set.contain = set()
+ checkRead(no_set)
+ checkWrite(rs)
+ checkRead(rs)
+ checkWrite(my_zero)
+ checkRead(my_zero)
+ checkRead(Backwards({"first_tag2":4, "second_tag1":2}))
+ try:
+ checkWrite(my_nega)
+ print "Hey, did this get fixed?"
+ except AttributeError:
+ # Sorry, doesn't work with negative tags.
+ pass
+
+ # One case where the serialized form changes, but only superficially.
+ o = Backwards({"first_tag2":4, "second_tag1":2})
+ trans_fast = TTransport.TMemoryBuffer()
+ trans_slow = TTransport.TMemoryBuffer()
+ prot_fast = TBinaryProtocol.TBinaryProtocolAccelerated(trans_fast)
+ prot_slow = TBinaryProtocol.TBinaryProtocol(trans_slow)
+
+ o.write(prot_fast)
+ o.write(prot_slow)
+ ORIG = trans_slow.getvalue()
+ MINE = trans_fast.getvalue()
+ if ORIG == MINE:
+ print "That shouldn't happen."
+
+
+ prot = TBinaryProtocol.TBinaryProtocolAccelerated(TTransport.TMemoryBuffer())
+ o.write(prot)
+ prot = TBinaryProtocol.TBinaryProtocol(
+ TTransport.TMemoryBuffer(
+ prot.trans.getvalue()))
+ c = o.__class__()
+ c.read(prot)
+ if c != o:
+ print "copy: "
+ pprint(eval(repr(c)))
+ print "orig: "
+ pprint(eval(repr(o)))
+
+
+
+def doBenchmark():
+
+ iters = 25000
+
+ setup = """
+from __main__ import hm, rs, TDevNullTransport
+from thrift.protocol import TBinaryProtocol
+trans = TDevNullTransport()
+prot = TBinaryProtocol.TBinaryProtocol%s(trans)
+"""
+
+ setup_fast = setup % "Accelerated"
+ setup_slow = setup % ""
+
+ print "Starting Benchmarks"
+
+ print "HolyMoley Standard = %f" % \
+ timeit.Timer('hm.write(prot)', setup_slow).timeit(number=iters)
+ print "HolyMoley Acceler. = %f" % \
+ timeit.Timer('hm.write(prot)', setup_fast).timeit(number=iters)
+
+ print "FastStruct Standard = %f" % \
+ timeit.Timer('rs.write(prot)', setup_slow).timeit(number=iters)
+ print "FastStruct Acceler. = %f" % \
+ timeit.Timer('rs.write(prot)', setup_fast).timeit(number=iters)
+
+
+
+doTest()
+doBenchmark()
+