THRIFT-1745 Python JSON protocol
TJSONProtocol.py: Frederic Delbos
THRIFT-847 Test Framework harmonization across all languages
Integration into py lib and test suite
git-svn-id: https://svn.apache.org/repos/asf/thrift/trunk@1404838 13f79535-47bb-0310-9956-ffa450edef68
diff --git a/lib/py/src/protocol/TJSONProtocol.py b/lib/py/src/protocol/TJSONProtocol.py
new file mode 100644
index 0000000..97bc763
--- /dev/null
+++ b/lib/py/src/protocol/TJSONProtocol.py
@@ -0,0 +1,451 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+from TProtocol import *
+import json, base64, sys
+
+__all__ = ['TJSONProtocol', 'TJSONProtocolFactory']
+
+VERSION = 1
+
+COMMA = ','
+COLON = ':'
+LBRACE = '{'
+RBRACE = '}'
+LBRACKET = '['
+RBRACKET = ']'
+QUOTE = '"'
+BACKSLASH = '\\'
+ZERO = '0'
+
+ESCSEQ = '\\u00'
+ESCAPE_CHAR = '"\\bfnrt'
+ESCAPE_CHAR_VALS = ['"', '\\', '\b', '\f', '\n', '\r', '\t']
+NUMERIC_CHAR = '+-.0123456789Ee'
+
+CTYPES = {TType.BOOL: 'tf',
+ TType.BYTE: 'i8',
+ TType.I16: 'i16',
+ TType.I32: 'i32',
+ TType.I64: 'i64',
+ TType.DOUBLE: 'dbl',
+ TType.STRING: 'str',
+ TType.STRUCT: 'rect',
+ TType.LIST: 'lst',
+ TType.SET: 'set',
+ TType.MAP: 'map'}
+
+JTYPES = {}
+for key in CTYPES.keys():
+ JTYPES[CTYPES[key]] = key
+
+
+class JSONBaseContext():
+
+ def __init__(self, protocol):
+ self.protocol = protocol
+ self.first = True
+
+ def doIO(self, function):
+ pass
+
+ def write(self):
+ pass
+
+ def read(self):
+ pass
+
+ def escapeNum(self):
+ return False
+
+
+class JSONListContext(JSONBaseContext):
+
+ def doIO(self, function):
+ if self.first is True:
+ self.first = False
+ else:
+ function(COMMA)
+
+ def write(self):
+ self.doIO(self.protocol.trans.write)
+
+ def read(self):
+ self.doIO(self.protocol.readJSONSyntaxChar)
+
+
+class JSONPairContext(JSONBaseContext):
+ colon = True
+
+ def doIO(self, function):
+ if self.first is True:
+ self.first = False
+ self.colon = True
+ else:
+ function(COLON if self.colon == True else COMMA)
+ self.colon = not self.colon
+
+ def write(self):
+ self.doIO(self.protocol.trans.write)
+
+ def read(self):
+ self.doIO(self.protocol.readJSONSyntaxChar)
+
+ def escapeNum(self):
+ return self.colon
+
+
+class LookaheadReader():
+ hasData = False
+ data = ''
+
+ def __init__(self, protocol):
+ self.protocol = protocol
+
+ def read(self):
+ if self.hasData is True:
+ self.hasData = False
+ else:
+ self.data = self.protocol.trans.read(1)
+ return self.data
+
+ def peek(self):
+ if self.hasData is False:
+ self.data = self.protocol.trans.read(1)
+ self.hasData = True
+ return self.data
+
+class TJSONProtocolBase(TProtocolBase):
+
+ def __init__(self, trans):
+ TProtocolBase.__init__(self, trans)
+
+ def resetWriteContext(self):
+ self.contextStack = []
+ self.context = JSONBaseContext(self)
+
+ def resetReadContext(self):
+ self.resetWriteContext()
+ self.reader = LookaheadReader(self)
+
+ def pushContext(self, ctx):
+ self.contextStack.append(ctx)
+ self.context = ctx
+
+ def popContext(self):
+ self.contextStack.pop()
+
+ def writeJSONString(self, string):
+ self.context.write()
+ self.trans.write(json.dumps(string))
+
+ def writeJSONNumber(self, number):
+ self.context.write()
+ jsNumber = str(number)
+ if self.context.escapeNum():
+ jsNumber = "%s%s%s" % (QUOTE, jsNumber, QUOTE)
+ self.trans.write(jsNumber)
+
+ def writeJSONBase64(self, binary):
+ self.context.write()
+ self.trans.write(QUOTE)
+ self.trans.write(base64.b64encode(binary))
+ self.trans.write(QUOTE)
+
+ def writeJSONObjectStart(self):
+ self.context.write()
+ self.trans.write(LBRACE)
+ self.pushContext(JSONPairContext(self))
+
+ def writeJSONObjectEnd(self):
+ self.popContext()
+ self.trans.write(RBRACE)
+
+ def writeJSONArrayStart(self):
+ self.context.write()
+ self.trans.write(LBRACKET)
+ self.pushContext(JSONListContext(self))
+
+ def writeJSONArrayEnd(self):
+ self.popContext()
+ self.trans.write(RBRACKET)
+
+ def readJSONSyntaxChar(self, character):
+ current = self.reader.read()
+ if character != current:
+ raise TProtocolException(TProtocolException.INVALID_DATA,
+ "Unexpected character: %s" % current)
+
+ def readJSONString(self, skipContext):
+ string = []
+ if skipContext is False:
+ self.context.read()
+ self.readJSONSyntaxChar(QUOTE)
+ while True:
+ character = self.reader.read()
+ if character == QUOTE:
+ break
+ if character == ESCSEQ[0]:
+ character = self.reader.read()
+ if character == ESCSEQ[1]:
+ self.readJSONSyntaxChar(ZERO)
+ self.readJSONSyntaxChar(ZERO)
+ character = json.JSONDecoder().decode('"\u00%s"' % self.trans.read(2))
+ else:
+ off = ESCAPE_CHAR.find(char)
+ if off == -1:
+ raise TProtocolException(TProtocolException.INVALID_DATA,
+ "Expected control char")
+ character = ESCAPE_CHAR_VALS[off]
+ string.append(character)
+ return ''.join(string)
+
+ def isJSONNumeric(self, character):
+ return (True if NUMERIC_CHAR.find(character) != - 1 else False)
+
+ def readJSONQuotes(self):
+ if (self.context.escapeNum()):
+ self.readJSONSyntaxChar(QUOTE)
+
+ def readJSONNumericChars(self):
+ numeric = []
+ while True:
+ character = self.reader.peek()
+ if self.isJSONNumeric(character) is False:
+ break
+ numeric.append(self.reader.read())
+ return ''.join(numeric)
+
+ def readJSONInteger(self):
+ self.context.read()
+ self.readJSONQuotes()
+ numeric = self.readJSONNumericChars()
+ self.readJSONQuotes()
+ try:
+ return int(numeric)
+ except ValueError:
+ raise TProtocolException(TProtocolException.INVALID_DATA,
+ "Bad data encounted in numeric data")
+
+ def readJSONDouble(self):
+ self.context.read()
+ if self.reader.peek() == QUOTE:
+ string = self.readJSONString(True)
+ try:
+ double = float(string)
+ if self.context.escapeNum is False and double != inf and double != nan:
+ raise TProtocolException(TProtocolException.INVALID_DATA,
+ "Numeric data unexpectedly quoted")
+ return double
+ except ValueError:
+ raise TProtocolException(TProtocolException.INVALID_DATA,
+ "Bad data encounted in numeric data")
+ else:
+ if self.context.escapeNum() is True:
+ self.readJSONSyntaxChar(QUOTE)
+ try:
+ return float(self.readJSONNumericChars())
+ except ValueErro:
+ raise TProtocolException(TProtocolException.INVALID_DATA,
+ "Bad data encounted in numeric data")
+
+ def readJSONBase64(self):
+ string = self.readJSONString(False)
+ return base64.b64decode(string)
+
+ def readJSONObjectStart(self):
+ self.context.read()
+ self.readJSONSyntaxChar(LBRACE)
+ self.pushContext(JSONPairContext(self))
+
+ def readJSONObjectEnd(self):
+ self.readJSONSyntaxChar(RBRACE)
+ self.popContext()
+
+ def readJSONArrayStart(self):
+ self.context.read()
+ self.readJSONSyntaxChar(LBRACKET)
+ self.pushContext(JSONListContext(self))
+
+ def readJSONArrayEnd(self):
+ self.readJSONSyntaxChar(RBRACKET)
+ self.popContext()
+
+
+class TJSONProtocol(TJSONProtocolBase):
+
+ def readMessageBegin(self):
+ self.resetReadContext()
+ self.readJSONArrayStart()
+ if self.readJSONInteger() != VERSION:
+ raise TProtocolException(TProtocolException.BAD_VERSION,
+ "Message contained bad version.")
+ name = self.readJSONString(False)
+ typen = self.readJSONInteger()
+ seqid = self.readJSONInteger()
+ return (name, typen, seqid)
+
+ def readMessageEnd(self):
+ self.readJSONArrayEnd()
+
+ def readStructBegin(self):
+ self.readJSONObjectStart()
+
+ def readStructEnd(self):
+ self.readJSONObjectEnd()
+
+ def readFieldBegin(self):
+ character = self.reader.peek()
+ type = 0
+ id = 0
+ if character == RBRACE:
+ type = TType.STOP
+ else:
+ id = self.readJSONInteger()
+ self.readJSONObjectStart()
+ type = JTYPES[self.readJSONString(False)]
+ return (None, type, id)
+
+ def readFieldEnd(self):
+ self.readJSONObjectEnd()
+
+ def readMapBegin(self):
+ self.readJSONArrayStart()
+ keyType = JTYPES[self.readJSONString(False)]
+ valueType = JTYPES[self.readJSONString(False)]
+ size = self.readJSONInteger()
+ self.readJSONObjectStart()
+ return (keyType, valueType, size)
+
+ def readMapEnd(self):
+ self.readJSONObjectEnd()
+ self.readJSONArrayEnd()
+
+ def readCollectionBegin(self):
+ self.readJSONArrayStart()
+ elemType = JTYPES[self.readJSONString(False)]
+ size = self.readJSONInteger()
+ return (type, size)
+ readListBegin = readCollectionBegin
+ readSetBegin = readCollectionBegin
+
+ def readCollectionEnd(self):
+ self.readJSONArrayEnd()
+ readSetEnd = readCollectionEnd
+ readListEnd = readCollectionEnd
+
+ def readBool(self):
+ return (False if self.readJSONInteger() == 0 else True)
+
+ def readNumber(self):
+ return self.readJSONInteger()
+ readByte = readNumber
+ readI16 = readNumber
+ readI32 = readNumber
+ readI64 = readNumber
+
+ def readDouble(self):
+ return self.readJSONDouble()
+
+ def readString(self):
+ return self.readJSONString(False)
+
+ def readBinary(self):
+ return self.readJSONBase64()
+
+ def writeMessageBegin(self, name, request_type, seqid):
+ self.resetWriteContext()
+ self.writeJSONArrayStart()
+ self.writeJSONNumber(VERSION)
+ self.writeJSONString(name)
+ self.writeJSONNumber(request_type)
+ self.writeJSONNumber(seqid)
+
+ def writeMessageEnd(self):
+ self.writeJSONArrayEnd()
+
+ def writeStructBegin(self, name):
+ self.writeJSONObjectStart()
+
+ def writeStructEnd(self):
+ self.writeJSONObjectEnd()
+
+ def writeFieldBegin(self, name, type, id):
+ self.writeJSONNumber(id)
+ self.writeJSONObjectStart()
+ self.writeJSONString(CTYPES[type])
+
+ def writeFieldEnd(self):
+ self.writeJSONObjectEnd()
+
+ def writeFieldStop(self):
+ pass
+
+ def writeMapBegin(self, ktype, vtype, size):
+ self.writeJSONArrayStart()
+ self.writeJSONString(CTYPES[ktype])
+ self.writeJSONString(CTYPES[vtype])
+ self.writeJSONNumber(size)
+ self.writeJSONObjectStart()
+
+ def writeMapEnd(self):
+ self.writeJSONObjectEnd()
+ self.writeJSONArrayEnd()
+
+ def writeListBegin(self, etype, size):
+ self.writeJSONArrayStart()
+ self.writeJSONString(CTYPES[etype])
+ self.writeJSONNumber(size)
+
+ def writeListEnd(self):
+ self.writeJSONArrayEnd()
+
+ def writeSetBegin(self, etype, size):
+ self.writeJSONArrayStart()
+ self.writeJSONString(CTYPES[etype])
+ self.writeJSONNumber(size)
+
+ def writeSetEnd(self):
+ self.writeJSONArrayEnd()
+
+ def writeBool(self, boolean):
+ self.writeJSONNumber(1 if boolean is True else 0)
+
+ def writeInteger(self, integer):
+ self.writeJSONNumber(integer)
+ writeByte = writeInteger
+ writeI16 = writeInteger
+ writeI32 = writeInteger
+ writeI64 = writeInteger
+
+ def writeDouble(self, dbl):
+ self.writeJSONNumber(dbl)
+
+ def writeString(self, string):
+ self.writeJSONString(string)
+
+ def writeBinary(self, binary):
+ self.writeJSONBase64(binary)
+
+class TJSONProtocolFactory:
+ def __init__(self):
+ pass
+
+ def getProtocol(self, trans):
+ return TJSONProtocol(trans)
diff --git a/lib/py/src/protocol/__init__.py b/lib/py/src/protocol/__init__.py
index d53359b..7eefb45 100644
--- a/lib/py/src/protocol/__init__.py
+++ b/lib/py/src/protocol/__init__.py
@@ -17,4 +17,4 @@
# under the License.
#
-__all__ = ['TProtocol', 'TBinaryProtocol', 'fastbinary', 'TBase']
+__all__ = ['fastbinary', 'TBase', 'TBinaryProtocol', 'TCompactProtocol', 'TJSONProtocol', 'TProtocol']
diff --git a/test/py/RunClientServer.py b/test/py/RunClientServer.py
index 8a7fda6..f9121c8 100755
--- a/test/py/RunClientServer.py
+++ b/test/py/RunClientServer.py
@@ -55,7 +55,9 @@
PROTOS= [
'accel',
'binary',
- 'compact' ]
+ 'compact']
+# FIXME: add json
+# disabled because json HTTP test hangs... why?
SERVERS = [
"TSimpleServer",
diff --git a/test/py/TestClient.py b/test/py/TestClient.py
index 71001b1..471e030 100755
--- a/test/py/TestClient.py
+++ b/test/py/TestClient.py
@@ -19,8 +19,8 @@
# under the License.
#
-import sys, glob
-sys.path.insert(0, glob.glob('../../lib/py/build/lib.*')[0])
+import sys, glob, os
+sys.path.insert(0, glob.glob(os.path.join(os.path.dirname(__file__),'../../lib/py/build/lib.*'))[0])
import unittest
import time
@@ -63,6 +63,7 @@
from thrift.transport import TZlibTransport
from thrift.protocol import TBinaryProtocol
from thrift.protocol import TCompactProtocol
+from thrift.protocol import TJSONProtocol
class AbstractTest(unittest.TestCase):
def setUp(self):
@@ -125,7 +126,7 @@
def testNest(self):
inner = Xtruct(string_thing="Zero", byte_thing=1, i32_thing=-3,
i64_thing=-5)
- x = Xtruct2(struct_thing=inner)
+ x = Xtruct2(struct_thing=inner, byte_thing=0, i32_thing=0)
y = self.client.testNest(x)
self.assertEqual(y, x)
@@ -163,13 +164,13 @@
pass
def testMulti(self):
- xpected = Xtruct(byte_thing=74, i32_thing=0xff00ff, i64_thing=0xffffffffd0d0)
+ xpected = Xtruct(string_thing='Hello2', byte_thing=74, i32_thing=0xff00ff, i64_thing=0xffffffffd0d0)
y = self.client.testMulti(xpected.byte_thing,
xpected.i32_thing,
xpected.i64_thing,
{ 0:'abc' },
Numberz.FIVE,
- 0xf0f0f0)
+ 0xf0f0f0)
self.assertEqual(y, xpected)
def testException(self):
@@ -208,6 +209,9 @@
class CompactTest(AbstractTest):
protocol_factory = TCompactProtocol.TCompactProtocolFactory()
+class JSONTest(AbstractTest):
+ protocol_factory = TJSONProtocol.TJSONProtocolFactory()
+
class AcceleratedBinaryTest(AbstractTest):
protocol_factory = TBinaryProtocol.TBinaryProtocolAcceleratedFactory()
@@ -220,6 +224,8 @@
suite.addTest(loader.loadTestsFromTestCase(AcceleratedBinaryTest))
elif options.proto == 'compact':
suite.addTest(loader.loadTestsFromTestCase(CompactTest))
+ elif options.proto == 'json':
+ suite.addTest(loader.loadTestsFromTestCase(JSONTest))
else:
raise AssertionError('Unknown protocol given with --proto: %s' % options.proto)
return suite
diff --git a/test/py/TestServer.py b/test/py/TestServer.py
index 6f4af44..1eae097 100755
--- a/test/py/TestServer.py
+++ b/test/py/TestServer.py
@@ -19,8 +19,8 @@
# under the License.
#
from __future__ import division
-import sys, glob, time
-sys.path.insert(0, glob.glob('../../lib/py/build/lib.*')[0])
+import sys, glob, time, os
+sys.path.insert(0, glob.glob(os.path.join(os.path.dirname(__file__),'../../lib/py/build/lib.*'))[0])
from optparse import OptionParser
parser = OptionParser()
@@ -40,7 +40,7 @@
dest="verbose", const=0,
help="minimal output")
parser.add_option('--proto', dest="proto", type="string",
- help="protocol to use, one of: accel, binary, compact")
+ help="protocol to use, one of: accel, binary, compact, json")
parser.set_defaults(port=9090, verbose=1, proto='binary')
options, args = parser.parse_args()
@@ -53,11 +53,13 @@
from thrift.transport import TZlibTransport
from thrift.protocol import TBinaryProtocol
from thrift.protocol import TCompactProtocol
+from thrift.protocol import TJSONProtocol
from thrift.server import TServer, TNonblockingServer, THttpServer
PROT_FACTORIES = {'binary': TBinaryProtocol.TBinaryProtocolFactory,
'accel': TBinaryProtocol.TBinaryProtocolAcceleratedFactory,
- 'compact': TCompactProtocol.TCompactProtocolFactory}
+ 'compact': TCompactProtocol.TCompactProtocolFactory,
+ 'json': TJSONProtocol.TJSONProtocolFactory}
class TestHandler:
@@ -156,7 +158,7 @@
def testMulti(self, arg0, arg1, arg2, arg3, arg4, arg5):
if options.verbose > 1:
print 'testMulti(%s)' % [arg0, arg1, arg2, arg3, arg4, arg5]
- x = Xtruct(byte_thing=arg0, i32_thing=arg1, i64_thing=arg2)
+ x = Xtruct(string_thing='Hello2', byte_thing=arg0, i32_thing=arg1, i64_thing=arg2)
return x
# set up the protocol factory form the --proto option
diff --git a/test/test.sh b/test/test.sh
index 30ca838..36e57c3 100755
--- a/test/test.sh
+++ b/test/test.sh
@@ -55,6 +55,7 @@
echo "=================== client message ==================="
tail log/${testname}_client.log
echo "======================================================"
+ echo ""
print_header
fi
sleep 10
@@ -93,7 +94,42 @@
done;
done;
-
+do_test "py-py" "binary" "buffered-ip" \
+ "py/TestClient.py --proto=binary --port=9090 --host=localhost --genpydir=py/gen-py" \
+ "py/TestServer.py --proto=binary --port=9090 --genpydir=py/gen-py TSimpleServer" \
+ "10"
+do_test "py-py" "json" "buffered-ip" \
+ "py/TestClient.py --proto=json --port=9090 --host=localhost --genpydir=py/gen-py" \
+ "py/TestServer.py --proto=json --port=9090 --genpydir=py/gen-py TSimpleServer" \
+ "10"
+do_test "py-cpp" "binary" "buffered-ip" \
+ "py/TestClient.py --proto=binary --port=9090 --host=localhost --genpydir=py/gen-py" \
+ "cpp/TestServer" \
+ "10"
+do_test "py-cpp" "json" "buffered-ip" \
+ "py/TestClient.py --proto=json --port=9090 --host=localhost --genpydir=py/gen-py" \
+ "cpp/TestServer --protocol=json" \
+ "10"
+do_test "cpp-py" "binary" "buffered-ip" \
+ "cpp/TestClient --protocol=binary --port=9090" \
+ "py/TestServer.py --proto=binary --port=9090 --genpydir=py/gen-py TSimpleServer" \
+ "10"
+do_test "cpp-py" "json" "buffered-ip" \
+ "cpp/TestClient --protocol=json --port=9090" \
+ "py/TestServer.py --proto=json --port=9090 --genpydir=py/gen-py TSimpleServer" \
+ "10"
+do_test "py-java" "binary" "buffered-ip" \
+ "py/TestClient.py --proto=binary --port=9090 --host=localhost --genpydir=py/gen-py" \
+ "ant -f ../lib/java/build.xml testserver" \
+ "100"
+do_test "py-java" "json" "buffered-ip" \
+ "py/TestClient.py --proto=json --port=9090 --host=localhost --genpydir=py/gen-py" \
+ "ant -f ../lib/java/build.xml testserver" \
+ "100"
+do_test "java-py" "binary" "buffered-ip" \
+ "ant -f ../lib/java/build.xml testclient" \
+ "py/TestServer.py --proto=binary --port=9090 --genpydir=py/gen-py TSimpleServer" \
+ "10"
do_test "java-java" "binary" "buffered-ip" \
"ant -f ../lib/java/build.xml testclient" \
"ant -f ../lib/java/build.xml testserver" \