THRIFT-3637 Implement compact protocol for dart
This closes #916
diff --git a/lib/dart/lib/src/protocol/t_compact_protocol.dart b/lib/dart/lib/src/protocol/t_compact_protocol.dart
new file mode 100644
index 0000000..c5dc515
--- /dev/null
+++ b/lib/dart/lib/src/protocol/t_compact_protocol.dart
@@ -0,0 +1,470 @@
+/// Licensed to the Apache Software Foundation (ASF) under one
+/// or more contributor license agreements. See the NOTICE file
+/// distributed with this work for additional information
+/// regarding copyright ownership. The ASF licenses this file
+/// to you under the Apache License, Version 2.0 (the
+/// 'License'); you may not use this file except in compliance
+/// with the License. You may obtain a copy of the License at
+///
+/// http://www.apache.org/licenses/LICENSE-2.0
+///
+/// Unless required by applicable law or agreed to in writing,
+/// software distributed under the License is distributed on an
+/// 'AS IS' BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+/// KIND, either express or implied. See the License for the
+/// specific language governing permissions and limitations
+/// under the License.
+
+part of thrift;
+
+class TCompactProtocolFactory implements TProtocolFactory<TCompactProtocol> {
+ TCompactProtocolFactory();
+
+ TCompactProtocol getProtocol(TTransport transport) {
+ return new TCompactProtocol(transport);
+ }
+}
+
+/// Compact protocol implementation for Thrift.
+///
+/// Use of fixnum library is required due to bugs like
+/// https://github.com/dart-lang/sdk/issues/15361
+///
+/// Adapted from the Java version.
+class TCompactProtocol extends TProtocol {
+ static const int PROTOCOL_ID = 0x82;
+ static const int VERSION = 1;
+ static const int VERSION_MASK = 0x1f;
+ static const int TYPE_MASK = 0xE0;
+ static const int TYPE_BITS = 0x07;
+ static const int TYPE_SHIFT_AMOUNT = 5;
+ static final TField TSTOP = new TField("", TType.STOP, 0);
+
+ static const int TYPE_BOOLEAN_TRUE = 0x01;
+ static const int TYPE_BOOLEAN_FALSE = 0x02;
+ static const int TYPE_BYTE = 0x03;
+ static const int TYPE_I16 = 0x04;
+ static const int TYPE_I32 = 0x05;
+ static const int TYPE_I64 = 0x06;
+ static const int TYPE_DOUBLE = 0x07;
+ static const int TYPE_BINARY = 0x08;
+ static const int TYPE_LIST = 0x09;
+ static const int TYPE_SET = 0x0A;
+ static const int TYPE_MAP = 0x0B;
+ static const int TYPE_STRUCT = 0x0C;
+
+ static final List<int> _typeMap = new List.unmodifiable(new List(16)
+ ..[TType.STOP] = TType.STOP
+ ..[TType.BOOL] = TYPE_BOOLEAN_TRUE
+ ..[TType.BYTE] = TYPE_BYTE
+ ..[TType.I16] = TYPE_I16
+ ..[TType.I32] = TYPE_I32
+ ..[TType.I64] = TYPE_I64
+ ..[TType.DOUBLE] = TYPE_DOUBLE
+ ..[TType.STRING] = TYPE_BINARY
+ ..[TType.LIST] = TYPE_LIST
+ ..[TType.SET] = TYPE_SET
+ ..[TType.MAP] = TYPE_MAP
+ ..[TType.STRUCT] = TYPE_STRUCT);
+
+ static const Utf8Codec _utf8Codec = const Utf8Codec();
+
+ // Pretend this is a stack
+ DoubleLinkedQueue<int> _lastField = new DoubleLinkedQueue<int>();
+ int _lastFieldId = 0;
+
+ TField _booleanField = null;
+ bool _boolValue = null;
+
+ final Uint8List tempList = new Uint8List(10);
+ final ByteData tempBD = new ByteData(10);
+
+ TCompactProtocol(TTransport transport) : super(transport);
+
+ /// Write
+ void writeMessageBegin(TMessage message) {
+ writeByte(PROTOCOL_ID);
+ writeByte((VERSION & VERSION_MASK) |
+ ((message.type << TYPE_SHIFT_AMOUNT) & TYPE_MASK));
+ _writeVarInt32(new Int32(message.seqid));
+ writeString(message.name);
+ }
+
+ void writeMessageEnd() {}
+
+ void writeStructBegin(TStruct struct) {
+ _lastField.addLast(_lastFieldId);
+ _lastFieldId = 0;
+ }
+
+ void writeStructEnd() {
+ _lastFieldId = _lastField.removeLast();
+ }
+
+ void writeFieldBegin(TField field) {
+ if (field.type == TType.BOOL) {
+ _booleanField = field;
+ } else {
+ _writeFieldBegin(field, -1);
+ }
+ }
+
+ void _writeFieldBegin(TField field, int typeOverride) {
+ int typeToWrite =
+ typeOverride == -1 ? _getCompactType(field.type) : typeOverride;
+
+ if (field.id > _lastFieldId && field.id - _lastFieldId <= 15) {
+ writeByte((field.id - _lastFieldId) << 4 | typeToWrite);
+ } else {
+ writeByte(typeToWrite);
+ writeI16(field.id);
+ }
+
+ _lastFieldId = field.id;
+ }
+
+ void writeFieldEnd() {}
+
+ void writeFieldStop() {
+ writeByte(TType.STOP);
+ }
+
+ void writeMapBegin(TMap map) {
+ if (map.length == 0) {
+ writeByte(0);
+ } else {
+ _writeVarInt32(new Int32(map.length));
+ writeByte(
+ _getCompactType(map.keyType) << 4 | _getCompactType(map.valueType));
+ }
+ }
+
+ void writeMapEnd() {}
+
+ void writeListBegin(TList list) {
+ _writeCollectionBegin(list.elementType, list.length);
+ }
+
+ void writeListEnd() {}
+
+ void writeSetBegin(TSet set) {
+ _writeCollectionBegin(set.elementType, set.length);
+ }
+
+ void writeSetEnd() {}
+
+ void writeBool(bool b) {
+ if (b == null) b = false;
+ if (_booleanField != null) {
+ _writeFieldBegin(
+ _booleanField, b ? TYPE_BOOLEAN_TRUE : TYPE_BOOLEAN_FALSE);
+ _booleanField = null;
+ } else {
+ writeByte(b ? TYPE_BOOLEAN_TRUE : TYPE_BOOLEAN_FALSE);
+ }
+ }
+
+ void writeByte(int b) {
+ if (b == null) b = 0;
+ tempList[0] = b;
+ transport.write(tempList, 0, 1);
+ }
+
+ void writeI16(int i16) {
+ if (i16 == null) i16 = 0;
+ _writeVarInt32(_int32ToZigZag(new Int32(i16)));
+ }
+
+ void writeI32(int i32) {
+ if (i32 == null) i32 = 0;
+ _writeVarInt32(_int32ToZigZag(new Int32(i32)));
+ }
+
+ void writeI64(int i64) {
+ if (i64 == null) i64 = 0;
+ _writeVarInt64(_int64ToZigZag(new Int64(i64)));
+ }
+
+ void writeDouble(double d) {
+ if (d == null) d = 0.0;
+ tempBD.setFloat64(0, d);
+ transport.write(tempBD.buffer.asUint8List(), 0, 8);
+ }
+
+ void writeString(String str) {
+ Uint8List bytes =
+ str != null ? _utf8Codec.encode(str) : new Uint8List.fromList([]);
+ writeBinary(bytes);
+ }
+
+ void writeBinary(Uint8List bytes) {
+ _writeVarInt32(new Int32(bytes.length));
+ transport.write(bytes, 0, bytes.length);
+ }
+
+ void _writeVarInt32(Int32 n) {
+ int idx = 0;
+ while (true) {
+ if ((n & ~0x7F) == 0) {
+ tempList[idx++] = (n & 0xFF).toInt();
+ break;
+ } else {
+ tempList[idx++] = (((n & 0x7F) | 0x80) & 0xFF).toInt();
+ n = n.shiftRightUnsigned(7);
+ }
+ }
+ transport.write(tempList, 0, idx);
+ }
+
+ void _writeVarInt64(Int64 n) {
+ int idx = 0;
+ while (true) {
+ if ((n & ~0x7F) == 0) {
+ tempList[idx++] = (n & 0xFF).toInt();
+ break;
+ } else {
+ tempList[idx++] = (((n & 0x7F) | 0x80) & 0xFF).toInt();
+ n = n.shiftRightUnsigned(7);
+ }
+ }
+ transport.write(tempList, 0, idx);
+ }
+
+ void _writeCollectionBegin(int elemType, int length) {
+ if (length <= 14) {
+ writeByte(length << 4 | _getCompactType(elemType));
+ } else {
+ writeByte(0xF0 | _getCompactType(elemType));
+ _writeVarInt32(new Int32(length));
+ }
+ }
+
+ Int32 _int32ToZigZag(Int32 n) {
+ return (n << 1) ^ (n >> 31);
+ }
+
+ Int64 _int64ToZigZag(Int64 n) {
+ return (n << 1) ^ (n >> 63);
+ }
+
+ /// Read
+ TMessage readMessageBegin() {
+ int protocolId = readByte();
+ if (protocolId != PROTOCOL_ID) {
+ throw new TProtocolError(TProtocolErrorType.BAD_VERSION,
+ 'Expected protocol id $PROTOCOL_ID but got $protocolId');
+ }
+ int versionAndType = readByte();
+ int version = versionAndType & VERSION_MASK;
+ if (version != VERSION) {
+ throw new TProtocolError(TProtocolErrorType.BAD_VERSION,
+ 'Expected version $VERSION but got $version');
+ }
+ int type = (versionAndType >> TYPE_SHIFT_AMOUNT) & TYPE_BITS;
+ int seqId = _readVarInt32().toInt();
+ String messageName = readString();
+ return new TMessage(messageName, type, seqId);
+ }
+
+ void readMessageEnd() {}
+
+ TStruct readStructBegin() {
+ _lastField.addLast(_lastFieldId);
+ _lastFieldId = 0;
+ // TODO make this a constant?
+ return new TStruct();
+ }
+
+ void readStructEnd() {
+ _lastFieldId = _lastField.removeLast();
+ }
+
+ TField readFieldBegin() {
+ int type = readByte();
+ if (type == TType.STOP) {
+ return TSTOP;
+ }
+
+ int fieldId;
+ int modifier = (type & 0xF0) >> 4;
+ if (modifier == 0) {
+ fieldId = readI16();
+ } else {
+ fieldId = _lastFieldId + modifier;
+ }
+
+ TField field = new TField('', _getTType(type & 0x0F), fieldId);
+ if (_isBoolType(type)) {
+ _boolValue = (type & 0x0F) == TYPE_BOOLEAN_TRUE;
+ }
+
+ _lastFieldId = field.id;
+ return field;
+ }
+
+ void readFieldEnd() {}
+
+ TMap readMapBegin() {
+ int length = _readVarInt32().toInt();
+ _checkNegReadLength(length);
+
+ int keyAndValueType = length == 0 ? 0 : readByte();
+ int keyType = _getTType(keyAndValueType >> 4);
+ int valueType = _getTType(keyAndValueType & 0x0F);
+ return new TMap(keyType, valueType, length);
+ }
+
+ void readMapEnd() {}
+
+ TList readListBegin() {
+ int lengthAndType = readByte();
+ int length = (lengthAndType >> 4) & 0x0F;
+ if (length == 15) {
+ length = _readVarInt32().toInt();
+ }
+ _checkNegReadLength(length);
+ int type = _getTType(lengthAndType);
+ return new TList(type, length);
+ }
+
+ void readListEnd() {}
+
+ TSet readSetBegin() {
+ TList tlist = readListBegin();
+ return new TSet(tlist.elementType, tlist.length);
+ }
+
+ void readSetEnd() {}
+
+ bool readBool() {
+ if (_boolValue != null) {
+ bool result = _boolValue;
+ _boolValue = null;
+ return result;
+ }
+ return readByte() == TYPE_BOOLEAN_TRUE;
+ }
+
+ int readByte() {
+ transport.readAll(tempList, 0, 1);
+ return tempList.buffer.asByteData().getUint8(0);
+ }
+
+ int readI16() {
+ return _zigzagToInt32(_readVarInt32()).toInt();
+ }
+
+ int readI32() {
+ return _zigzagToInt32(_readVarInt32()).toInt();
+ }
+
+ int readI64() {
+ return _zigzagToInt64(_readVarInt64()).toInt();
+ }
+
+ double readDouble() {
+ transport.readAll(tempList, 0, 8);
+ return tempList.buffer.asByteData().getFloat64(0);
+ }
+
+ String readString() {
+ int length = _readVarInt32().toInt();
+ _checkNegReadLength(length);
+
+ // TODO look at using temp for small strings?
+ Uint8List buff = new Uint8List(length);
+ transport.readAll(buff, 0, length);
+ return _utf8Codec.decode(buff);
+ }
+
+ Uint8List readBinary() {
+ int length = _readVarInt32().toInt();
+ _checkNegReadLength(length);
+
+ Uint8List buff = new Uint8List(length);
+ transport.readAll(buff, 0, length);
+ return buff;
+ }
+
+ Int32 _readVarInt32() {
+ Int32 result = Int32.ZERO;
+ int shift = 0;
+ while (true) {
+ Int32 b = new Int32(readByte());
+ result |= (b & 0x7f) << shift;
+ if ((b & 0x80) != 0x80) break;
+ shift += 7;
+ }
+ return result;
+ }
+
+ Int64 _readVarInt64() {
+ Int64 result = Int64.ZERO;
+ int shift = 0;
+ while (true) {
+ Int64 b = new Int64(readByte());
+ result |= (b & 0x7f) << shift;
+ if ((b & 0x80) != 0x80) break;
+ shift += 7;
+ }
+ return result;
+ }
+
+ Int32 _zigzagToInt32(Int32 n) {
+ return (n.shiftRightUnsigned(1)) ^ -(n & 1);
+ }
+
+ Int64 _zigzagToInt64(Int64 n) {
+ return (n.shiftRightUnsigned(1)) ^ -(n & 1);
+ }
+
+ void _checkNegReadLength(int length) {
+ if (length < 0) {
+ throw new TProtocolError(
+ TProtocolErrorType.NEGATIVE_SIZE, 'Negative length: $length');
+ }
+ }
+
+ int _getCompactType(int ttype) {
+ return _typeMap[ttype];
+ }
+
+ int _getTType(int type) {
+ switch (type & 0x0F) {
+ case TType.STOP:
+ return TType.STOP;
+ case TYPE_BOOLEAN_FALSE:
+ case TYPE_BOOLEAN_TRUE:
+ return TType.BOOL;
+ case TYPE_BYTE:
+ return TType.BYTE;
+ case TYPE_I16:
+ return TType.I16;
+ case TYPE_I32:
+ return TType.I32;
+ case TYPE_I64:
+ return TType.I64;
+ case TYPE_DOUBLE:
+ return TType.DOUBLE;
+ case TYPE_BINARY:
+ return TType.STRING;
+ case TYPE_LIST:
+ return TType.LIST;
+ case TYPE_SET:
+ return TType.SET;
+ case TYPE_MAP:
+ return TType.MAP;
+ case TYPE_STRUCT:
+ return TType.STRUCT;
+ default:
+ throw new TProtocolError(
+ TProtocolErrorType.INVALID_DATA, "Unknown type: ${type & 0x0F}");
+ }
+ }
+
+ bool _isBoolType(int b) {
+ int lowerNibble = b & 0x0F;
+ return lowerNibble == TYPE_BOOLEAN_TRUE ||
+ lowerNibble == TYPE_BOOLEAN_FALSE;
+ }
+}
diff --git a/lib/dart/lib/thrift.dart b/lib/dart/lib/thrift.dart
index 2483726..27eb546 100644
--- a/lib/dart/lib/thrift.dart
+++ b/lib/dart/lib/thrift.dart
@@ -18,11 +18,13 @@
library thrift;
import 'dart:async';
+import 'dart:collection';
import 'dart:convert' show Utf8Codec;
import 'dart:typed_data' show ByteData;
import 'dart:typed_data' show Uint8List;
import 'package:crypto/crypto.dart' show CryptoUtils;
+import 'package:fixnum/fixnum.dart';
import 'package:http/http.dart' show Client;
import 'package:logging/logging.dart';
@@ -32,6 +34,7 @@
part 'src/t_processor.dart';
part 'src/protocol/t_binary_protocol.dart';
+part 'src/protocol/t_compact_protocol.dart';
part 'src/protocol/t_field.dart';
part 'src/protocol/t_json_protocol.dart';
part 'src/protocol/t_list.dart';
diff --git a/lib/dart/pubspec.yaml b/lib/dart/pubspec.yaml
index 99d490c..7012f5d 100644
--- a/lib/dart/pubspec.yaml
+++ b/lib/dart/pubspec.yaml
@@ -26,6 +26,7 @@
sdk: ">=1.12.0 <2.0.0"
dependencies:
crypto: "^0.9.0"
+ fixnum: "^0.10.2"
http: "^0.11.3"
logging: "^0.11.0"
dev_dependencies:
diff --git a/lib/dart/test/protocol/t_protocol_test.dart b/lib/dart/test/protocol/t_protocol_test.dart
index 7362884..0e6cde5 100644
--- a/lib/dart/test/protocol/t_protocol_test.dart
+++ b/lib/dart/test/protocol/t_protocol_test.dart
@@ -353,7 +353,6 @@
protocol.writeMessageBegin(message);
});
-
test('Test escaped unicode', () async {
/*
KOR_KAI
@@ -372,7 +371,8 @@
await protocol.transport.flush();
var subject = protocol.readString();
- expect(subject, UTF8.decode([0x01, 0xE0, 0xB8, 0x81, 0x20, 0xF0, 0x9D, 0x84, 0x9E]));
+ expect(subject,
+ UTF8.decode([0x01, 0xE0, 0xB8, 0x81, 0x20, 0xF0, 0x9D, 0x84, 0x9E]));
});
group('shared tests', sharedTests);
@@ -386,6 +386,15 @@
group('shared tests', sharedTests);
});
+
+ group('compact', () {
+ setUp(() {
+ protocol = new TCompactProtocol(new TBufferedTransport());
+ protocol.writeMessageBegin(message);
+ });
+
+ group('shared tests', sharedTests);
+ });
}
class Primitive {
diff --git a/test/dart/test_client/bin/main.dart b/test/dart/test_client/bin/main.dart
index 5ad3cde..2f5bb70 100644
--- a/test/dart/test_client/bin/main.dart
+++ b/test/dart/test_client/bin/main.dart
@@ -112,9 +112,13 @@
});
parser.addOption('protocol',
defaultsTo: 'binary',
- allowed: ['binary', 'json'],
+ allowed: ['binary', 'compact', 'json'],
help: 'The protocol name',
- allowedHelp: {'binary': 'TBinaryProtocol', 'json': 'TJsonProtocol'});
+ allowedHelp: {
+ 'binary': 'TBinaryProtocol',
+ 'compact': 'TCompactProtocol',
+ 'json': 'TJsonProtocol'
+ });
parser.addFlag('verbose', defaultsTo: false);
ArgResults results;
@@ -132,6 +136,8 @@
TProtocolFactory getProtocolFactory(String protocolType) {
if (protocolType == 'binary') {
return new TBinaryProtocolFactory();
+ } else if (protocolType == 'compact') {
+ return new TCompactProtocolFactory();
} else if (protocolType == 'json') {
return new TJsonProtocolFactory();
}
diff --git a/test/tests.json b/test/tests.json
index 04caaa0..12dcd54 100644
--- a/test/tests.json
+++ b/test/tests.json
@@ -406,6 +406,7 @@
],
"protocols": [
"binary",
+ "compact",
"json"
],
"command": [