diff options
author | Mark Erickson <merickson@apache.org> | 2016-02-29 15:27:59 -0600 |
---|---|---|
committer | Mark Erickson <merickson@apache.org> | 2016-02-29 15:27:59 -0600 |
commit | b5f126f6d6908796944569cc4c5a5ae2c6614231 (patch) | |
tree | 41223cb580f0f8f57b46334bf2aef2134a594e7e /lib/dart | |
parent | b52d03d7a0c1ff2c1fe31c3999127d2daf98bc93 (diff) | |
download | thrift-b5f126f6d6908796944569cc4c5a5ae2c6614231.tar.gz |
THRIFT-3637 Implement compact protocol for dart
This closes #916
Diffstat (limited to 'lib/dart')
-rw-r--r-- | lib/dart/lib/src/protocol/t_compact_protocol.dart | 470 | ||||
-rw-r--r-- | lib/dart/lib/thrift.dart | 3 | ||||
-rw-r--r-- | lib/dart/pubspec.yaml | 1 | ||||
-rw-r--r-- | lib/dart/test/protocol/t_protocol_test.dart | 13 |
4 files changed, 485 insertions, 2 deletions
diff --git a/lib/dart/lib/src/protocol/t_compact_protocol.dart b/lib/dart/lib/src/protocol/t_compact_protocol.dart new file mode 100644 index 000000000..c5dc51500 --- /dev/null +++ b/lib/dart/lib/src/protocol/t_compact_protocol.dart @@ -0,0 +1,470 @@ +/// Licensed to the Apache Software Foundation (ASF) under one +/// or more contributor license agreements. See the NOTICE file +/// distributed with this work for additional information +/// regarding copyright ownership. The ASF licenses this file +/// to you under the Apache License, Version 2.0 (the +/// 'License'); you may not use this file except in compliance +/// with the License. You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, +/// software distributed under the License is distributed on an +/// 'AS IS' BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +/// KIND, either express or implied. See the License for the +/// specific language governing permissions and limitations +/// under the License. + +part of thrift; + +class TCompactProtocolFactory implements TProtocolFactory<TCompactProtocol> { + TCompactProtocolFactory(); + + TCompactProtocol getProtocol(TTransport transport) { + return new TCompactProtocol(transport); + } +} + +/// Compact protocol implementation for Thrift. +/// +/// Use of fixnum library is required due to bugs like +/// https://github.com/dart-lang/sdk/issues/15361 +/// +/// Adapted from the Java version. +class TCompactProtocol extends TProtocol { + static const int PROTOCOL_ID = 0x82; + static const int VERSION = 1; + static const int VERSION_MASK = 0x1f; + static const int TYPE_MASK = 0xE0; + static const int TYPE_BITS = 0x07; + static const int TYPE_SHIFT_AMOUNT = 5; + static final TField TSTOP = new TField("", TType.STOP, 0); + + static const int TYPE_BOOLEAN_TRUE = 0x01; + static const int TYPE_BOOLEAN_FALSE = 0x02; + static const int TYPE_BYTE = 0x03; + static const int TYPE_I16 = 0x04; + static const int TYPE_I32 = 0x05; + static const int TYPE_I64 = 0x06; + static const int TYPE_DOUBLE = 0x07; + static const int TYPE_BINARY = 0x08; + static const int TYPE_LIST = 0x09; + static const int TYPE_SET = 0x0A; + static const int TYPE_MAP = 0x0B; + static const int TYPE_STRUCT = 0x0C; + + static final List<int> _typeMap = new List.unmodifiable(new List(16) + ..[TType.STOP] = TType.STOP + ..[TType.BOOL] = TYPE_BOOLEAN_TRUE + ..[TType.BYTE] = TYPE_BYTE + ..[TType.I16] = TYPE_I16 + ..[TType.I32] = TYPE_I32 + ..[TType.I64] = TYPE_I64 + ..[TType.DOUBLE] = TYPE_DOUBLE + ..[TType.STRING] = TYPE_BINARY + ..[TType.LIST] = TYPE_LIST + ..[TType.SET] = TYPE_SET + ..[TType.MAP] = TYPE_MAP + ..[TType.STRUCT] = TYPE_STRUCT); + + static const Utf8Codec _utf8Codec = const Utf8Codec(); + + // Pretend this is a stack + DoubleLinkedQueue<int> _lastField = new DoubleLinkedQueue<int>(); + int _lastFieldId = 0; + + TField _booleanField = null; + bool _boolValue = null; + + final Uint8List tempList = new Uint8List(10); + final ByteData tempBD = new ByteData(10); + + TCompactProtocol(TTransport transport) : super(transport); + + /// Write + void writeMessageBegin(TMessage message) { + writeByte(PROTOCOL_ID); + writeByte((VERSION & VERSION_MASK) | + ((message.type << TYPE_SHIFT_AMOUNT) & TYPE_MASK)); + _writeVarInt32(new Int32(message.seqid)); + writeString(message.name); + } + + void writeMessageEnd() {} + + void writeStructBegin(TStruct struct) { + _lastField.addLast(_lastFieldId); + _lastFieldId = 0; + } + + void writeStructEnd() { + _lastFieldId = _lastField.removeLast(); + } + + void writeFieldBegin(TField field) { + if (field.type == TType.BOOL) { + _booleanField = field; + } else { + _writeFieldBegin(field, -1); + } + } + + void _writeFieldBegin(TField field, int typeOverride) { + int typeToWrite = + typeOverride == -1 ? _getCompactType(field.type) : typeOverride; + + if (field.id > _lastFieldId && field.id - _lastFieldId <= 15) { + writeByte((field.id - _lastFieldId) << 4 | typeToWrite); + } else { + writeByte(typeToWrite); + writeI16(field.id); + } + + _lastFieldId = field.id; + } + + void writeFieldEnd() {} + + void writeFieldStop() { + writeByte(TType.STOP); + } + + void writeMapBegin(TMap map) { + if (map.length == 0) { + writeByte(0); + } else { + _writeVarInt32(new Int32(map.length)); + writeByte( + _getCompactType(map.keyType) << 4 | _getCompactType(map.valueType)); + } + } + + void writeMapEnd() {} + + void writeListBegin(TList list) { + _writeCollectionBegin(list.elementType, list.length); + } + + void writeListEnd() {} + + void writeSetBegin(TSet set) { + _writeCollectionBegin(set.elementType, set.length); + } + + void writeSetEnd() {} + + void writeBool(bool b) { + if (b == null) b = false; + if (_booleanField != null) { + _writeFieldBegin( + _booleanField, b ? TYPE_BOOLEAN_TRUE : TYPE_BOOLEAN_FALSE); + _booleanField = null; + } else { + writeByte(b ? TYPE_BOOLEAN_TRUE : TYPE_BOOLEAN_FALSE); + } + } + + void writeByte(int b) { + if (b == null) b = 0; + tempList[0] = b; + transport.write(tempList, 0, 1); + } + + void writeI16(int i16) { + if (i16 == null) i16 = 0; + _writeVarInt32(_int32ToZigZag(new Int32(i16))); + } + + void writeI32(int i32) { + if (i32 == null) i32 = 0; + _writeVarInt32(_int32ToZigZag(new Int32(i32))); + } + + void writeI64(int i64) { + if (i64 == null) i64 = 0; + _writeVarInt64(_int64ToZigZag(new Int64(i64))); + } + + void writeDouble(double d) { + if (d == null) d = 0.0; + tempBD.setFloat64(0, d); + transport.write(tempBD.buffer.asUint8List(), 0, 8); + } + + void writeString(String str) { + Uint8List bytes = + str != null ? _utf8Codec.encode(str) : new Uint8List.fromList([]); + writeBinary(bytes); + } + + void writeBinary(Uint8List bytes) { + _writeVarInt32(new Int32(bytes.length)); + transport.write(bytes, 0, bytes.length); + } + + void _writeVarInt32(Int32 n) { + int idx = 0; + while (true) { + if ((n & ~0x7F) == 0) { + tempList[idx++] = (n & 0xFF).toInt(); + break; + } else { + tempList[idx++] = (((n & 0x7F) | 0x80) & 0xFF).toInt(); + n = n.shiftRightUnsigned(7); + } + } + transport.write(tempList, 0, idx); + } + + void _writeVarInt64(Int64 n) { + int idx = 0; + while (true) { + if ((n & ~0x7F) == 0) { + tempList[idx++] = (n & 0xFF).toInt(); + break; + } else { + tempList[idx++] = (((n & 0x7F) | 0x80) & 0xFF).toInt(); + n = n.shiftRightUnsigned(7); + } + } + transport.write(tempList, 0, idx); + } + + void _writeCollectionBegin(int elemType, int length) { + if (length <= 14) { + writeByte(length << 4 | _getCompactType(elemType)); + } else { + writeByte(0xF0 | _getCompactType(elemType)); + _writeVarInt32(new Int32(length)); + } + } + + Int32 _int32ToZigZag(Int32 n) { + return (n << 1) ^ (n >> 31); + } + + Int64 _int64ToZigZag(Int64 n) { + return (n << 1) ^ (n >> 63); + } + + /// Read + TMessage readMessageBegin() { + int protocolId = readByte(); + if (protocolId != PROTOCOL_ID) { + throw new TProtocolError(TProtocolErrorType.BAD_VERSION, + 'Expected protocol id $PROTOCOL_ID but got $protocolId'); + } + int versionAndType = readByte(); + int version = versionAndType & VERSION_MASK; + if (version != VERSION) { + throw new TProtocolError(TProtocolErrorType.BAD_VERSION, + 'Expected version $VERSION but got $version'); + } + int type = (versionAndType >> TYPE_SHIFT_AMOUNT) & TYPE_BITS; + int seqId = _readVarInt32().toInt(); + String messageName = readString(); + return new TMessage(messageName, type, seqId); + } + + void readMessageEnd() {} + + TStruct readStructBegin() { + _lastField.addLast(_lastFieldId); + _lastFieldId = 0; + // TODO make this a constant? + return new TStruct(); + } + + void readStructEnd() { + _lastFieldId = _lastField.removeLast(); + } + + TField readFieldBegin() { + int type = readByte(); + if (type == TType.STOP) { + return TSTOP; + } + + int fieldId; + int modifier = (type & 0xF0) >> 4; + if (modifier == 0) { + fieldId = readI16(); + } else { + fieldId = _lastFieldId + modifier; + } + + TField field = new TField('', _getTType(type & 0x0F), fieldId); + if (_isBoolType(type)) { + _boolValue = (type & 0x0F) == TYPE_BOOLEAN_TRUE; + } + + _lastFieldId = field.id; + return field; + } + + void readFieldEnd() {} + + TMap readMapBegin() { + int length = _readVarInt32().toInt(); + _checkNegReadLength(length); + + int keyAndValueType = length == 0 ? 0 : readByte(); + int keyType = _getTType(keyAndValueType >> 4); + int valueType = _getTType(keyAndValueType & 0x0F); + return new TMap(keyType, valueType, length); + } + + void readMapEnd() {} + + TList readListBegin() { + int lengthAndType = readByte(); + int length = (lengthAndType >> 4) & 0x0F; + if (length == 15) { + length = _readVarInt32().toInt(); + } + _checkNegReadLength(length); + int type = _getTType(lengthAndType); + return new TList(type, length); + } + + void readListEnd() {} + + TSet readSetBegin() { + TList tlist = readListBegin(); + return new TSet(tlist.elementType, tlist.length); + } + + void readSetEnd() {} + + bool readBool() { + if (_boolValue != null) { + bool result = _boolValue; + _boolValue = null; + return result; + } + return readByte() == TYPE_BOOLEAN_TRUE; + } + + int readByte() { + transport.readAll(tempList, 0, 1); + return tempList.buffer.asByteData().getUint8(0); + } + + int readI16() { + return _zigzagToInt32(_readVarInt32()).toInt(); + } + + int readI32() { + return _zigzagToInt32(_readVarInt32()).toInt(); + } + + int readI64() { + return _zigzagToInt64(_readVarInt64()).toInt(); + } + + double readDouble() { + transport.readAll(tempList, 0, 8); + return tempList.buffer.asByteData().getFloat64(0); + } + + String readString() { + int length = _readVarInt32().toInt(); + _checkNegReadLength(length); + + // TODO look at using temp for small strings? + Uint8List buff = new Uint8List(length); + transport.readAll(buff, 0, length); + return _utf8Codec.decode(buff); + } + + Uint8List readBinary() { + int length = _readVarInt32().toInt(); + _checkNegReadLength(length); + + Uint8List buff = new Uint8List(length); + transport.readAll(buff, 0, length); + return buff; + } + + Int32 _readVarInt32() { + Int32 result = Int32.ZERO; + int shift = 0; + while (true) { + Int32 b = new Int32(readByte()); + result |= (b & 0x7f) << shift; + if ((b & 0x80) != 0x80) break; + shift += 7; + } + return result; + } + + Int64 _readVarInt64() { + Int64 result = Int64.ZERO; + int shift = 0; + while (true) { + Int64 b = new Int64(readByte()); + result |= (b & 0x7f) << shift; + if ((b & 0x80) != 0x80) break; + shift += 7; + } + return result; + } + + Int32 _zigzagToInt32(Int32 n) { + return (n.shiftRightUnsigned(1)) ^ -(n & 1); + } + + Int64 _zigzagToInt64(Int64 n) { + return (n.shiftRightUnsigned(1)) ^ -(n & 1); + } + + void _checkNegReadLength(int length) { + if (length < 0) { + throw new TProtocolError( + TProtocolErrorType.NEGATIVE_SIZE, 'Negative length: $length'); + } + } + + int _getCompactType(int ttype) { + return _typeMap[ttype]; + } + + int _getTType(int type) { + switch (type & 0x0F) { + case TType.STOP: + return TType.STOP; + case TYPE_BOOLEAN_FALSE: + case TYPE_BOOLEAN_TRUE: + return TType.BOOL; + case TYPE_BYTE: + return TType.BYTE; + case TYPE_I16: + return TType.I16; + case TYPE_I32: + return TType.I32; + case TYPE_I64: + return TType.I64; + case TYPE_DOUBLE: + return TType.DOUBLE; + case TYPE_BINARY: + return TType.STRING; + case TYPE_LIST: + return TType.LIST; + case TYPE_SET: + return TType.SET; + case TYPE_MAP: + return TType.MAP; + case TYPE_STRUCT: + return TType.STRUCT; + default: + throw new TProtocolError( + TProtocolErrorType.INVALID_DATA, "Unknown type: ${type & 0x0F}"); + } + } + + bool _isBoolType(int b) { + int lowerNibble = b & 0x0F; + return lowerNibble == TYPE_BOOLEAN_TRUE || + lowerNibble == TYPE_BOOLEAN_FALSE; + } +} diff --git a/lib/dart/lib/thrift.dart b/lib/dart/lib/thrift.dart index 2483726ee..27eb54646 100644 --- a/lib/dart/lib/thrift.dart +++ b/lib/dart/lib/thrift.dart @@ -18,11 +18,13 @@ library thrift; import 'dart:async'; +import 'dart:collection'; import 'dart:convert' show Utf8Codec; import 'dart:typed_data' show ByteData; import 'dart:typed_data' show Uint8List; import 'package:crypto/crypto.dart' show CryptoUtils; +import 'package:fixnum/fixnum.dart'; import 'package:http/http.dart' show Client; import 'package:logging/logging.dart'; @@ -32,6 +34,7 @@ part 'src/t_error.dart'; part 'src/t_processor.dart'; part 'src/protocol/t_binary_protocol.dart'; +part 'src/protocol/t_compact_protocol.dart'; part 'src/protocol/t_field.dart'; part 'src/protocol/t_json_protocol.dart'; part 'src/protocol/t_list.dart'; diff --git a/lib/dart/pubspec.yaml b/lib/dart/pubspec.yaml index 99d490cea..7012f5d55 100644 --- a/lib/dart/pubspec.yaml +++ b/lib/dart/pubspec.yaml @@ -26,6 +26,7 @@ environment: sdk: ">=1.12.0 <2.0.0" dependencies: crypto: "^0.9.0" + fixnum: "^0.10.2" http: "^0.11.3" logging: "^0.11.0" dev_dependencies: diff --git a/lib/dart/test/protocol/t_protocol_test.dart b/lib/dart/test/protocol/t_protocol_test.dart index 736288403..0e6cde50d 100644 --- a/lib/dart/test/protocol/t_protocol_test.dart +++ b/lib/dart/test/protocol/t_protocol_test.dart @@ -353,7 +353,6 @@ void main() { protocol.writeMessageBegin(message); }); - test('Test escaped unicode', () async { /* KOR_KAI @@ -372,7 +371,8 @@ void main() { await protocol.transport.flush(); var subject = protocol.readString(); - expect(subject, UTF8.decode([0x01, 0xE0, 0xB8, 0x81, 0x20, 0xF0, 0x9D, 0x84, 0x9E])); + expect(subject, + UTF8.decode([0x01, 0xE0, 0xB8, 0x81, 0x20, 0xF0, 0x9D, 0x84, 0x9E])); }); group('shared tests', sharedTests); @@ -386,6 +386,15 @@ void main() { group('shared tests', sharedTests); }); + + group('compact', () { + setUp(() { + protocol = new TCompactProtocol(new TBufferedTransport()); + protocol.writeMessageBegin(message); + }); + + group('shared tests', sharedTests); + }); } class Primitive { |