THRIFT-3241 fatal error: runtime: cannot map pages in arena address space
Client: Go
Patch: Jens Geyer
This closes #564
diff --git a/lib/go/thrift/binary_protocol.go b/lib/go/thrift/binary_protocol.go
index 09f94d4..d1059f2 100644
--- a/lib/go/thrift/binary_protocol.go
+++ b/lib/go/thrift/binary_protocol.go
@@ -258,7 +258,7 @@
if p.strictRead {
return name, typeId, seqId, NewTProtocolExceptionWithType(BAD_VERSION, fmt.Errorf("Missing version in ReadMessageBegin"))
}
- name, e2 := p.readStringBody(int(size))
+ name, e2 := p.readStringBody(size)
if e2 != nil {
return name, typeId, seqId, e2
}
@@ -434,7 +434,7 @@
return
}
- return p.readStringBody(int(size))
+ return p.readStringBody(size)
}
func (p *TBinaryProtocol) ReadBinary() ([]byte, error) {
@@ -445,6 +445,9 @@
if size < 0 {
return nil, invalidDataLength
}
+ if uint64(size) > p.trans.RemainingBytes() {
+ return nil, invalidDataLength
+ }
isize := int(size)
buf := make([]byte, isize)
@@ -469,12 +472,15 @@
return NewTProtocolException(err)
}
-func (p *TBinaryProtocol) readStringBody(size int) (value string, err error) {
+func (p *TBinaryProtocol) readStringBody(size int32) (value string, err error) {
if size < 0 {
return "", nil
}
+ if uint64(size) > p.trans.RemainingBytes() {
+ return "", invalidDataLength
+ }
var buf []byte
- if size <= len(p.buffer) {
+ if int(size) <= len(p.buffer) {
buf = p.buffer[0:size]
} else {
buf = make([]byte, size)
diff --git a/lib/go/thrift/buffered_transport.go b/lib/go/thrift/buffered_transport.go
index d258b70..8bcad1e 100644
--- a/lib/go/thrift/buffered_transport.go
+++ b/lib/go/thrift/buffered_transport.go
@@ -68,3 +68,8 @@
}
return p.tp.Flush()
}
+
+func (p *TBufferedTransport) RemainingBytes() (num_bytes uint64) {
+ return p.tp.RemainingBytes()
+}
+
diff --git a/lib/go/thrift/compact_protocol.go b/lib/go/thrift/compact_protocol.go
index 1a7a8da..731bd16 100644
--- a/lib/go/thrift/compact_protocol.go
+++ b/lib/go/thrift/compact_protocol.go
@@ -561,6 +561,9 @@
if length < 0 {
return "", invalidDataLength
}
+ if uint64(length) > p.trans.RemainingBytes() {
+ return "", invalidDataLength
+ }
if length == 0 {
return "", nil
@@ -587,6 +590,9 @@
if length < 0 {
return nil, invalidDataLength
}
+ if uint64(length) > p.trans.RemainingBytes() {
+ return nil, invalidDataLength
+ }
buf := make([]byte, length)
_, e = io.ReadFull(p.trans, buf)
diff --git a/lib/go/thrift/framed_transport.go b/lib/go/thrift/framed_transport.go
index 19fb85f..d0bae21 100644
--- a/lib/go/thrift/framed_transport.go
+++ b/lib/go/thrift/framed_transport.go
@@ -33,21 +33,21 @@
transport TTransport
buf bytes.Buffer
reader *bufio.Reader
- frameSize int //Current remaining size of the frame. if ==0 read next frame header
+ frameSize uint32 //Current remaining size of the frame. if ==0 read next frame header
buffer [4]byte
- maxLength int
+ maxLength uint32
}
type tFramedTransportFactory struct {
factory TTransportFactory
- maxLength int
+ maxLength uint32
}
func NewTFramedTransportFactory(factory TTransportFactory) TTransportFactory {
return &tFramedTransportFactory{factory: factory, maxLength: DEFAULT_MAX_LENGTH}
}
-func NewTFramedTransportFactoryMaxLength(factory TTransportFactory, maxLength int) TTransportFactory {
+func NewTFramedTransportFactoryMaxLength(factory TTransportFactory, maxLength uint32) TTransportFactory {
return &tFramedTransportFactory{factory: factory, maxLength: maxLength}
}
@@ -59,7 +59,7 @@
return &TFramedTransport{transport: transport, reader: bufio.NewReader(transport), maxLength: DEFAULT_MAX_LENGTH}
}
-func NewTFramedTransportMaxLength(transport TTransport, maxLength int) *TFramedTransport {
+func NewTFramedTransportMaxLength(transport TTransport, maxLength uint32) *TFramedTransport {
return &TFramedTransport{transport: transport, reader: bufio.NewReader(transport), maxLength: maxLength}
}
@@ -82,7 +82,7 @@
return
}
}
- if p.frameSize < len(buf) {
+ if p.frameSize < uint32(len(buf)) {
frameSize := p.frameSize
tmp := make([]byte, p.frameSize)
l, err = p.Read(tmp)
@@ -93,7 +93,7 @@
}
}
got, err := p.reader.Read(buf)
- p.frameSize = p.frameSize - got
+ p.frameSize = p.frameSize - uint32(got)
//sanity check
if p.frameSize < 0 {
return 0, NewTTransportException(UNKNOWN_TRANSPORT_EXCEPTION, "Negative frame size")
@@ -149,14 +149,19 @@
return NewTTransportExceptionFromError(err)
}
-func (p *TFramedTransport) readFrameHeader() (int, error) {
+func (p *TFramedTransport) readFrameHeader() (uint32, error) {
buf := p.buffer[:4]
if _, err := io.ReadFull(p.reader, buf); err != nil {
return 0, err
}
- size := int(binary.BigEndian.Uint32(buf))
+ size := binary.BigEndian.Uint32(buf)
if size < 0 || size > p.maxLength {
return 0, NewTTransportException(UNKNOWN_TRANSPORT_EXCEPTION, fmt.Sprintf("Incorrect frame size (%d)", size))
}
return size, nil
}
+
+func (p *TFramedTransport) RemainingBytes() (num_bytes uint64) {
+ return uint64(p.frameSize)
+}
+
diff --git a/lib/go/thrift/http_client.go b/lib/go/thrift/http_client.go
index b7cb101..0f6e704 100644
--- a/lib/go/thrift/http_client.go
+++ b/lib/go/thrift/http_client.go
@@ -191,3 +191,14 @@
p.response = response
return nil
}
+
+func (p *THttpClient) RemainingBytes() (num_bytes uint64) {
+ len := p.response.ContentLength
+ if len >= 0 {
+ return uint64(len)
+ }
+
+ const maxSize = ^uint64(0)
+ return maxSize // the thruth is, we just don't know unless framed is used
+}
+
diff --git a/lib/go/thrift/iostream_transport.go b/lib/go/thrift/iostream_transport.go
index 8fc0214..794872f 100644
--- a/lib/go/thrift/iostream_transport.go
+++ b/lib/go/thrift/iostream_transport.go
@@ -206,3 +206,9 @@
}
return
}
+
+func (p *StreamTransport) RemainingBytes() (num_bytes uint64) {
+ const maxSize = ^uint64(0)
+ return maxSize // the thruth is, we just don't know unless framed is used
+}
+
diff --git a/lib/go/thrift/memory_buffer.go b/lib/go/thrift/memory_buffer.go
index 59e4542..b62fd56 100644
--- a/lib/go/thrift/memory_buffer.go
+++ b/lib/go/thrift/memory_buffer.go
@@ -73,3 +73,7 @@
func (p *TMemoryBuffer) Flush() error {
return nil
}
+
+func (p *TMemoryBuffer) RemainingBytes() (num_bytes uint64) {
+ return uint64(p.Buffer.Len())
+}
diff --git a/lib/go/thrift/rich_transport.go b/lib/go/thrift/rich_transport.go
index d7268d9..8e296a9 100644
--- a/lib/go/thrift/rich_transport.go
+++ b/lib/go/thrift/rich_transport.go
@@ -42,6 +42,10 @@
return r.Write([]byte(s))
}
+func (r *RichTransport) RemainingBytes() (num_bytes uint64) {
+ return r.TTransport.RemainingBytes()
+}
+
func readByte(r io.Reader) (c byte, err error) {
v := [1]byte{0}
n, err := r.Read(v[0:1])
@@ -62,3 +66,4 @@
_, err := w.Write(v[0:1])
return err
}
+
diff --git a/lib/go/thrift/socket.go b/lib/go/thrift/socket.go
index 6c24748..82e28b4 100644
--- a/lib/go/thrift/socket.go
+++ b/lib/go/thrift/socket.go
@@ -158,3 +158,9 @@
}
return p.conn.Close()
}
+
+func (p *TSocket) RemainingBytes() (num_bytes uint64) {
+ const maxSize = ^uint64(0)
+ return maxSize // the thruth is, we just don't know unless framed is used
+}
+
diff --git a/lib/go/thrift/ssl_socket.go b/lib/go/thrift/ssl_socket.go
index 4faaf8c..04d3850 100644
--- a/lib/go/thrift/ssl_socket.go
+++ b/lib/go/thrift/ssl_socket.go
@@ -163,3 +163,9 @@
}
return p.conn.Close()
}
+
+func (p *TSSLSocket) RemainingBytes() (num_bytes uint64) {
+ const maxSize = ^uint64(0)
+ return maxSize // the thruth is, we just don't know unless framed is used
+}
+
diff --git a/lib/go/thrift/transport.go b/lib/go/thrift/transport.go
index 8c0622d..4538996 100644
--- a/lib/go/thrift/transport.go
+++ b/lib/go/thrift/transport.go
@@ -30,10 +30,16 @@
Flush() (err error)
}
+type ReadSizeProvider interface {
+ RemainingBytes() (num_bytes uint64)
+}
+
+
// Encapsulates the I/O layer
type TTransport interface {
io.ReadWriteCloser
Flusher
+ ReadSizeProvider
// Opens the transport for communication
Open() error
@@ -46,6 +52,7 @@
WriteString(s string) (n int, err error)
}
+
// This is "enchanced" transport with extra capabilities. You need to use one of these
// to construct protocol.
// Notably, TSocket does not implement this interface, and it is always a mistake to use
@@ -56,4 +63,6 @@
io.ByteWriter
stringWriter
Flusher
+ ReadSizeProvider
}
+