blob: 93c49115422da7ae8a2af295411772c91f9d6f20 [file] [log] [blame]
David Reissea2cba82009-03-30 21:35:00 +00001/*
2 * Licensed to the Apache Software Foundation (ASF) under one
3 * or more contributor license agreements. See the NOTICE file
4 * distributed with this work for additional information
5 * regarding copyright ownership. The ASF licenses this file
6 * to you under the Apache License, Version 2.0 (the
7 * "License"); you may not use this file except in compliance
8 * with the License. You may obtain a copy of the License at
9 *
10 * http://www.apache.org/licenses/LICENSE-2.0
11 *
12 * Unless required by applicable law or agreed to in writing,
13 * software distributed under the License is distributed on an
14 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15 * KIND, either express or implied. See the License for the
16 * specific language governing permissions and limitations
17 * under the License.
18 */
David Reiss382fc302007-08-25 18:01:30 +000019
20#include <Python.h>
21#include "cStringIO.h"
David Reiss382fc302007-08-25 18:01:30 +000022#include <stdint.h>
Roger Meierc3f033f2011-09-13 13:54:05 +000023#ifndef _WIN32
24# include <stdbool.h>
25# include <netinet/in.h>
26#else
27# include <WinSock2.h>
28# pragma comment (lib, "ws2_32.lib")
29# define BIG_ENDIAN (4321)
30# define LITTLE_ENDIAN (1234)
31# define BYTE_ORDER LITTLE_ENDIAN
32# if defined(_MSC_VER) && _MSC_VER < 1600
33 typedef int _Bool;
34# define bool _Bool
Roger Meier7daf00c2015-06-03 11:45:35 +020035# define false 0
Roger Meierc3f033f2011-09-13 13:54:05 +000036# define true 1
37# endif
38# define inline __inline
39#endif
David Reiss382fc302007-08-25 18:01:30 +000040
David Reiss49809102009-03-18 23:14:13 +000041/* Fix endianness issues on Solaris */
42#if defined (__SVR4) && defined (__sun)
43 #if defined(__i386) && !defined(__i386__)
44 #define __i386__
45 #endif
46
47 #ifndef BIG_ENDIAN
48 #define BIG_ENDIAN (4321)
49 #endif
50 #ifndef LITTLE_ENDIAN
51 #define LITTLE_ENDIAN (1234)
52 #endif
53
54 /* I386 is LE, even on Solaris */
55 #if !defined(BYTE_ORDER) && defined(__i386__)
56 #define BYTE_ORDER LITTLE_ENDIAN
57 #endif
58#endif
59
David Reiss382fc302007-08-25 18:01:30 +000060// TODO(dreiss): defval appears to be unused. Look into removing it.
61// TODO(dreiss): Make parse_spec_args recursive, and cache the output
62// permanently in the object. (Malloc and orphan.)
63// TODO(dreiss): Why do we need cStringIO for reading, why not just char*?
64// Can cStringIO let us work with a BufferedTransport?
65// TODO(dreiss): Don't ignore the rv from cwrite (maybe).
66
67/* ====== BEGIN UTILITIES ====== */
68
69#define INIT_OUTBUF_SIZE 128
70
71// Stolen out of TProtocol.h.
72// It would be a huge pain to have both get this from one place.
73typedef enum TType {
74 T_STOP = 0,
75 T_VOID = 1,
76 T_BOOL = 2,
77 T_BYTE = 3,
78 T_I08 = 3,
79 T_I16 = 6,
80 T_I32 = 8,
81 T_U64 = 9,
82 T_I64 = 10,
83 T_DOUBLE = 4,
84 T_STRING = 11,
85 T_UTF7 = 11,
86 T_STRUCT = 12,
87 T_MAP = 13,
88 T_SET = 14,
89 T_LIST = 15,
90 T_UTF8 = 16,
91 T_UTF16 = 17
92} TType;
93
David Reissfdd8b5a2009-02-17 20:06:08 +000094#ifndef __BYTE_ORDER
95# if defined(BYTE_ORDER) && defined(LITTLE_ENDIAN) && defined(BIG_ENDIAN)
96# define __BYTE_ORDER BYTE_ORDER
97# define __LITTLE_ENDIAN LITTLE_ENDIAN
98# define __BIG_ENDIAN BIG_ENDIAN
99# else
100# error "Cannot determine endianness"
101# endif
102#endif
103
David Reiss382fc302007-08-25 18:01:30 +0000104// Same comment as the enum. Sorry.
105#if __BYTE_ORDER == __BIG_ENDIAN
106# define ntohll(n) (n)
107# define htonll(n) (n)
108#elif __BYTE_ORDER == __LITTLE_ENDIAN
109# if defined(__GNUC__) && defined(__GLIBC__)
110# include <byteswap.h>
111# define ntohll(n) bswap_64(n)
112# define htonll(n) bswap_64(n)
113# else /* GNUC & GLIBC */
114# define ntohll(n) ( (((unsigned long long)ntohl(n)) << 32) + ntohl(n >> 32) )
115# define htonll(n) ( (((unsigned long long)htonl(n)) << 32) + htonl(n >> 32) )
116# endif /* GNUC & GLIBC */
117#else /* __BYTE_ORDER */
118# error "Can't define htonll or ntohll!"
119#endif
120
121// Doing a benchmark shows that interning actually makes a difference, amazingly.
122#define INTERN_STRING(value) _intern_ ## value
123
124#define INT_CONV_ERROR_OCCURRED(v) ( ((v) == -1) && PyErr_Occurred() )
125#define CHECK_RANGE(v, min, max) ( ((v) <= (max)) && ((v) >= (min)) )
126
David Reiss5ec8e262007-08-26 02:21:24 +0000127// Py_ssize_t was not defined before Python 2.5
128#if (PY_VERSION_HEX < 0x02050000)
129typedef int Py_ssize_t;
130#endif
131
David Reiss382fc302007-08-25 18:01:30 +0000132/**
133 * A cache of the spec_args for a set or list,
134 * so we don't have to keep calling PyTuple_GET_ITEM.
135 */
136typedef struct {
137 TType element_type;
138 PyObject* typeargs;
139} SetListTypeArgs;
140
141/**
142 * A cache of the spec_args for a map,
143 * so we don't have to keep calling PyTuple_GET_ITEM.
144 */
145typedef struct {
146 TType ktag;
147 TType vtag;
148 PyObject* ktypeargs;
149 PyObject* vtypeargs;
150} MapTypeArgs;
151
152/**
153 * A cache of the spec_args for a struct,
154 * so we don't have to keep calling PyTuple_GET_ITEM.
155 */
156typedef struct {
157 PyObject* klass;
158 PyObject* spec;
159} StructTypeArgs;
160
161/**
162 * A cache of the item spec from a struct specification,
163 * so we don't have to keep calling PyTuple_GET_ITEM.
164 */
165typedef struct {
166 int tag;
167 TType type;
168 PyObject* attrname;
169 PyObject* typeargs;
170 PyObject* defval;
171} StructItemSpec;
172
173/**
174 * A cache of the two key attributes of a CReadableTransport,
175 * so we don't have to keep calling PyObject_GetAttr.
176 */
177typedef struct {
178 PyObject* stringiobuf;
179 PyObject* refill_callable;
180} DecodeBuffer;
181
182/** Pointer to interned string to speed up attribute lookup. */
183static PyObject* INTERN_STRING(cstringio_buf);
184/** Pointer to interned string to speed up attribute lookup. */
185static PyObject* INTERN_STRING(cstringio_refill);
186
187static inline bool
188check_ssize_t_32(Py_ssize_t len) {
189 // error from getting the int
190 if (INT_CONV_ERROR_OCCURRED(len)) {
191 return false;
192 }
193 if (!CHECK_RANGE(len, 0, INT32_MAX)) {
194 PyErr_SetString(PyExc_OverflowError, "string size out of range");
195 return false;
196 }
197 return true;
198}
199
Roger Meier7daf00c2015-06-03 11:45:35 +0200200#define MAX_LIST_SIZE (10000)
201
202static inline bool
203check_list_length(Py_ssize_t len) {
204 // error from getting the int
205 if (INT_CONV_ERROR_OCCURRED(len)) {
206 return false;
207 }
208 if (!CHECK_RANGE(len, 0, MAX_LIST_SIZE)) {
209 PyErr_SetString(PyExc_OverflowError, "list size out of the sanity limit (10000 items max)");
210 return false;
211 }
212 return true;
213}
214
David Reiss382fc302007-08-25 18:01:30 +0000215static inline bool
216parse_pyint(PyObject* o, int32_t* ret, int32_t min, int32_t max) {
217 long val = PyInt_AsLong(o);
218
219 if (INT_CONV_ERROR_OCCURRED(val)) {
220 return false;
221 }
222 if (!CHECK_RANGE(val, min, max)) {
223 PyErr_SetString(PyExc_OverflowError, "int out of range");
224 return false;
225 }
226
227 *ret = (int32_t) val;
228 return true;
229}
230
231
232/* --- FUNCTIONS TO PARSE STRUCT SPECIFICATOINS --- */
233
234static bool
235parse_set_list_args(SetListTypeArgs* dest, PyObject* typeargs) {
236 if (PyTuple_Size(typeargs) != 2) {
237 PyErr_SetString(PyExc_TypeError, "expecting tuple of size 2 for list/set type args");
238 return false;
239 }
240
241 dest->element_type = PyInt_AsLong(PyTuple_GET_ITEM(typeargs, 0));
242 if (INT_CONV_ERROR_OCCURRED(dest->element_type)) {
243 return false;
244 }
245
246 dest->typeargs = PyTuple_GET_ITEM(typeargs, 1);
247
248 return true;
249}
250
251static bool
252parse_map_args(MapTypeArgs* dest, PyObject* typeargs) {
253 if (PyTuple_Size(typeargs) != 4) {
254 PyErr_SetString(PyExc_TypeError, "expecting 4 arguments for typeargs to map");
255 return false;
256 }
257
258 dest->ktag = PyInt_AsLong(PyTuple_GET_ITEM(typeargs, 0));
259 if (INT_CONV_ERROR_OCCURRED(dest->ktag)) {
260 return false;
261 }
262
263 dest->vtag = PyInt_AsLong(PyTuple_GET_ITEM(typeargs, 2));
264 if (INT_CONV_ERROR_OCCURRED(dest->vtag)) {
265 return false;
266 }
267
268 dest->ktypeargs = PyTuple_GET_ITEM(typeargs, 1);
269 dest->vtypeargs = PyTuple_GET_ITEM(typeargs, 3);
270
271 return true;
272}
273
274static bool
275parse_struct_args(StructTypeArgs* dest, PyObject* typeargs) {
276 if (PyTuple_Size(typeargs) != 2) {
277 PyErr_SetString(PyExc_TypeError, "expecting tuple of size 2 for struct args");
278 return false;
279 }
280
281 dest->klass = PyTuple_GET_ITEM(typeargs, 0);
282 dest->spec = PyTuple_GET_ITEM(typeargs, 1);
283
284 return true;
285}
286
287static int
288parse_struct_item_spec(StructItemSpec* dest, PyObject* spec_tuple) {
289
290 // i'd like to use ParseArgs here, but it seems to be a bottleneck.
291 if (PyTuple_Size(spec_tuple) != 5) {
292 PyErr_SetString(PyExc_TypeError, "expecting 5 arguments for spec tuple");
293 return false;
294 }
295
296 dest->tag = PyInt_AsLong(PyTuple_GET_ITEM(spec_tuple, 0));
297 if (INT_CONV_ERROR_OCCURRED(dest->tag)) {
298 return false;
299 }
300
301 dest->type = PyInt_AsLong(PyTuple_GET_ITEM(spec_tuple, 1));
302 if (INT_CONV_ERROR_OCCURRED(dest->type)) {
303 return false;
304 }
305
306 dest->attrname = PyTuple_GET_ITEM(spec_tuple, 2);
307 dest->typeargs = PyTuple_GET_ITEM(spec_tuple, 3);
308 dest->defval = PyTuple_GET_ITEM(spec_tuple, 4);
309 return true;
310}
311
312/* ====== END UTILITIES ====== */
313
314
315/* ====== BEGIN WRITING FUNCTIONS ====== */
316
317/* --- LOW-LEVEL WRITING FUNCTIONS --- */
318
319static void writeByte(PyObject* outbuf, int8_t val) {
320 int8_t net = val;
321 PycStringIO->cwrite(outbuf, (char*)&net, sizeof(int8_t));
322}
323
324static void writeI16(PyObject* outbuf, int16_t val) {
325 int16_t net = (int16_t)htons(val);
326 PycStringIO->cwrite(outbuf, (char*)&net, sizeof(int16_t));
327}
328
329static void writeI32(PyObject* outbuf, int32_t val) {
330 int32_t net = (int32_t)htonl(val);
331 PycStringIO->cwrite(outbuf, (char*)&net, sizeof(int32_t));
332}
333
334static void writeI64(PyObject* outbuf, int64_t val) {
335 int64_t net = (int64_t)htonll(val);
336 PycStringIO->cwrite(outbuf, (char*)&net, sizeof(int64_t));
337}
338
339static void writeDouble(PyObject* outbuf, double dub) {
340 // Unfortunately, bitwise_cast doesn't work in C. Bad C!
341 union {
342 double f;
343 int64_t t;
344 } transfer;
345 transfer.f = dub;
346 writeI64(outbuf, transfer.t);
347}
348
349
Konrad Grochowski3b5dacb2014-11-24 10:55:31 +0100350/* --- MAIN RECURSIVE OUTPUT FUNCTION -- */
David Reiss382fc302007-08-25 18:01:30 +0000351
352static int
353output_val(PyObject* output, PyObject* value, TType type, PyObject* typeargs) {
354 /*
355 * Refcounting Strategy:
356 *
357 * We assume that elements of the thrift_spec tuple are not going to be
358 * mutated, so we don't ref count those at all. Other than that, we try to
359 * keep a reference to all the user-created objects while we work with them.
360 * output_val assumes that a reference is already held. The *caller* is
361 * responsible for handling references
362 */
363
364 switch (type) {
365
366 case T_BOOL: {
367 int v = PyObject_IsTrue(value);
368 if (v == -1) {
369 return false;
370 }
371
372 writeByte(output, (int8_t) v);
373 break;
374 }
375 case T_I08: {
376 int32_t val;
377
378 if (!parse_pyint(value, &val, INT8_MIN, INT8_MAX)) {
379 return false;
380 }
381
382 writeByte(output, (int8_t) val);
383 break;
384 }
385 case T_I16: {
386 int32_t val;
387
388 if (!parse_pyint(value, &val, INT16_MIN, INT16_MAX)) {
389 return false;
390 }
391
392 writeI16(output, (int16_t) val);
393 break;
394 }
395 case T_I32: {
396 int32_t val;
397
398 if (!parse_pyint(value, &val, INT32_MIN, INT32_MAX)) {
399 return false;
400 }
401
402 writeI32(output, val);
403 break;
404 }
405 case T_I64: {
406 int64_t nval = PyLong_AsLongLong(value);
407
408 if (INT_CONV_ERROR_OCCURRED(nval)) {
409 return false;
410 }
411
412 if (!CHECK_RANGE(nval, INT64_MIN, INT64_MAX)) {
413 PyErr_SetString(PyExc_OverflowError, "int out of range");
414 return false;
415 }
416
417 writeI64(output, nval);
418 break;
419 }
420
421 case T_DOUBLE: {
422 double nval = PyFloat_AsDouble(value);
423 if (nval == -1.0 && PyErr_Occurred()) {
424 return false;
425 }
426
427 writeDouble(output, nval);
428 break;
429 }
430
431 case T_STRING: {
432 Py_ssize_t len = PyString_Size(value);
433
434 if (!check_ssize_t_32(len)) {
435 return false;
436 }
437
438 writeI32(output, (int32_t) len);
439 PycStringIO->cwrite(output, PyString_AsString(value), (int32_t) len);
440 break;
441 }
442
443 case T_LIST:
444 case T_SET: {
445 Py_ssize_t len;
446 SetListTypeArgs parsedargs;
447 PyObject *item;
448 PyObject *iterator;
449
450 if (!parse_set_list_args(&parsedargs, typeargs)) {
451 return false;
452 }
453
454 len = PyObject_Length(value);
455
456 if (!check_ssize_t_32(len)) {
457 return false;
458 }
459
460 writeByte(output, parsedargs.element_type);
461 writeI32(output, (int32_t) len);
462
463 iterator = PyObject_GetIter(value);
464 if (iterator == NULL) {
465 return false;
466 }
467
468 while ((item = PyIter_Next(iterator))) {
469 if (!output_val(output, item, parsedargs.element_type, parsedargs.typeargs)) {
470 Py_DECREF(item);
471 Py_DECREF(iterator);
472 return false;
473 }
474 Py_DECREF(item);
475 }
476
477 Py_DECREF(iterator);
478
479 if (PyErr_Occurred()) {
480 return false;
481 }
482
483 break;
484 }
485
486 case T_MAP: {
487 PyObject *k, *v;
David Reiss58434e62008-10-07 21:08:10 +0000488 Py_ssize_t pos = 0;
David Reiss382fc302007-08-25 18:01:30 +0000489 Py_ssize_t len;
490
491 MapTypeArgs parsedargs;
492
493 len = PyDict_Size(value);
494 if (!check_ssize_t_32(len)) {
495 return false;
496 }
497
498 if (!parse_map_args(&parsedargs, typeargs)) {
499 return false;
500 }
501
502 writeByte(output, parsedargs.ktag);
503 writeByte(output, parsedargs.vtag);
504 writeI32(output, len);
505
506 // TODO(bmaurer): should support any mapping, not just dicts
507 while (PyDict_Next(value, &pos, &k, &v)) {
508 // TODO(dreiss): Think hard about whether these INCREFs actually
509 // turn any unsafe scenarios into safe scenarios.
510 Py_INCREF(k);
511 Py_INCREF(v);
512
513 if (!output_val(output, k, parsedargs.ktag, parsedargs.ktypeargs)
514 || !output_val(output, v, parsedargs.vtag, parsedargs.vtypeargs)) {
515 Py_DECREF(k);
516 Py_DECREF(v);
517 return false;
518 }
Kevin Clark127d01c2009-03-24 01:40:56 +0000519 Py_DECREF(k);
520 Py_DECREF(v);
David Reiss382fc302007-08-25 18:01:30 +0000521 }
522 break;
523 }
524
525 // TODO(dreiss): Consider breaking this out as a function
526 // the way we did for decode_struct.
527 case T_STRUCT: {
528 StructTypeArgs parsedargs;
529 Py_ssize_t nspec;
530 Py_ssize_t i;
531
532 if (!parse_struct_args(&parsedargs, typeargs)) {
533 return false;
534 }
535
536 nspec = PyTuple_Size(parsedargs.spec);
537
538 if (nspec == -1) {
539 return false;
540 }
541
542 for (i = 0; i < nspec; i++) {
543 StructItemSpec parsedspec;
544 PyObject* spec_tuple;
545 PyObject* instval = NULL;
546
547 spec_tuple = PyTuple_GET_ITEM(parsedargs.spec, i);
548 if (spec_tuple == Py_None) {
549 continue;
550 }
551
552 if (!parse_struct_item_spec (&parsedspec, spec_tuple)) {
553 return false;
554 }
555
556 instval = PyObject_GetAttr(value, parsedspec.attrname);
557
558 if (!instval) {
559 return false;
560 }
561
562 if (instval == Py_None) {
563 Py_DECREF(instval);
564 continue;
565 }
566
567 writeByte(output, (int8_t) parsedspec.type);
568 writeI16(output, parsedspec.tag);
569
570 if (!output_val(output, instval, parsedspec.type, parsedspec.typeargs)) {
571 Py_DECREF(instval);
572 return false;
573 }
574
575 Py_DECREF(instval);
576 }
577
578 writeByte(output, (int8_t)T_STOP);
579 break;
580 }
581
582 case T_STOP:
583 case T_VOID:
584 case T_UTF16:
585 case T_UTF8:
586 case T_U64:
587 default:
588 PyErr_SetString(PyExc_TypeError, "Unexpected TType");
589 return false;
590
591 }
592
593 return true;
594}
595
596
597/* --- TOP-LEVEL WRAPPER FOR OUTPUT -- */
598
599static PyObject *
600encode_binary(PyObject *self, PyObject *args) {
601 PyObject* enc_obj;
602 PyObject* type_args;
603 PyObject* buf;
604 PyObject* ret = NULL;
605
606 if (!PyArg_ParseTuple(args, "OO", &enc_obj, &type_args)) {
607 return NULL;
608 }
609
610 buf = PycStringIO->NewOutput(INIT_OUTBUF_SIZE);
611 if (output_val(buf, enc_obj, T_STRUCT, type_args)) {
612 ret = PycStringIO->cgetvalue(buf);
613 }
614
615 Py_DECREF(buf);
616 return ret;
617}
618
619/* ====== END WRITING FUNCTIONS ====== */
620
621
622/* ====== BEGIN READING FUNCTIONS ====== */
623
624/* --- LOW-LEVEL READING FUNCTIONS --- */
625
626static void
627free_decodebuf(DecodeBuffer* d) {
628 Py_XDECREF(d->stringiobuf);
629 Py_XDECREF(d->refill_callable);
630}
631
632static bool
633decode_buffer_from_obj(DecodeBuffer* dest, PyObject* obj) {
634 dest->stringiobuf = PyObject_GetAttr(obj, INTERN_STRING(cstringio_buf));
635 if (!dest->stringiobuf) {
636 return false;
637 }
638
639 if (!PycStringIO_InputCheck(dest->stringiobuf)) {
640 free_decodebuf(dest);
641 PyErr_SetString(PyExc_TypeError, "expecting stringio input");
642 return false;
643 }
644
645 dest->refill_callable = PyObject_GetAttr(obj, INTERN_STRING(cstringio_refill));
646
647 if(!dest->refill_callable) {
648 free_decodebuf(dest);
649 return false;
650 }
651
652 if (!PyCallable_Check(dest->refill_callable)) {
653 free_decodebuf(dest);
654 PyErr_SetString(PyExc_TypeError, "expecting callable");
655 return false;
656 }
657
658 return true;
659}
660
661static bool readBytes(DecodeBuffer* input, char** output, int len) {
662 int read;
663
664 // TODO(dreiss): Don't fear the malloc. Think about taking a copy of
665 // the partial read instead of forcing the transport
666 // to prepend it to its buffer.
667
668 read = PycStringIO->cread(input->stringiobuf, output, len);
669
670 if (read == len) {
671 return true;
672 } else if (read == -1) {
673 return false;
674 } else {
675 PyObject* newiobuf;
676
677 // using building functions as this is a rare codepath
678 newiobuf = PyObject_CallFunction(
David Reiss2c2e6d22007-09-05 01:14:09 +0000679 input->refill_callable, "s#i", *output, read, len, NULL);
David Reiss382fc302007-08-25 18:01:30 +0000680 if (newiobuf == NULL) {
681 return false;
682 }
683
684 // must do this *AFTER* the call so that we don't deref the io buffer
685 Py_CLEAR(input->stringiobuf);
686 input->stringiobuf = newiobuf;
687
688 read = PycStringIO->cread(input->stringiobuf, output, len);
689
690 if (read == len) {
691 return true;
692 } else if (read == -1) {
693 return false;
694 } else {
695 // TODO(dreiss): This could be a valid code path for big binary blobs.
696 PyErr_SetString(PyExc_TypeError,
697 "refill claimed to have refilled the buffer, but didn't!!");
698 return false;
699 }
700 }
701}
702
703static int8_t readByte(DecodeBuffer* input) {
704 char* buf;
705 if (!readBytes(input, &buf, sizeof(int8_t))) {
706 return -1;
707 }
708
709 return *(int8_t*) buf;
710}
711
712static int16_t readI16(DecodeBuffer* input) {
713 char* buf;
714 if (!readBytes(input, &buf, sizeof(int16_t))) {
715 return -1;
716 }
717
718 return (int16_t) ntohs(*(int16_t*) buf);
719}
720
721static int32_t readI32(DecodeBuffer* input) {
722 char* buf;
723 if (!readBytes(input, &buf, sizeof(int32_t))) {
724 return -1;
725 }
726 return (int32_t) ntohl(*(int32_t*) buf);
727}
728
729
730static int64_t readI64(DecodeBuffer* input) {
731 char* buf;
732 if (!readBytes(input, &buf, sizeof(int64_t))) {
733 return -1;
734 }
735
736 return (int64_t) ntohll(*(int64_t*) buf);
737}
738
739static double readDouble(DecodeBuffer* input) {
740 union {
741 int64_t f;
742 double t;
743 } transfer;
744
745 transfer.f = readI64(input);
746 if (transfer.f == -1) {
747 return -1;
748 }
749 return transfer.t;
750}
751
752static bool
753checkTypeByte(DecodeBuffer* input, TType expected) {
754 TType got = readByte(input);
Mark Slee53d9c0c2007-11-26 21:15:40 +0000755 if (INT_CONV_ERROR_OCCURRED(got)) {
756 return false;
757 }
David Reiss382fc302007-08-25 18:01:30 +0000758
759 if (expected != got) {
760 PyErr_SetString(PyExc_TypeError, "got wrong ttype while reading field");
761 return false;
762 }
763 return true;
764}
765
766static bool
767skip(DecodeBuffer* input, TType type) {
768#define SKIPBYTES(n) \
769 do { \
770 if (!readBytes(input, &dummy_buf, (n))) { \
771 return false; \
772 } \
773 } while(0)
774
775 char* dummy_buf;
776
777 switch (type) {
778
779 case T_BOOL:
780 case T_I08: SKIPBYTES(1); break;
781 case T_I16: SKIPBYTES(2); break;
782 case T_I32: SKIPBYTES(4); break;
783 case T_I64:
784 case T_DOUBLE: SKIPBYTES(8); break;
785
786 case T_STRING: {
787 // TODO(dreiss): Find out if these check_ssize_t32s are really necessary.
788 int len = readI32(input);
789 if (!check_ssize_t_32(len)) {
790 return false;
791 }
792 SKIPBYTES(len);
793 break;
794 }
795
796 case T_LIST:
797 case T_SET: {
798 TType etype;
799 int len, i;
800
801 etype = readByte(input);
802 if (etype == -1) {
803 return false;
804 }
805
806 len = readI32(input);
807 if (!check_ssize_t_32(len)) {
808 return false;
809 }
810
811 for (i = 0; i < len; i++) {
812 if (!skip(input, etype)) {
813 return false;
814 }
815 }
816 break;
817 }
818
819 case T_MAP: {
820 TType ktype, vtype;
821 int len, i;
822
823 ktype = readByte(input);
824 if (ktype == -1) {
825 return false;
826 }
827
828 vtype = readByte(input);
829 if (vtype == -1) {
830 return false;
831 }
832
833 len = readI32(input);
834 if (!check_ssize_t_32(len)) {
835 return false;
836 }
837
838 for (i = 0; i < len; i++) {
839 if (!(skip(input, ktype) && skip(input, vtype))) {
840 return false;
841 }
842 }
843 break;
844 }
845
846 case T_STRUCT: {
847 while (true) {
848 TType type;
849
850 type = readByte(input);
851 if (type == -1) {
852 return false;
853 }
854
855 if (type == T_STOP)
856 break;
857
858 SKIPBYTES(2); // tag
859 if (!skip(input, type)) {
860 return false;
861 }
862 }
863 break;
864 }
865
866 case T_STOP:
867 case T_VOID:
868 case T_UTF16:
869 case T_UTF8:
870 case T_U64:
871 default:
872 PyErr_SetString(PyExc_TypeError, "Unexpected TType");
873 return false;
874
875 }
876
David Reissbc444b02008-02-14 20:20:08 +0000877 return true;
David Reiss382fc302007-08-25 18:01:30 +0000878
879#undef SKIPBYTES
880}
881
882
883/* --- HELPER FUNCTION FOR DECODE_VAL --- */
884
885static PyObject*
886decode_val(DecodeBuffer* input, TType type, PyObject* typeargs);
887
888static bool
889decode_struct(DecodeBuffer* input, PyObject* output, PyObject* spec_seq) {
890 int spec_seq_len = PyTuple_Size(spec_seq);
891 if (spec_seq_len == -1) {
892 return false;
893 }
894
895 while (true) {
896 TType type;
897 int16_t tag;
898 PyObject* item_spec;
899 PyObject* fieldval = NULL;
900 StructItemSpec parsedspec;
901
902 type = readByte(input);
Mark Slee53d9c0c2007-11-26 21:15:40 +0000903 if (type == -1) {
904 return false;
905 }
David Reiss382fc302007-08-25 18:01:30 +0000906 if (type == T_STOP) {
907 break;
908 }
909 tag = readI16(input);
Mark Slee53d9c0c2007-11-26 21:15:40 +0000910 if (INT_CONV_ERROR_OCCURRED(tag)) {
911 return false;
912 }
David Reiss382fc302007-08-25 18:01:30 +0000913 if (tag >= 0 && tag < spec_seq_len) {
914 item_spec = PyTuple_GET_ITEM(spec_seq, tag);
915 } else {
916 item_spec = Py_None;
917 }
918
919 if (item_spec == Py_None) {
920 if (!skip(input, type)) {
921 return false;
David Reissbc444b02008-02-14 20:20:08 +0000922 } else {
923 continue;
David Reiss382fc302007-08-25 18:01:30 +0000924 }
925 }
926
927 if (!parse_struct_item_spec(&parsedspec, item_spec)) {
928 return false;
929 }
930 if (parsedspec.type != type) {
David Reissa528f542009-03-24 22:48:40 +0000931 if (!skip(input, type)) {
932 PyErr_SetString(PyExc_TypeError, "struct field had wrong type while reading and can't be skipped");
933 return false;
934 } else {
935 continue;
936 }
David Reiss382fc302007-08-25 18:01:30 +0000937 }
938
939 fieldval = decode_val(input, parsedspec.type, parsedspec.typeargs);
940 if (fieldval == NULL) {
941 return false;
942 }
943
944 if (PyObject_SetAttr(output, parsedspec.attrname, fieldval) == -1) {
945 Py_DECREF(fieldval);
946 return false;
947 }
948 Py_DECREF(fieldval);
949 }
950 return true;
951}
952
953
Konrad Grochowski3b5dacb2014-11-24 10:55:31 +0100954/* --- MAIN RECURSIVE INPUT FUNCTION --- */
David Reiss382fc302007-08-25 18:01:30 +0000955
956// Returns a new reference.
957static PyObject*
958decode_val(DecodeBuffer* input, TType type, PyObject* typeargs) {
959 switch (type) {
960
961 case T_BOOL: {
962 int8_t v = readByte(input);
963 if (INT_CONV_ERROR_OCCURRED(v)) {
964 return NULL;
965 }
966
967 switch (v) {
968 case 0: Py_RETURN_FALSE;
969 case 1: Py_RETURN_TRUE;
970 // Don't laugh. This is a potentially serious issue.
971 default: PyErr_SetString(PyExc_TypeError, "boolean out of range"); return NULL;
972 }
973 break;
974 }
975 case T_I08: {
976 int8_t v = readByte(input);
977 if (INT_CONV_ERROR_OCCURRED(v)) {
978 return NULL;
979 }
980
981 return PyInt_FromLong(v);
982 }
983 case T_I16: {
984 int16_t v = readI16(input);
985 if (INT_CONV_ERROR_OCCURRED(v)) {
986 return NULL;
987 }
988 return PyInt_FromLong(v);
989 }
990 case T_I32: {
991 int32_t v = readI32(input);
992 if (INT_CONV_ERROR_OCCURRED(v)) {
993 return NULL;
994 }
995 return PyInt_FromLong(v);
996 }
997
998 case T_I64: {
999 int64_t v = readI64(input);
1000 if (INT_CONV_ERROR_OCCURRED(v)) {
1001 return NULL;
1002 }
1003 // TODO(dreiss): Find out if we can take this fastpath always when
1004 // sizeof(long) == sizeof(long long).
1005 if (CHECK_RANGE(v, LONG_MIN, LONG_MAX)) {
1006 return PyInt_FromLong((long) v);
1007 }
1008
1009 return PyLong_FromLongLong(v);
1010 }
1011
1012 case T_DOUBLE: {
1013 double v = readDouble(input);
1014 if (v == -1.0 && PyErr_Occurred()) {
1015 return false;
1016 }
1017 return PyFloat_FromDouble(v);
1018 }
1019
1020 case T_STRING: {
1021 Py_ssize_t len = readI32(input);
1022 char* buf;
1023 if (!readBytes(input, &buf, len)) {
1024 return NULL;
1025 }
1026
1027 return PyString_FromStringAndSize(buf, len);
1028 }
1029
1030 case T_LIST:
1031 case T_SET: {
1032 SetListTypeArgs parsedargs;
1033 int32_t len;
1034 PyObject* ret = NULL;
1035 int i;
1036
1037 if (!parse_set_list_args(&parsedargs, typeargs)) {
1038 return NULL;
1039 }
1040
1041 if (!checkTypeByte(input, parsedargs.element_type)) {
1042 return NULL;
1043 }
1044
1045 len = readI32(input);
Roger Meier7daf00c2015-06-03 11:45:35 +02001046 if (!check_list_length(len)) {
David Reiss382fc302007-08-25 18:01:30 +00001047 return NULL;
1048 }
1049
1050 ret = PyList_New(len);
1051 if (!ret) {
1052 return NULL;
1053 }
1054
1055 for (i = 0; i < len; i++) {
1056 PyObject* item = decode_val(input, parsedargs.element_type, parsedargs.typeargs);
1057 if (!item) {
1058 Py_DECREF(ret);
1059 return NULL;
1060 }
1061 PyList_SET_ITEM(ret, i, item);
1062 }
1063
1064 // TODO(dreiss): Consider biting the bullet and making two separate cases
1065 // for list and set, avoiding this post facto conversion.
1066 if (type == T_SET) {
1067 PyObject* setret;
1068#if (PY_VERSION_HEX < 0x02050000)
1069 // hack needed for older versions
1070 setret = PyObject_CallFunctionObjArgs((PyObject*)&PySet_Type, ret, NULL);
1071#else
1072 // official version
1073 setret = PySet_New(ret);
1074#endif
1075 Py_DECREF(ret);
1076 return setret;
1077 }
1078 return ret;
1079 }
1080
1081 case T_MAP: {
1082 int32_t len;
1083 int i;
1084 MapTypeArgs parsedargs;
1085 PyObject* ret = NULL;
1086
1087 if (!parse_map_args(&parsedargs, typeargs)) {
1088 return NULL;
1089 }
1090
1091 if (!checkTypeByte(input, parsedargs.ktag)) {
1092 return NULL;
1093 }
1094 if (!checkTypeByte(input, parsedargs.vtag)) {
1095 return NULL;
1096 }
1097
1098 len = readI32(input);
1099 if (!check_ssize_t_32(len)) {
1100 return false;
1101 }
1102
1103 ret = PyDict_New();
1104 if (!ret) {
1105 goto error;
1106 }
1107
1108 for (i = 0; i < len; i++) {
1109 PyObject* k = NULL;
1110 PyObject* v = NULL;
1111 k = decode_val(input, parsedargs.ktag, parsedargs.ktypeargs);
1112 if (k == NULL) {
1113 goto loop_error;
1114 }
1115 v = decode_val(input, parsedargs.vtag, parsedargs.vtypeargs);
1116 if (v == NULL) {
1117 goto loop_error;
1118 }
1119 if (PyDict_SetItem(ret, k, v) == -1) {
1120 goto loop_error;
1121 }
1122
1123 Py_DECREF(k);
1124 Py_DECREF(v);
1125 continue;
1126
1127 // Yuck! Destructors, anyone?
1128 loop_error:
1129 Py_XDECREF(k);
1130 Py_XDECREF(v);
1131 goto error;
1132 }
1133
1134 return ret;
1135
1136 error:
1137 Py_XDECREF(ret);
1138 return NULL;
1139 }
1140
1141 case T_STRUCT: {
1142 StructTypeArgs parsedargs;
Roger Meierc3f033f2011-09-13 13:54:05 +00001143 PyObject* ret;
David Reiss382fc302007-08-25 18:01:30 +00001144 if (!parse_struct_args(&parsedargs, typeargs)) {
1145 return NULL;
1146 }
1147
Roger Meierc3f033f2011-09-13 13:54:05 +00001148 ret = PyObject_CallObject(parsedargs.klass, NULL);
David Reiss382fc302007-08-25 18:01:30 +00001149 if (!ret) {
1150 return NULL;
1151 }
1152
1153 if (!decode_struct(input, ret, parsedargs.spec)) {
1154 Py_DECREF(ret);
1155 return NULL;
1156 }
1157
1158 return ret;
1159 }
1160
1161 case T_STOP:
1162 case T_VOID:
1163 case T_UTF16:
1164 case T_UTF8:
1165 case T_U64:
1166 default:
1167 PyErr_SetString(PyExc_TypeError, "Unexpected TType");
1168 return NULL;
1169 }
1170}
1171
1172
1173/* --- TOP-LEVEL WRAPPER FOR INPUT -- */
1174
1175static PyObject*
1176decode_binary(PyObject *self, PyObject *args) {
1177 PyObject* output_obj = NULL;
1178 PyObject* transport = NULL;
1179 PyObject* typeargs = NULL;
1180 StructTypeArgs parsedargs;
Roger Meierc3f033f2011-09-13 13:54:05 +00001181 DecodeBuffer input = {0, 0};
Roger Meier7daf00c2015-06-03 11:45:35 +02001182
David Reiss382fc302007-08-25 18:01:30 +00001183 if (!PyArg_ParseTuple(args, "OOO", &output_obj, &transport, &typeargs)) {
1184 return NULL;
1185 }
1186
1187 if (!parse_struct_args(&parsedargs, typeargs)) {
1188 return NULL;
1189 }
1190
1191 if (!decode_buffer_from_obj(&input, transport)) {
1192 return NULL;
1193 }
1194
1195 if (!decode_struct(&input, output_obj, parsedargs.spec)) {
1196 free_decodebuf(&input);
1197 return NULL;
1198 }
1199
1200 free_decodebuf(&input);
1201
1202 Py_RETURN_NONE;
1203}
1204
1205/* ====== END READING FUNCTIONS ====== */
1206
1207
1208/* -- PYTHON MODULE SETUP STUFF --- */
1209
1210static PyMethodDef ThriftFastBinaryMethods[] = {
1211
1212 {"encode_binary", encode_binary, METH_VARARGS, ""},
1213 {"decode_binary", decode_binary, METH_VARARGS, ""},
1214
1215 {NULL, NULL, 0, NULL} /* Sentinel */
1216};
1217
1218PyMODINIT_FUNC
1219initfastbinary(void) {
1220#define INIT_INTERN_STRING(value) \
1221 do { \
1222 INTERN_STRING(value) = PyString_InternFromString(#value); \
1223 if(!INTERN_STRING(value)) return; \
1224 } while(0)
1225
1226 INIT_INTERN_STRING(cstringio_buf);
1227 INIT_INTERN_STRING(cstringio_refill);
1228#undef INIT_INTERN_STRING
1229
1230 PycString_IMPORT;
1231 if (PycStringIO == NULL) return;
1232
1233 (void) Py_InitModule("thrift.protocol.fastbinary", ThriftFastBinaryMethods);
1234}