PageRenderTime 93ms CodeModel.GetById 13ms app.highlight 72ms RepoModel.GetById 2ms app.codeStats 0ms

/tags/release-0.1-rc2/hive/external/service/lib/py/thrift/protocol/fastbinary.c

#
C | 1171 lines | 856 code | 223 blank | 92 comment | 178 complexity | 9301a1fc32150646407bd114fb0c6776 MD5 | raw file
   1// Copyright (c) 2006- Facebook
   2// Distributed under the Thrift Software License
   3//
   4// See accompanying file LICENSE or visit the Thrift site at:
   5// http://developers.facebook.com/thrift/
   6//
   7// NOTE:  This code was contributed by an external developer.
   8//        The internal Thrift team has reviewed and tested it,
   9//        but we cannot guarantee that it is production-ready.
  10//        Please feel free to report bugs and/or success stories
  11//        to the public mailing list.
  12
  13#include <Python.h>
  14#include "cStringIO.h"
  15#include <stdbool.h>
  16#include <stdint.h>
  17#include <netinet/in.h>
  18
  19// TODO(dreiss): defval appears to be unused.  Look into removing it.
  20// TODO(dreiss): Make parse_spec_args recursive, and cache the output
  21//               permanently in the object.  (Malloc and orphan.)
  22// TODO(dreiss): Why do we need cStringIO for reading, why not just char*?
  23//               Can cStringIO let us work with a BufferedTransport?
  24// TODO(dreiss): Don't ignore the rv from cwrite (maybe).
  25
  26/* ====== BEGIN UTILITIES ====== */
  27
  28#define INIT_OUTBUF_SIZE 128
  29
  30// Stolen out of TProtocol.h.
  31// It would be a huge pain to have both get this from one place.
  32typedef enum TType {
  33  T_STOP       = 0,
  34  T_VOID       = 1,
  35  T_BOOL       = 2,
  36  T_BYTE       = 3,
  37  T_I08        = 3,
  38  T_I16        = 6,
  39  T_I32        = 8,
  40  T_U64        = 9,
  41  T_I64        = 10,
  42  T_DOUBLE     = 4,
  43  T_STRING     = 11,
  44  T_UTF7       = 11,
  45  T_STRUCT     = 12,
  46  T_MAP        = 13,
  47  T_SET        = 14,
  48  T_LIST       = 15,
  49  T_UTF8       = 16,
  50  T_UTF16      = 17
  51} TType;
  52
  53#ifndef __BYTE_ORDER
  54# if defined(BYTE_ORDER) && defined(LITTLE_ENDIAN) && defined(BIG_ENDIAN)
  55#  define __BYTE_ORDER BYTE_ORDER
  56#  define __LITTLE_ENDIAN LITTLE_ENDIAN
  57#  define __BIG_ENDIAN BIG_ENDIAN
  58# else
  59#  error "Cannot determine endianness"
  60# endif
  61#endif
  62
  63// Same comment as the enum.  Sorry.
  64#if __BYTE_ORDER == __BIG_ENDIAN
  65# define ntohll(n) (n)
  66# define htonll(n) (n)
  67#elif __BYTE_ORDER == __LITTLE_ENDIAN
  68# if defined(__GNUC__) && defined(__GLIBC__)
  69#  include <byteswap.h>
  70#  define ntohll(n) bswap_64(n)
  71#  define htonll(n) bswap_64(n)
  72# else /* GNUC & GLIBC */
  73#  define ntohll(n) ( (((unsigned long long)ntohl(n)) << 32) + ntohl(n >> 32) )
  74#  define htonll(n) ( (((unsigned long long)htonl(n)) << 32) + htonl(n >> 32) )
  75# endif /* GNUC & GLIBC */
  76#else /* __BYTE_ORDER */
  77# error "Can't define htonll or ntohll!"
  78#endif
  79
  80// Doing a benchmark shows that interning actually makes a difference, amazingly.
  81#define INTERN_STRING(value) _intern_ ## value
  82
  83#define INT_CONV_ERROR_OCCURRED(v) ( ((v) == -1) && PyErr_Occurred() )
  84#define CHECK_RANGE(v, min, max) ( ((v) <= (max)) && ((v) >= (min)) )
  85
  86// Py_ssize_t was not defined before Python 2.5
  87#if (PY_VERSION_HEX < 0x02050000)
  88typedef int Py_ssize_t;
  89#endif
  90
  91/**
  92 * A cache of the spec_args for a set or list,
  93 * so we don't have to keep calling PyTuple_GET_ITEM.
  94 */
  95typedef struct {
  96  TType element_type;
  97  PyObject* typeargs;
  98} SetListTypeArgs;
  99
 100/**
 101 * A cache of the spec_args for a map,
 102 * so we don't have to keep calling PyTuple_GET_ITEM.
 103 */
 104typedef struct {
 105  TType ktag;
 106  TType vtag;
 107  PyObject* ktypeargs;
 108  PyObject* vtypeargs;
 109} MapTypeArgs;
 110
 111/**
 112 * A cache of the spec_args for a struct,
 113 * so we don't have to keep calling PyTuple_GET_ITEM.
 114 */
 115typedef struct {
 116  PyObject* klass;
 117  PyObject* spec;
 118} StructTypeArgs;
 119
 120/**
 121 * A cache of the item spec from a struct specification,
 122 * so we don't have to keep calling PyTuple_GET_ITEM.
 123 */
 124typedef struct {
 125  int tag;
 126  TType type;
 127  PyObject* attrname;
 128  PyObject* typeargs;
 129  PyObject* defval;
 130} StructItemSpec;
 131
 132/**
 133 * A cache of the two key attributes of a CReadableTransport,
 134 * so we don't have to keep calling PyObject_GetAttr.
 135 */
 136typedef struct {
 137  PyObject* stringiobuf;
 138  PyObject* refill_callable;
 139} DecodeBuffer;
 140
 141/** Pointer to interned string to speed up attribute lookup. */
 142static PyObject* INTERN_STRING(cstringio_buf);
 143/** Pointer to interned string to speed up attribute lookup. */
 144static PyObject* INTERN_STRING(cstringio_refill);
 145
 146static inline bool
 147check_ssize_t_32(Py_ssize_t len) {
 148  // error from getting the int
 149  if (INT_CONV_ERROR_OCCURRED(len)) {
 150    return false;
 151  }
 152  if (!CHECK_RANGE(len, 0, INT32_MAX)) {
 153    PyErr_SetString(PyExc_OverflowError, "string size out of range");
 154    return false;
 155  }
 156  return true;
 157}
 158
 159static inline bool
 160parse_pyint(PyObject* o, int32_t* ret, int32_t min, int32_t max) {
 161  long val = PyInt_AsLong(o);
 162
 163  if (INT_CONV_ERROR_OCCURRED(val)) {
 164    return false;
 165  }
 166  if (!CHECK_RANGE(val, min, max)) {
 167    PyErr_SetString(PyExc_OverflowError, "int out of range");
 168    return false;
 169  }
 170
 171  *ret = (int32_t) val;
 172  return true;
 173}
 174
 175
 176/* --- FUNCTIONS TO PARSE STRUCT SPECIFICATOINS --- */
 177
 178static bool
 179parse_set_list_args(SetListTypeArgs* dest, PyObject* typeargs) {
 180  if (PyTuple_Size(typeargs) != 2) {
 181    PyErr_SetString(PyExc_TypeError, "expecting tuple of size 2 for list/set type args");
 182    return false;
 183  }
 184
 185  dest->element_type = PyInt_AsLong(PyTuple_GET_ITEM(typeargs, 0));
 186  if (INT_CONV_ERROR_OCCURRED(dest->element_type)) {
 187    return false;
 188  }
 189
 190  dest->typeargs = PyTuple_GET_ITEM(typeargs, 1);
 191
 192  return true;
 193}
 194
 195static bool
 196parse_map_args(MapTypeArgs* dest, PyObject* typeargs) {
 197  if (PyTuple_Size(typeargs) != 4) {
 198    PyErr_SetString(PyExc_TypeError, "expecting 4 arguments for typeargs to map");
 199    return false;
 200  }
 201
 202  dest->ktag = PyInt_AsLong(PyTuple_GET_ITEM(typeargs, 0));
 203  if (INT_CONV_ERROR_OCCURRED(dest->ktag)) {
 204    return false;
 205  }
 206
 207  dest->vtag = PyInt_AsLong(PyTuple_GET_ITEM(typeargs, 2));
 208  if (INT_CONV_ERROR_OCCURRED(dest->vtag)) {
 209    return false;
 210  }
 211
 212  dest->ktypeargs = PyTuple_GET_ITEM(typeargs, 1);
 213  dest->vtypeargs = PyTuple_GET_ITEM(typeargs, 3);
 214
 215  return true;
 216}
 217
 218static bool
 219parse_struct_args(StructTypeArgs* dest, PyObject* typeargs) {
 220  if (PyTuple_Size(typeargs) != 2) {
 221    PyErr_SetString(PyExc_TypeError, "expecting tuple of size 2 for struct args");
 222    return false;
 223  }
 224
 225  dest->klass = PyTuple_GET_ITEM(typeargs, 0);
 226  dest->spec = PyTuple_GET_ITEM(typeargs, 1);
 227
 228  return true;
 229}
 230
 231static int
 232parse_struct_item_spec(StructItemSpec* dest, PyObject* spec_tuple) {
 233
 234  // i'd like to use ParseArgs here, but it seems to be a bottleneck.
 235  if (PyTuple_Size(spec_tuple) != 5) {
 236    PyErr_SetString(PyExc_TypeError, "expecting 5 arguments for spec tuple");
 237    return false;
 238  }
 239
 240  dest->tag = PyInt_AsLong(PyTuple_GET_ITEM(spec_tuple, 0));
 241  if (INT_CONV_ERROR_OCCURRED(dest->tag)) {
 242    return false;
 243  }
 244
 245  dest->type = PyInt_AsLong(PyTuple_GET_ITEM(spec_tuple, 1));
 246  if (INT_CONV_ERROR_OCCURRED(dest->type)) {
 247    return false;
 248  }
 249
 250  dest->attrname = PyTuple_GET_ITEM(spec_tuple, 2);
 251  dest->typeargs = PyTuple_GET_ITEM(spec_tuple, 3);
 252  dest->defval = PyTuple_GET_ITEM(spec_tuple, 4);
 253  return true;
 254}
 255
 256/* ====== END UTILITIES ====== */
 257
 258
 259/* ====== BEGIN WRITING FUNCTIONS ====== */
 260
 261/* --- LOW-LEVEL WRITING FUNCTIONS --- */
 262
 263static void writeByte(PyObject* outbuf, int8_t val) {
 264  int8_t net = val;
 265  PycStringIO->cwrite(outbuf, (char*)&net, sizeof(int8_t));
 266}
 267
 268static void writeI16(PyObject* outbuf, int16_t val) {
 269  int16_t net = (int16_t)htons(val);
 270  PycStringIO->cwrite(outbuf, (char*)&net, sizeof(int16_t));
 271}
 272
 273static void writeI32(PyObject* outbuf, int32_t val) {
 274  int32_t net = (int32_t)htonl(val);
 275  PycStringIO->cwrite(outbuf, (char*)&net, sizeof(int32_t));
 276}
 277
 278static void writeI64(PyObject* outbuf, int64_t val) {
 279  int64_t net = (int64_t)htonll(val);
 280  PycStringIO->cwrite(outbuf, (char*)&net, sizeof(int64_t));
 281}
 282
 283static void writeDouble(PyObject* outbuf, double dub) {
 284  // Unfortunately, bitwise_cast doesn't work in C.  Bad C!
 285  union {
 286    double f;
 287    int64_t t;
 288  } transfer;
 289  transfer.f = dub;
 290  writeI64(outbuf, transfer.t);
 291}
 292
 293
 294/* --- MAIN RECURSIVE OUTPUT FUCNTION -- */
 295
 296static int
 297output_val(PyObject* output, PyObject* value, TType type, PyObject* typeargs) {
 298  /*
 299   * Refcounting Strategy:
 300   *
 301   * We assume that elements of the thrift_spec tuple are not going to be
 302   * mutated, so we don't ref count those at all. Other than that, we try to
 303   * keep a reference to all the user-created objects while we work with them.
 304   * output_val assumes that a reference is already held. The *caller* is
 305   * responsible for handling references
 306   */
 307
 308  switch (type) {
 309
 310  case T_BOOL: {
 311    int v = PyObject_IsTrue(value);
 312    if (v == -1) {
 313      return false;
 314    }
 315
 316    writeByte(output, (int8_t) v);
 317    break;
 318  }
 319  case T_I08: {
 320    int32_t val;
 321
 322    if (!parse_pyint(value, &val, INT8_MIN, INT8_MAX)) {
 323      return false;
 324    }
 325
 326    writeByte(output, (int8_t) val);
 327    break;
 328  }
 329  case T_I16: {
 330    int32_t val;
 331
 332    if (!parse_pyint(value, &val, INT16_MIN, INT16_MAX)) {
 333      return false;
 334    }
 335
 336    writeI16(output, (int16_t) val);
 337    break;
 338  }
 339  case T_I32: {
 340    int32_t val;
 341
 342    if (!parse_pyint(value, &val, INT32_MIN, INT32_MAX)) {
 343      return false;
 344    }
 345
 346    writeI32(output, val);
 347    break;
 348  }
 349  case T_I64: {
 350    int64_t nval = PyLong_AsLongLong(value);
 351
 352    if (INT_CONV_ERROR_OCCURRED(nval)) {
 353      return false;
 354    }
 355
 356    if (!CHECK_RANGE(nval, INT64_MIN, INT64_MAX)) {
 357      PyErr_SetString(PyExc_OverflowError, "int out of range");
 358      return false;
 359    }
 360
 361    writeI64(output, nval);
 362    break;
 363  }
 364
 365  case T_DOUBLE: {
 366    double nval = PyFloat_AsDouble(value);
 367    if (nval == -1.0 && PyErr_Occurred()) {
 368      return false;
 369    }
 370
 371    writeDouble(output, nval);
 372    break;
 373  }
 374
 375  case T_STRING: {
 376    Py_ssize_t len = PyString_Size(value);
 377
 378    if (!check_ssize_t_32(len)) {
 379      return false;
 380    }
 381
 382    writeI32(output, (int32_t) len);
 383    PycStringIO->cwrite(output, PyString_AsString(value), (int32_t) len);
 384    break;
 385  }
 386
 387  case T_LIST:
 388  case T_SET: {
 389    Py_ssize_t len;
 390    SetListTypeArgs parsedargs;
 391    PyObject *item;
 392    PyObject *iterator;
 393
 394    if (!parse_set_list_args(&parsedargs, typeargs)) {
 395      return false;
 396    }
 397
 398    len = PyObject_Length(value);
 399
 400    if (!check_ssize_t_32(len)) {
 401      return false;
 402    }
 403
 404    writeByte(output, parsedargs.element_type);
 405    writeI32(output, (int32_t) len);
 406
 407    iterator =  PyObject_GetIter(value);
 408    if (iterator == NULL) {
 409      return false;
 410    }
 411
 412    while ((item = PyIter_Next(iterator))) {
 413      if (!output_val(output, item, parsedargs.element_type, parsedargs.typeargs)) {
 414        Py_DECREF(item);
 415        Py_DECREF(iterator);
 416        return false;
 417      }
 418      Py_DECREF(item);
 419    }
 420
 421    Py_DECREF(iterator);
 422
 423    if (PyErr_Occurred()) {
 424      return false;
 425    }
 426
 427    break;
 428  }
 429
 430  case T_MAP: {
 431    PyObject *k, *v;
 432    Py_ssize_t pos = 0;
 433    Py_ssize_t len;
 434
 435    MapTypeArgs parsedargs;
 436
 437    len = PyDict_Size(value);
 438    if (!check_ssize_t_32(len)) {
 439      return false;
 440    }
 441
 442    if (!parse_map_args(&parsedargs, typeargs)) {
 443      return false;
 444    }
 445
 446    writeByte(output, parsedargs.ktag);
 447    writeByte(output, parsedargs.vtag);
 448    writeI32(output, len);
 449
 450    // TODO(bmaurer): should support any mapping, not just dicts
 451    while (PyDict_Next(value, &pos, &k, &v)) {
 452      // TODO(dreiss): Think hard about whether these INCREFs actually
 453      //               turn any unsafe scenarios into safe scenarios.
 454      Py_INCREF(k);
 455      Py_INCREF(v);
 456
 457      if (!output_val(output, k, parsedargs.ktag, parsedargs.ktypeargs)
 458          || !output_val(output, v, parsedargs.vtag, parsedargs.vtypeargs)) {
 459        Py_DECREF(k);
 460        Py_DECREF(v);
 461        return false;
 462      }
 463    }
 464    break;
 465  }
 466
 467  // TODO(dreiss): Consider breaking this out as a function
 468  //               the way we did for decode_struct.
 469  case T_STRUCT: {
 470    StructTypeArgs parsedargs;
 471    Py_ssize_t nspec;
 472    Py_ssize_t i;
 473
 474    if (!parse_struct_args(&parsedargs, typeargs)) {
 475      return false;
 476    }
 477
 478    nspec = PyTuple_Size(parsedargs.spec);
 479
 480    if (nspec == -1) {
 481      return false;
 482    }
 483
 484    for (i = 0; i < nspec; i++) {
 485      StructItemSpec parsedspec;
 486      PyObject* spec_tuple;
 487      PyObject* instval = NULL;
 488
 489      spec_tuple = PyTuple_GET_ITEM(parsedargs.spec, i);
 490      if (spec_tuple == Py_None) {
 491        continue;
 492      }
 493
 494      if (!parse_struct_item_spec (&parsedspec, spec_tuple)) {
 495        return false;
 496      }
 497
 498      instval = PyObject_GetAttr(value, parsedspec.attrname);
 499
 500      if (!instval) {
 501        return false;
 502      }
 503
 504      if (instval == Py_None) {
 505        Py_DECREF(instval);
 506        continue;
 507      }
 508
 509      writeByte(output, (int8_t) parsedspec.type);
 510      writeI16(output, parsedspec.tag);
 511
 512      if (!output_val(output, instval, parsedspec.type, parsedspec.typeargs)) {
 513        Py_DECREF(instval);
 514        return false;
 515      }
 516
 517      Py_DECREF(instval);
 518    }
 519
 520    writeByte(output, (int8_t)T_STOP);
 521    break;
 522  }
 523
 524  case T_STOP:
 525  case T_VOID:
 526  case T_UTF16:
 527  case T_UTF8:
 528  case T_U64:
 529  default:
 530    PyErr_SetString(PyExc_TypeError, "Unexpected TType");
 531    return false;
 532
 533  }
 534
 535  return true;
 536}
 537
 538
 539/* --- TOP-LEVEL WRAPPER FOR OUTPUT -- */
 540
 541static PyObject *
 542encode_binary(PyObject *self, PyObject *args) {
 543  PyObject* enc_obj;
 544  PyObject* type_args;
 545  PyObject* buf;
 546  PyObject* ret = NULL;
 547
 548  if (!PyArg_ParseTuple(args, "OO", &enc_obj, &type_args)) {
 549    return NULL;
 550  }
 551
 552  buf = PycStringIO->NewOutput(INIT_OUTBUF_SIZE);
 553  if (output_val(buf, enc_obj, T_STRUCT, type_args)) {
 554    ret = PycStringIO->cgetvalue(buf);
 555  }
 556
 557  Py_DECREF(buf);
 558  return ret;
 559}
 560
 561/* ====== END WRITING FUNCTIONS ====== */
 562
 563
 564/* ====== BEGIN READING FUNCTIONS ====== */
 565
 566/* --- LOW-LEVEL READING FUNCTIONS --- */
 567
 568static void
 569free_decodebuf(DecodeBuffer* d) {
 570  Py_XDECREF(d->stringiobuf);
 571  Py_XDECREF(d->refill_callable);
 572}
 573
 574static bool
 575decode_buffer_from_obj(DecodeBuffer* dest, PyObject* obj) {
 576  dest->stringiobuf = PyObject_GetAttr(obj, INTERN_STRING(cstringio_buf));
 577  if (!dest->stringiobuf) {
 578    return false;
 579  }
 580
 581  if (!PycStringIO_InputCheck(dest->stringiobuf)) {
 582    free_decodebuf(dest);
 583    PyErr_SetString(PyExc_TypeError, "expecting stringio input");
 584    return false;
 585  }
 586
 587  dest->refill_callable = PyObject_GetAttr(obj, INTERN_STRING(cstringio_refill));
 588
 589  if(!dest->refill_callable) {
 590    free_decodebuf(dest);
 591    return false;
 592  }
 593
 594  if (!PyCallable_Check(dest->refill_callable)) {
 595    free_decodebuf(dest);
 596    PyErr_SetString(PyExc_TypeError, "expecting callable");
 597    return false;
 598  }
 599
 600  return true;
 601}
 602
 603static bool readBytes(DecodeBuffer* input, char** output, int len) {
 604  int read;
 605
 606  // TODO(dreiss): Don't fear the malloc.  Think about taking a copy of
 607  //               the partial read instead of forcing the transport
 608  //               to prepend it to its buffer.
 609
 610  read = PycStringIO->cread(input->stringiobuf, output, len);
 611
 612  if (read == len) {
 613    return true;
 614  } else if (read == -1) {
 615    return false;
 616  } else {
 617    PyObject* newiobuf;
 618
 619    // using building functions as this is a rare codepath
 620    newiobuf = PyObject_CallFunction(
 621        input->refill_callable, "s#i", *output, read, len, NULL);
 622    if (newiobuf == NULL) {
 623      return false;
 624    }
 625
 626    // must do this *AFTER* the call so that we don't deref the io buffer
 627    Py_CLEAR(input->stringiobuf);
 628    input->stringiobuf = newiobuf;
 629
 630    read = PycStringIO->cread(input->stringiobuf, output, len);
 631
 632    if (read == len) {
 633      return true;
 634    } else if (read == -1) {
 635      return false;
 636    } else {
 637      // TODO(dreiss): This could be a valid code path for big binary blobs.
 638      PyErr_SetString(PyExc_TypeError,
 639          "refill claimed to have refilled the buffer, but didn't!!");
 640      return false;
 641    }
 642  }
 643}
 644
 645static int8_t readByte(DecodeBuffer* input) {
 646  char* buf;
 647  if (!readBytes(input, &buf, sizeof(int8_t))) {
 648    return -1;
 649  }
 650
 651  return *(int8_t*) buf;
 652}
 653
 654static int16_t readI16(DecodeBuffer* input) {
 655  char* buf;
 656  if (!readBytes(input, &buf, sizeof(int16_t))) {
 657    return -1;
 658  }
 659
 660  return (int16_t) ntohs(*(int16_t*) buf);
 661}
 662
 663static int32_t readI32(DecodeBuffer* input) {
 664  char* buf;
 665  if (!readBytes(input, &buf, sizeof(int32_t))) {
 666    return -1;
 667  }
 668  return (int32_t) ntohl(*(int32_t*) buf);
 669}
 670
 671
 672static int64_t readI64(DecodeBuffer* input) {
 673  char* buf;
 674  if (!readBytes(input, &buf, sizeof(int64_t))) {
 675    return -1;
 676  }
 677
 678  return (int64_t) ntohll(*(int64_t*) buf);
 679}
 680
 681static double readDouble(DecodeBuffer* input) {
 682  union {
 683    int64_t f;
 684    double t;
 685  } transfer;
 686
 687  transfer.f = readI64(input);
 688  if (transfer.f == -1) {
 689    return -1;
 690  }
 691  return transfer.t;
 692}
 693
 694static bool
 695checkTypeByte(DecodeBuffer* input, TType expected) {
 696  TType got = readByte(input);
 697  if (INT_CONV_ERROR_OCCURRED(got)) {
 698    return false;
 699  }
 700
 701  if (expected != got) {
 702    PyErr_SetString(PyExc_TypeError, "got wrong ttype while reading field");
 703    return false;
 704  }
 705  return true;
 706}
 707
 708static bool
 709skip(DecodeBuffer* input, TType type) {
 710#define SKIPBYTES(n) \
 711  do { \
 712    if (!readBytes(input, &dummy_buf, (n))) { \
 713      return false; \
 714    } \
 715  } while(0)
 716
 717  char* dummy_buf;
 718
 719  switch (type) {
 720
 721  case T_BOOL:
 722  case T_I08: SKIPBYTES(1); break;
 723  case T_I16: SKIPBYTES(2); break;
 724  case T_I32: SKIPBYTES(4); break;
 725  case T_I64:
 726  case T_DOUBLE: SKIPBYTES(8); break;
 727
 728  case T_STRING: {
 729    // TODO(dreiss): Find out if these check_ssize_t32s are really necessary.
 730    int len = readI32(input);
 731    if (!check_ssize_t_32(len)) {
 732      return false;
 733    }
 734    SKIPBYTES(len);
 735    break;
 736  }
 737
 738  case T_LIST:
 739  case T_SET: {
 740    TType etype;
 741    int len, i;
 742
 743    etype = readByte(input);
 744    if (etype == -1) {
 745      return false;
 746    }
 747
 748    len = readI32(input);
 749    if (!check_ssize_t_32(len)) {
 750      return false;
 751    }
 752
 753    for (i = 0; i < len; i++) {
 754      if (!skip(input, etype)) {
 755        return false;
 756      }
 757    }
 758    break;
 759  }
 760
 761  case T_MAP: {
 762    TType ktype, vtype;
 763    int len, i;
 764
 765    ktype = readByte(input);
 766    if (ktype == -1) {
 767      return false;
 768    }
 769
 770    vtype = readByte(input);
 771    if (vtype == -1) {
 772      return false;
 773    }
 774
 775    len = readI32(input);
 776    if (!check_ssize_t_32(len)) {
 777      return false;
 778    }
 779
 780    for (i = 0; i < len; i++) {
 781      if (!(skip(input, ktype) && skip(input, vtype))) {
 782        return false;
 783      }
 784    }
 785    break;
 786  }
 787
 788  case T_STRUCT: {
 789    while (true) {
 790      TType type;
 791
 792      type = readByte(input);
 793      if (type == -1) {
 794        return false;
 795      }
 796
 797      if (type == T_STOP)
 798        break;
 799
 800      SKIPBYTES(2); // tag
 801      if (!skip(input, type)) {
 802        return false;
 803      }
 804    }
 805    break;
 806  }
 807
 808  case T_STOP:
 809  case T_VOID:
 810  case T_UTF16:
 811  case T_UTF8:
 812  case T_U64:
 813  default:
 814    PyErr_SetString(PyExc_TypeError, "Unexpected TType");
 815    return false;
 816
 817  }
 818
 819  return true;
 820
 821#undef SKIPBYTES
 822}
 823
 824
 825/* --- HELPER FUNCTION FOR DECODE_VAL --- */
 826
 827static PyObject*
 828decode_val(DecodeBuffer* input, TType type, PyObject* typeargs);
 829
 830static bool
 831decode_struct(DecodeBuffer* input, PyObject* output, PyObject* spec_seq) {
 832  int spec_seq_len = PyTuple_Size(spec_seq);
 833  if (spec_seq_len == -1) {
 834    return false;
 835  }
 836
 837  while (true) {
 838    TType type;
 839    int16_t tag;
 840    PyObject* item_spec;
 841    PyObject* fieldval = NULL;
 842    StructItemSpec parsedspec;
 843
 844    type = readByte(input);
 845    if (type == -1) {
 846      return false;
 847    }
 848    if (type == T_STOP) {
 849      break;
 850    }
 851    tag = readI16(input);
 852    if (INT_CONV_ERROR_OCCURRED(tag)) {
 853      return false;
 854    }
 855    if (tag >= 0 && tag < spec_seq_len) {
 856      item_spec = PyTuple_GET_ITEM(spec_seq, tag);
 857    } else {
 858      item_spec = Py_None;
 859    }
 860
 861    if (item_spec == Py_None) {
 862      if (!skip(input, type)) {
 863        return false;
 864      } else {
 865        continue;
 866      }
 867    }
 868
 869    if (!parse_struct_item_spec(&parsedspec, item_spec)) {
 870      return false;
 871    }
 872    if (parsedspec.type != type) {
 873      PyErr_SetString(PyExc_TypeError, "struct field had wrong type while reading");
 874      return false;
 875    }
 876
 877    fieldval = decode_val(input, parsedspec.type, parsedspec.typeargs);
 878    if (fieldval == NULL) {
 879      return false;
 880    }
 881
 882    if (PyObject_SetAttr(output, parsedspec.attrname, fieldval) == -1) {
 883      Py_DECREF(fieldval);
 884      return false;
 885    }
 886    Py_DECREF(fieldval);
 887  }
 888  return true;
 889}
 890
 891
 892/* --- MAIN RECURSIVE INPUT FUCNTION --- */
 893
 894// Returns a new reference.
 895static PyObject*
 896decode_val(DecodeBuffer* input, TType type, PyObject* typeargs) {
 897  switch (type) {
 898
 899  case T_BOOL: {
 900    int8_t v = readByte(input);
 901    if (INT_CONV_ERROR_OCCURRED(v)) {
 902      return NULL;
 903    }
 904
 905    switch (v) {
 906    case 0: Py_RETURN_FALSE;
 907    case 1: Py_RETURN_TRUE;
 908    // Don't laugh.  This is a potentially serious issue.
 909    default: PyErr_SetString(PyExc_TypeError, "boolean out of range"); return NULL;
 910    }
 911    break;
 912  }
 913  case T_I08: {
 914    int8_t v = readByte(input);
 915    if (INT_CONV_ERROR_OCCURRED(v)) {
 916      return NULL;
 917    }
 918
 919    return PyInt_FromLong(v);
 920  }
 921  case T_I16: {
 922    int16_t v = readI16(input);
 923    if (INT_CONV_ERROR_OCCURRED(v)) {
 924      return NULL;
 925    }
 926    return PyInt_FromLong(v);
 927  }
 928  case T_I32: {
 929    int32_t v = readI32(input);
 930    if (INT_CONV_ERROR_OCCURRED(v)) {
 931      return NULL;
 932    }
 933    return PyInt_FromLong(v);
 934  }
 935
 936  case T_I64: {
 937    int64_t v = readI64(input);
 938    if (INT_CONV_ERROR_OCCURRED(v)) {
 939      return NULL;
 940    }
 941    // TODO(dreiss): Find out if we can take this fastpath always when
 942    //               sizeof(long) == sizeof(long long).
 943    if (CHECK_RANGE(v, LONG_MIN, LONG_MAX)) {
 944      return PyInt_FromLong((long) v);
 945    }
 946
 947    return PyLong_FromLongLong(v);
 948  }
 949
 950  case T_DOUBLE: {
 951    double v = readDouble(input);
 952    if (v == -1.0 && PyErr_Occurred()) {
 953      return false;
 954    }
 955    return PyFloat_FromDouble(v);
 956  }
 957
 958  case T_STRING: {
 959    Py_ssize_t len = readI32(input);
 960    char* buf;
 961    if (!readBytes(input, &buf, len)) {
 962      return NULL;
 963    }
 964
 965    return PyString_FromStringAndSize(buf, len);
 966  }
 967
 968  case T_LIST:
 969  case T_SET: {
 970    SetListTypeArgs parsedargs;
 971    int32_t len;
 972    PyObject* ret = NULL;
 973    int i;
 974
 975    if (!parse_set_list_args(&parsedargs, typeargs)) {
 976      return NULL;
 977    }
 978
 979    if (!checkTypeByte(input, parsedargs.element_type)) {
 980      return NULL;
 981    }
 982
 983    len = readI32(input);
 984    if (!check_ssize_t_32(len)) {
 985      return NULL;
 986    }
 987
 988    ret = PyList_New(len);
 989    if (!ret) {
 990      return NULL;
 991    }
 992
 993    for (i = 0; i < len; i++) {
 994      PyObject* item = decode_val(input, parsedargs.element_type, parsedargs.typeargs);
 995      if (!item) {
 996        Py_DECREF(ret);
 997        return NULL;
 998      }
 999      PyList_SET_ITEM(ret, i, item);
1000    }
1001
1002    // TODO(dreiss): Consider biting the bullet and making two separate cases
1003    //               for list and set, avoiding this post facto conversion.
1004    if (type == T_SET) {
1005      PyObject* setret;
1006#if (PY_VERSION_HEX < 0x02050000)
1007      // hack needed for older versions
1008      setret = PyObject_CallFunctionObjArgs((PyObject*)&PySet_Type, ret, NULL);
1009#else
1010      // official version
1011      setret = PySet_New(ret);
1012#endif
1013      Py_DECREF(ret);
1014      return setret;
1015    }
1016    return ret;
1017  }
1018
1019  case T_MAP: {
1020    int32_t len;
1021    int i;
1022    MapTypeArgs parsedargs;
1023    PyObject* ret = NULL;
1024
1025    if (!parse_map_args(&parsedargs, typeargs)) {
1026      return NULL;
1027    }
1028
1029    if (!checkTypeByte(input, parsedargs.ktag)) {
1030      return NULL;
1031    }
1032    if (!checkTypeByte(input, parsedargs.vtag)) {
1033      return NULL;
1034    }
1035
1036    len = readI32(input);
1037    if (!check_ssize_t_32(len)) {
1038      return false;
1039    }
1040
1041    ret = PyDict_New();
1042    if (!ret) {
1043      goto error;
1044    }
1045
1046    for (i = 0; i < len; i++) {
1047      PyObject* k = NULL;
1048      PyObject* v = NULL;
1049      k = decode_val(input, parsedargs.ktag, parsedargs.ktypeargs);
1050      if (k == NULL) {
1051        goto loop_error;
1052      }
1053      v = decode_val(input, parsedargs.vtag, parsedargs.vtypeargs);
1054      if (v == NULL) {
1055        goto loop_error;
1056      }
1057      if (PyDict_SetItem(ret, k, v) == -1) {
1058        goto loop_error;
1059      }
1060
1061      Py_DECREF(k);
1062      Py_DECREF(v);
1063      continue;
1064
1065      // Yuck!  Destructors, anyone?
1066      loop_error:
1067      Py_XDECREF(k);
1068      Py_XDECREF(v);
1069      goto error;
1070    }
1071
1072    return ret;
1073
1074    error:
1075    Py_XDECREF(ret);
1076    return NULL;
1077  }
1078
1079  case T_STRUCT: {
1080    StructTypeArgs parsedargs;
1081    if (!parse_struct_args(&parsedargs, typeargs)) {
1082      return NULL;
1083    }
1084
1085    PyObject* ret = PyObject_CallObject(parsedargs.klass, NULL);
1086    if (!ret) {
1087      return NULL;
1088    }
1089
1090    if (!decode_struct(input, ret, parsedargs.spec)) {
1091      Py_DECREF(ret);
1092      return NULL;
1093    }
1094
1095    return ret;
1096  }
1097
1098  case T_STOP:
1099  case T_VOID:
1100  case T_UTF16:
1101  case T_UTF8:
1102  case T_U64:
1103  default:
1104    PyErr_SetString(PyExc_TypeError, "Unexpected TType");
1105    return NULL;
1106  }
1107}
1108
1109
1110/* --- TOP-LEVEL WRAPPER FOR INPUT -- */
1111
1112static PyObject*
1113decode_binary(PyObject *self, PyObject *args) {
1114  PyObject* output_obj = NULL;
1115  PyObject* transport = NULL;
1116  PyObject* typeargs = NULL;
1117  StructTypeArgs parsedargs;
1118  DecodeBuffer input = {};
1119
1120  if (!PyArg_ParseTuple(args, "OOO", &output_obj, &transport, &typeargs)) {
1121    return NULL;
1122  }
1123
1124  if (!parse_struct_args(&parsedargs, typeargs)) {
1125    return NULL;
1126  }
1127
1128  if (!decode_buffer_from_obj(&input, transport)) {
1129    return NULL;
1130  }
1131
1132  if (!decode_struct(&input, output_obj, parsedargs.spec)) {
1133    free_decodebuf(&input);
1134    return NULL;
1135  }
1136
1137  free_decodebuf(&input);
1138
1139  Py_RETURN_NONE;
1140}
1141
1142/* ====== END READING FUNCTIONS ====== */
1143
1144
1145/* -- PYTHON MODULE SETUP STUFF --- */
1146
1147static PyMethodDef ThriftFastBinaryMethods[] = {
1148
1149  {"encode_binary",  encode_binary, METH_VARARGS, ""},
1150  {"decode_binary",  decode_binary, METH_VARARGS, ""},
1151
1152  {NULL, NULL, 0, NULL}        /* Sentinel */
1153};
1154
1155PyMODINIT_FUNC
1156initfastbinary(void) {
1157#define INIT_INTERN_STRING(value) \
1158  do { \
1159    INTERN_STRING(value) = PyString_InternFromString(#value); \
1160    if(!INTERN_STRING(value)) return; \
1161  } while(0)
1162
1163  INIT_INTERN_STRING(cstringio_buf);
1164  INIT_INTERN_STRING(cstringio_refill);
1165#undef INIT_INTERN_STRING
1166
1167  PycString_IMPORT;
1168  if (PycStringIO == NULL) return;
1169
1170  (void) Py_InitModule("thrift.protocol.fastbinary", ThriftFastBinaryMethods);
1171}