/Python/marshal.c

   1
   2/* Write Python objects to files and read them back.
   3   This is intended for writing and reading compiled Python code only;
   4   a true persistent storage facility would be much harder, since
   5   it would have to take circular links and sharing into account. */
   6
   7#define PY_SSIZE_T_CLEAN
   8
   9#include "Python.h"
  10#include "longintrepr.h"
  11#include "code.h"
  12#include "marshal.h"
  13
  14/* High water mark to determine when the marshalled object is dangerously deep
  15 * and risks coring the interpreter.  When the object stack gets this deep,
  16 * raise an exception instead of continuing.
  17 */
  18#define MAX_MARSHAL_STACK_DEPTH 2000
  19
  20#define TYPE_NULL		'0'
  21#define TYPE_NONE		'N'
  22#define TYPE_FALSE		'F'
  23#define TYPE_TRUE		'T'
  24#define TYPE_STOPITER		'S'
  25#define TYPE_ELLIPSIS   	'.'
  26#define TYPE_INT		'i'
  27#define TYPE_INT64		'I'
  28#define TYPE_FLOAT		'f'
  29#define TYPE_BINARY_FLOAT	'g'
  30#define TYPE_COMPLEX		'x'
  31#define TYPE_BINARY_COMPLEX	'y'
  32#define TYPE_LONG		'l'
  33#define TYPE_STRING		's'
  34#define TYPE_INTERNED		't'
  35#define TYPE_STRINGREF		'R'
  36#define TYPE_TUPLE		'('
  37#define TYPE_LIST		'['
  38#define TYPE_DICT		'{'
  39#define TYPE_CODE		'c'
  40#define TYPE_UNICODE		'u'
  41#define TYPE_UNKNOWN		'?'
  42#define TYPE_SET		'<'
  43#define TYPE_FROZENSET  	'>'
  44
  45typedef struct {
  46	FILE *fp;
  47	int error;
  48	int depth;
  49	/* If fp == NULL, the following are valid: */
  50	PyObject *str;
  51	char *ptr;
  52	char *end;
  53	PyObject *strings; /* dict on marshal, list on unmarshal */
  54	int version;
  55} WFILE;
  56
  57#define w_byte(c, p) if (((p)->fp)) putc((c), (p)->fp); \
  58		      else if ((p)->ptr != (p)->end) *(p)->ptr++ = (c); \
  59			   else w_more(c, p)
  60
  61static void
  62w_more(int c, WFILE *p)
  63{
  64	Py_ssize_t size, newsize;
  65	if (p->str == NULL)
  66		return; /* An error already occurred */
  67	size = PyString_Size(p->str);
  68	newsize = size + size + 1024;
  69	if (newsize > 32*1024*1024) {
  70		newsize = size + (size >> 3);	/* 12.5% overallocation */
  71	}
  72	if (_PyString_Resize(&p->str, newsize) != 0) {
  73		p->ptr = p->end = NULL;
  74	}
  75	else {
  76		p->ptr = PyString_AS_STRING((PyStringObject *)p->str) + size;
  77		p->end =
  78			PyString_AS_STRING((PyStringObject *)p->str) + newsize;
  79		*p->ptr++ = Py_SAFE_DOWNCAST(c, int, char);
  80	}
  81}
  82
  83static void
  84w_string(char *s, int n, WFILE *p)
  85{
  86	if (p->fp != NULL) {
  87		fwrite(s, 1, n, p->fp);
  88	}
  89	else {
  90		while (--n >= 0) {
  91			w_byte(*s, p);
  92			s++;
  93		}
  94	}
  95}
  96
  97static void
  98w_short(int x, WFILE *p)
  99{
 100	w_byte((char)( x      & 0xff), p);
 101	w_byte((char)((x>> 8) & 0xff), p);
 102}
 103
 104static void
 105w_long(long x, WFILE *p)
 106{
 107	w_byte((char)( x      & 0xff), p);
 108	w_byte((char)((x>> 8) & 0xff), p);
 109	w_byte((char)((x>>16) & 0xff), p);
 110	w_byte((char)((x>>24) & 0xff), p);
 111}
 112
 113#if SIZEOF_LONG > 4
 114static void
 115w_long64(long x, WFILE *p)
 116{
 117	w_long(x, p);
 118	w_long(x>>32, p);
 119}
 120#endif
 121
 122static void
 123w_object(PyObject *v, WFILE *p)
 124{
 125	Py_ssize_t i, n;
 126
 127	p->depth++;
 128
 129	if (p->depth > MAX_MARSHAL_STACK_DEPTH) {
 130		p->error = 2;
 131	}
 132	else if (v == NULL) {
 133		w_byte(TYPE_NULL, p);
 134	}
 135	else if (v == Py_None) {
 136		w_byte(TYPE_NONE, p);
 137	}
 138	else if (v == PyExc_StopIteration) {
 139		w_byte(TYPE_STOPITER, p);
 140	}
 141	else if (v == Py_Ellipsis) {
 142	        w_byte(TYPE_ELLIPSIS, p);
 143	}
 144	else if (v == Py_False) {
 145	        w_byte(TYPE_FALSE, p);
 146	}
 147	else if (v == Py_True) {
 148	        w_byte(TYPE_TRUE, p);
 149	}
 150	else if (PyInt_CheckExact(v)) {
 151		long x = PyInt_AS_LONG((PyIntObject *)v);
 152#if SIZEOF_LONG > 4
 153		long y = Py_ARITHMETIC_RIGHT_SHIFT(long, x, 31);
 154		if (y && y != -1) {
 155			w_byte(TYPE_INT64, p);
 156			w_long64(x, p);
 157		}
 158		else
 159#endif
 160			{
 161			w_byte(TYPE_INT, p);
 162			w_long(x, p);
 163		}
 164	}
 165	else if (PyLong_CheckExact(v)) {
 166		PyLongObject *ob = (PyLongObject *)v;
 167		w_byte(TYPE_LONG, p);
 168		n = ob->ob_size;
 169		w_long((long)n, p);
 170		if (n < 0)
 171			n = -n;
 172		for (i = 0; i < n; i++)
 173			w_short(ob->ob_digit[i], p);
 174	}
 175	else if (PyFloat_CheckExact(v)) {
 176		if (p->version > 1) {
 177			unsigned char buf[8];
 178			if (_PyFloat_Pack8(PyFloat_AsDouble(v), 
 179					   buf, 1) < 0) {
 180				p->error = 1;
 181				return;
 182			}
 183			w_byte(TYPE_BINARY_FLOAT, p);
 184			w_string((char*)buf, 8, p);
 185		}
 186		else {
 187			char buf[256]; /* Plenty to format any double */
 188			PyFloat_AsReprString(buf, (PyFloatObject *)v);
 189			n = strlen(buf);
 190			w_byte(TYPE_FLOAT, p);
 191			w_byte((int)n, p);
 192			w_string(buf, (int)n, p);
 193		}
 194	}
 195#ifndef WITHOUT_COMPLEX
 196	else if (PyComplex_CheckExact(v)) {
 197		if (p->version > 1) {
 198			unsigned char buf[8];
 199			if (_PyFloat_Pack8(PyComplex_RealAsDouble(v),
 200					   buf, 1) < 0) {
 201				p->error = 1;
 202				return;
 203			}
 204			w_byte(TYPE_BINARY_COMPLEX, p);
 205			w_string((char*)buf, 8, p);
 206			if (_PyFloat_Pack8(PyComplex_ImagAsDouble(v), 
 207					   buf, 1) < 0) {
 208				p->error = 1;
 209				return;
 210			}
 211			w_string((char*)buf, 8, p);
 212		}
 213		else {
 214			char buf[256]; /* Plenty to format any double */
 215			PyFloatObject *temp;
 216			w_byte(TYPE_COMPLEX, p);
 217			temp = (PyFloatObject*)PyFloat_FromDouble(
 218				PyComplex_RealAsDouble(v));
 219			if (!temp) {
 220				p->error = 1;
 221				return;
 222			}
 223			PyFloat_AsReprString(buf, temp);
 224			Py_DECREF(temp);
 225			n = strlen(buf);
 226			w_byte((int)n, p);
 227			w_string(buf, (int)n, p);
 228			temp = (PyFloatObject*)PyFloat_FromDouble(
 229				PyComplex_ImagAsDouble(v));
 230			if (!temp) {
 231				p->error = 1;
 232				return;
 233			}
 234			PyFloat_AsReprString(buf, temp);
 235			Py_DECREF(temp);
 236			n = strlen(buf);
 237			w_byte((int)n, p);
 238			w_string(buf, (int)n, p);
 239		}
 240	}
 241#endif
 242	else if (PyString_CheckExact(v)) {
 243		if (p->strings && PyString_CHECK_INTERNED(v)) {
 244			PyObject *o = PyDict_GetItem(p->strings, v);
 245			if (o) {
 246				long w = PyInt_AsLong(o);
 247				w_byte(TYPE_STRINGREF, p);
 248				w_long(w, p);
 249				goto exit;
 250			}
 251			else {
 252				int ok;
 253				o = PyInt_FromSsize_t(PyDict_Size(p->strings));
 254				ok = o &&
 255				     PyDict_SetItem(p->strings, v, o) >= 0;
 256				Py_XDECREF(o);
 257				if (!ok) {
 258					p->depth--;
 259					p->error = 1;
 260					return;
 261				}
 262				w_byte(TYPE_INTERNED, p);
 263			}
 264		}
 265		else {
 266			w_byte(TYPE_STRING, p);
 267		}
 268		n = PyString_GET_SIZE(v);
 269		if (n > INT_MAX) {
 270			/* huge strings are not supported */
 271			p->depth--;
 272			p->error = 1;
 273			return;
 274		}
 275		w_long((long)n, p);
 276		w_string(PyString_AS_STRING(v), (int)n, p);
 277	}
 278#ifdef Py_USING_UNICODE
 279	else if (PyUnicode_CheckExact(v)) {
 280	        PyObject *utf8;
 281		utf8 = PyUnicode_AsUTF8String(v);
 282		if (utf8 == NULL) {
 283			p->depth--;
 284			p->error = 1;
 285			return;
 286		}
 287		w_byte(TYPE_UNICODE, p);
 288		n = PyString_GET_SIZE(utf8);
 289		if (n > INT_MAX) {
 290			p->depth--;
 291			p->error = 1;
 292			return;
 293		}
 294		w_long((long)n, p);
 295		w_string(PyString_AS_STRING(utf8), (int)n, p);
 296		Py_DECREF(utf8);
 297	}
 298#endif
 299	else if (PyTuple_CheckExact(v)) {
 300		w_byte(TYPE_TUPLE, p);
 301		n = PyTuple_Size(v);
 302		w_long((long)n, p);
 303		for (i = 0; i < n; i++) {
 304			w_object(PyTuple_GET_ITEM(v, i), p);
 305		}
 306	}
 307	else if (PyList_CheckExact(v)) {
 308		w_byte(TYPE_LIST, p);
 309		n = PyList_GET_SIZE(v);
 310		w_long((long)n, p);
 311		for (i = 0; i < n; i++) {
 312			w_object(PyList_GET_ITEM(v, i), p);
 313		}
 314	}
 315	else if (PyDict_CheckExact(v)) {
 316		Py_ssize_t pos;
 317		PyObject *key, *value;
 318		w_byte(TYPE_DICT, p);
 319		/* This one is NULL object terminated! */
 320		pos = 0;
 321		while (PyDict_Next(v, &pos, &key, &value)) {
 322			w_object(key, p);
 323			w_object(value, p);
 324		}
 325		w_object((PyObject *)NULL, p);
 326	}
 327	else if (PyAnySet_CheckExact(v)) {
 328		PyObject *value, *it;
 329
 330		if (PyObject_TypeCheck(v, &PySet_Type))
 331			w_byte(TYPE_SET, p);
 332		else
 333			w_byte(TYPE_FROZENSET, p);
 334		n = PyObject_Size(v);
 335		if (n == -1) {
 336			p->depth--;
 337			p->error = 1;
 338			return;
 339		}
 340		w_long((long)n, p);
 341		it = PyObject_GetIter(v);
 342		if (it == NULL) {
 343			p->depth--;
 344			p->error = 1;
 345			return;
 346		}
 347		while ((value = PyIter_Next(it)) != NULL) {
 348			w_object(value, p);
 349			Py_DECREF(value);
 350		}
 351		Py_DECREF(it);
 352		if (PyErr_Occurred()) {
 353			p->depth--;
 354			p->error = 1;
 355			return;
 356		}
 357	}
 358	else if (PyCode_Check(v)) {
 359		PyCodeObject *co = (PyCodeObject *)v;
 360		w_byte(TYPE_CODE, p);
 361		w_long(co->co_argcount, p);
 362		w_long(co->co_nlocals, p);
 363		w_long(co->co_stacksize, p);
 364		w_long(co->co_flags, p);
 365		w_object(co->co_code, p);
 366		w_object(co->co_consts, p);
 367		w_object(co->co_names, p);
 368		w_object(co->co_varnames, p);
 369		w_object(co->co_freevars, p);
 370		w_object(co->co_cellvars, p);
 371		w_object(co->co_filename, p);
 372		w_object(co->co_name, p);
 373		w_long(co->co_firstlineno, p);
 374		w_object(co->co_lnotab, p);
 375	}
 376	else if (PyObject_CheckReadBuffer(v)) {
 377		/* Write unknown buffer-style objects as a string */
 378		char *s;
 379		PyBufferProcs *pb = v->ob_type->tp_as_buffer;
 380		w_byte(TYPE_STRING, p);
 381		n = (*pb->bf_getreadbuffer)(v, 0, (void **)&s);
 382		if (n > INT_MAX) {
 383			p->depth--;
 384			p->error = 1;
 385			return;
 386		}
 387		w_long((long)n, p);
 388		w_string(s, (int)n, p);
 389	}
 390	else {
 391		w_byte(TYPE_UNKNOWN, p);
 392		p->error = 1;
 393	}
 394   exit:
 395	p->depth--;
 396}
 397
 398/* version currently has no effect for writing longs. */
 399void
 400PyMarshal_WriteLongToFile(long x, FILE *fp, int version)
 401{
 402	WFILE wf;
 403	wf.fp = fp;
 404	wf.error = 0;
 405	wf.depth = 0;
 406	wf.strings = NULL;
 407	wf.version = version;
 408	w_long(x, &wf);
 409}
 410
 411void
 412PyMarshal_WriteObjectToFile(PyObject *x, FILE *fp, int version)
 413{
 414	WFILE wf;
 415	wf.fp = fp;
 416	wf.error = 0;
 417	wf.depth = 0;
 418	wf.strings = (version > 0) ? PyDict_New() : NULL;
 419	wf.version = version;
 420	w_object(x, &wf);
 421	Py_XDECREF(wf.strings);
 422}
 423
 424typedef WFILE RFILE; /* Same struct with different invariants */
 425
 426#define rs_byte(p) (((p)->ptr < (p)->end) ? (unsigned char)*(p)->ptr++ : EOF)
 427
 428#define r_byte(p) ((p)->fp ? getc((p)->fp) : rs_byte(p))
 429
 430static int
 431r_string(char *s, int n, RFILE *p)
 432{
 433	if (p->fp != NULL)
 434		/* The result fits into int because it must be <=n. */
 435		return (int)fread(s, 1, n, p->fp);
 436	if (p->end - p->ptr < n)
 437		n = (int)(p->end - p->ptr);
 438	memcpy(s, p->ptr, n);
 439	p->ptr += n;
 440	return n;
 441}
 442
 443static int
 444r_short(RFILE *p)
 445{
 446	register short x;
 447	x = r_byte(p);
 448	x |= r_byte(p) << 8;
 449	/* Sign-extension, in case short greater than 16 bits */
 450	x |= -(x & 0x8000);
 451	return x;
 452}
 453
 454static long
 455r_long(RFILE *p)
 456{
 457	register long x;
 458	register FILE *fp = p->fp;
 459	if (fp) {
 460		x = getc(fp);
 461		x |= (long)getc(fp) << 8;
 462		x |= (long)getc(fp) << 16;
 463		x |= (long)getc(fp) << 24;
 464	}
 465	else {
 466		x = rs_byte(p);
 467		x |= (long)rs_byte(p) << 8;
 468		x |= (long)rs_byte(p) << 16;
 469		x |= (long)rs_byte(p) << 24;
 470	}
 471#if SIZEOF_LONG > 4
 472	/* Sign extension for 64-bit machines */
 473	x |= -(x & 0x80000000L);
 474#endif
 475	return x;
 476}
 477
 478/* r_long64 deals with the TYPE_INT64 code.  On a machine with
 479   sizeof(long) > 4, it returns a Python int object, else a Python long
 480   object.  Note that w_long64 writes out TYPE_INT if 32 bits is enough,
 481   so there's no inefficiency here in returning a PyLong on 32-bit boxes
 482   for everything written via TYPE_INT64 (i.e., if an int is written via
 483   TYPE_INT64, it *needs* more than 32 bits).
 484*/
 485static PyObject *
 486r_long64(RFILE *p)
 487{
 488	long lo4 = r_long(p);
 489	long hi4 = r_long(p);
 490#if SIZEOF_LONG > 4
 491	long x = (hi4 << 32) | (lo4 & 0xFFFFFFFFL);
 492	return PyInt_FromLong(x);
 493#else
 494	unsigned char buf[8];
 495	int one = 1;
 496	int is_little_endian = (int)*(char*)&one;
 497	if (is_little_endian) {
 498		memcpy(buf, &lo4, 4);
 499		memcpy(buf+4, &hi4, 4);
 500	}
 501	else {
 502		memcpy(buf, &hi4, 4);
 503		memcpy(buf+4, &lo4, 4);
 504	}
 505	return _PyLong_FromByteArray(buf, 8, is_little_endian, 1);
 506#endif
 507}
 508
 509static PyObject *
 510r_object(RFILE *p)
 511{
 512	/* NULL is a valid return value, it does not necessarily means that
 513	   an exception is set. */
 514	PyObject *v, *v2;
 515	long i, n;
 516	int type = r_byte(p);
 517	PyObject *retval;
 518
 519	p->depth++;
 520
 521	if (p->depth > MAX_MARSHAL_STACK_DEPTH) {
 522		p->depth--;
 523		PyErr_SetString(PyExc_ValueError, "recursion limit exceeded");
 524		return NULL;
 525	}
 526
 527	switch (type) {
 528
 529	case EOF:
 530		PyErr_SetString(PyExc_EOFError,
 531				"EOF read where object expected");
 532		retval = NULL;
 533		break;
 534
 535	case TYPE_NULL:
 536		retval = NULL;
 537		break;
 538
 539	case TYPE_NONE:
 540		Py_INCREF(Py_None);
 541		retval = Py_None;
 542		break;
 543
 544	case TYPE_STOPITER:
 545		Py_INCREF(PyExc_StopIteration);
 546		retval = PyExc_StopIteration;
 547		break;
 548
 549	case TYPE_ELLIPSIS:
 550		Py_INCREF(Py_Ellipsis);
 551		retval = Py_Ellipsis;
 552		break;
 553
 554	case TYPE_FALSE:
 555		Py_INCREF(Py_False);
 556		retval = Py_False;
 557		break;
 558
 559	case TYPE_TRUE:
 560		Py_INCREF(Py_True);
 561		retval = Py_True;
 562		break;
 563
 564	case TYPE_INT:
 565		retval = PyInt_FromLong(r_long(p));
 566		break;
 567
 568	case TYPE_INT64:
 569		retval = r_long64(p);
 570		break;
 571
 572	case TYPE_LONG:
 573		{
 574			int size;
 575			PyLongObject *ob;
 576			n = r_long(p);
 577			if (n < -INT_MAX || n > INT_MAX) {
 578				PyErr_SetString(PyExc_ValueError,
 579						"bad marshal data");
 580				retval = NULL;
 581				break;
 582			}
 583			size = n<0 ? -n : n;
 584			ob = _PyLong_New(size);
 585			if (ob == NULL) {
 586				retval = NULL;
 587				break;
 588			}
 589			ob->ob_size = n;
 590			for (i = 0; i < size; i++) {
 591				int digit = r_short(p);
 592				if (digit < 0 ||
 593				    (digit == 0 && i == size-1)) {
 594					Py_DECREF(ob);
 595					PyErr_SetString(PyExc_ValueError,
 596							"bad marshal data");
 597					ob = NULL;
 598					break;
 599				}
 600				if (ob != NULL)
 601					ob->ob_digit[i] = digit;
 602			}
 603			retval = (PyObject *)ob;
 604			break;
 605		}
 606
 607	case TYPE_FLOAT:
 608		{
 609			char buf[256];
 610			double dx;
 611			n = r_byte(p);
 612			if (n == EOF || r_string(buf, (int)n, p) != n) {
 613				PyErr_SetString(PyExc_EOFError,
 614					"EOF read where object expected");
 615				retval = NULL;
 616				break;
 617			}
 618			buf[n] = '\0';
 619			retval = NULL;
 620			PyFPE_START_PROTECT("atof", break)
 621			dx = PyOS_ascii_atof(buf);
 622			PyFPE_END_PROTECT(dx)
 623			retval = PyFloat_FromDouble(dx);
 624			break;
 625		}
 626
 627	case TYPE_BINARY_FLOAT:
 628		{
 629			unsigned char buf[8];
 630			double x;
 631			if (r_string((char*)buf, 8, p) != 8) {
 632				PyErr_SetString(PyExc_EOFError,
 633					"EOF read where object expected");
 634				retval = NULL;
 635				break;
 636			}
 637			x = _PyFloat_Unpack8(buf, 1);
 638			if (x == -1.0 && PyErr_Occurred()) {
 639				retval = NULL;
 640				break;
 641			}
 642			retval = PyFloat_FromDouble(x);
 643			break;
 644		}
 645
 646#ifndef WITHOUT_COMPLEX
 647	case TYPE_COMPLEX:
 648		{
 649			char buf[256];
 650			Py_complex c;
 651			n = r_byte(p);
 652			if (n == EOF || r_string(buf, (int)n, p) != n) {
 653				PyErr_SetString(PyExc_EOFError,
 654					"EOF read where object expected");
 655				retval = NULL;
 656				break;
 657			}
 658			buf[n] = '\0';
 659			retval = NULL;
 660			PyFPE_START_PROTECT("atof", break;)
 661			c.real = PyOS_ascii_atof(buf);
 662			PyFPE_END_PROTECT(c)
 663			n = r_byte(p);
 664			if (n == EOF || r_string(buf, (int)n, p) != n) {
 665				PyErr_SetString(PyExc_EOFError,
 666					"EOF read where object expected");
 667				retval = NULL;
 668				break;
 669			}
 670			buf[n] = '\0';
 671			PyFPE_START_PROTECT("atof", break)
 672			c.imag = PyOS_ascii_atof(buf);
 673			PyFPE_END_PROTECT(c)
 674			retval = PyComplex_FromCComplex(c);
 675			break;
 676		}
 677
 678	case TYPE_BINARY_COMPLEX:
 679		{
 680			unsigned char buf[8];
 681			Py_complex c;
 682			if (r_string((char*)buf, 8, p) != 8) {
 683				PyErr_SetString(PyExc_EOFError,
 684					"EOF read where object expected");
 685				retval = NULL;
 686				break;
 687			}
 688			c.real = _PyFloat_Unpack8(buf, 1);
 689			if (c.real == -1.0 && PyErr_Occurred()) {
 690				retval = NULL;
 691				break;
 692			}
 693			if (r_string((char*)buf, 8, p) != 8) {
 694				PyErr_SetString(PyExc_EOFError,
 695					"EOF read where object expected");
 696				retval = NULL;
 697				break;
 698			}
 699			c.imag = _PyFloat_Unpack8(buf, 1);
 700			if (c.imag == -1.0 && PyErr_Occurred()) {
 701				retval = NULL;
 702				break;
 703			}
 704			retval = PyComplex_FromCComplex(c);
 705			break;
 706		}
 707#endif
 708
 709	case TYPE_INTERNED:
 710	case TYPE_STRING:
 711		n = r_long(p);
 712		if (n < 0 || n > INT_MAX) {
 713			PyErr_SetString(PyExc_ValueError, "bad marshal data");
 714			retval = NULL;
 715			break;
 716		}
 717		v = PyString_FromStringAndSize((char *)NULL, n);
 718		if (v == NULL) {
 719			retval = NULL;
 720			break;
 721		}
 722		if (r_string(PyString_AS_STRING(v), (int)n, p) != n) {
 723			Py_DECREF(v);
 724			PyErr_SetString(PyExc_EOFError,
 725					"EOF read where object expected");
 726			retval = NULL;
 727			break;
 728		}
 729		if (type == TYPE_INTERNED) {
 730			PyString_InternInPlace(&v);
 731			if (PyList_Append(p->strings, v) < 0) {
 732				retval = NULL;
 733				break;
 734			}
 735		}
 736		retval = v;
 737		break;
 738
 739	case TYPE_STRINGREF:
 740		n = r_long(p);
 741		if (n < 0 || n >= PyList_GET_SIZE(p->strings)) {
 742			PyErr_SetString(PyExc_ValueError, "bad marshal data");
 743			retval = NULL;
 744			break;
 745		}
 746		v = PyList_GET_ITEM(p->strings, n);
 747		Py_INCREF(v);
 748		retval = v;
 749		break;
 750
 751#ifdef Py_USING_UNICODE
 752	case TYPE_UNICODE:
 753	    {
 754		char *buffer;
 755
 756		n = r_long(p);
 757		if (n < 0 || n > INT_MAX) {
 758			PyErr_SetString(PyExc_ValueError, "bad marshal data");
 759			retval = NULL;
 760			break;
 761		}
 762		buffer = PyMem_NEW(char, n);
 763		if (buffer == NULL) {
 764			retval = PyErr_NoMemory();
 765			break;
 766		}
 767		if (r_string(buffer, (int)n, p) != n) {
 768			PyMem_DEL(buffer);
 769			PyErr_SetString(PyExc_EOFError,
 770				"EOF read where object expected");
 771			retval = NULL;
 772			break;
 773		}
 774		v = PyUnicode_DecodeUTF8(buffer, n, NULL);
 775		PyMem_DEL(buffer);
 776		retval = v;
 777		break;
 778	    }
 779#endif
 780
 781	case TYPE_TUPLE:
 782		n = r_long(p);
 783		if (n < 0 || n > INT_MAX) {
 784			PyErr_SetString(PyExc_ValueError, "bad marshal data");
 785			retval = NULL;
 786			break;
 787		}
 788		v = PyTuple_New((int)n);
 789		if (v == NULL) {
 790			retval = NULL;
 791			break;
 792		}
 793		for (i = 0; i < n; i++) {
 794			v2 = r_object(p);
 795			if ( v2 == NULL ) {
 796				if (!PyErr_Occurred())
 797					PyErr_SetString(PyExc_TypeError,
 798						"NULL object in marshal data");
 799				Py_DECREF(v);
 800				v = NULL;
 801				break;
 802			}
 803			PyTuple_SET_ITEM(v, (int)i, v2);
 804		}
 805		retval = v;
 806		break;
 807
 808	case TYPE_LIST:
 809		n = r_long(p);
 810		if (n < 0 || n > INT_MAX) {
 811			PyErr_SetString(PyExc_ValueError, "bad marshal data");
 812			retval = NULL;
 813			break;
 814		}
 815		v = PyList_New((int)n);
 816		if (v == NULL) {
 817			retval = NULL;
 818			break;
 819		}
 820		for (i = 0; i < n; i++) {
 821			v2 = r_object(p);
 822			if ( v2 == NULL ) {
 823				if (!PyErr_Occurred())
 824					PyErr_SetString(PyExc_TypeError,
 825						"NULL object in marshal data");
 826				Py_DECREF(v);
 827				v = NULL;
 828				break;
 829			}
 830			PyList_SET_ITEM(v, (int)i, v2);
 831		}
 832		retval = v;
 833		break;
 834
 835	case TYPE_DICT:
 836		v = PyDict_New();
 837		if (v == NULL) {
 838			retval = NULL;
 839			break;
 840		}
 841		for (;;) {
 842			PyObject *key, *val;
 843			key = r_object(p);
 844			if (key == NULL)
 845				break;
 846			val = r_object(p);
 847			if (val != NULL)
 848				PyDict_SetItem(v, key, val);
 849			Py_DECREF(key);
 850			Py_XDECREF(val);
 851		}
 852		if (PyErr_Occurred()) {
 853			Py_DECREF(v);
 854			v = NULL;
 855		}
 856		retval = v;
 857		break;
 858
 859	case TYPE_SET:
 860	case TYPE_FROZENSET:
 861		n = r_long(p);
 862		if (n < 0 || n > INT_MAX) {
 863			PyErr_SetString(PyExc_ValueError, "bad marshal data");
 864			retval = NULL;
 865			break;
 866		}
 867                v = (type == TYPE_SET) ? PySet_New(NULL) : PyFrozenSet_New(NULL);
 868		if (v == NULL) {
 869			retval = NULL;
 870			break;
 871		}
 872		for (i = 0; i < n; i++) {
 873			v2 = r_object(p);
 874			if ( v2 == NULL ) {
 875				if (!PyErr_Occurred())
 876					PyErr_SetString(PyExc_TypeError,
 877						"NULL object in marshal data");
 878				Py_DECREF(v);
 879				v = NULL;
 880				break;
 881			}
 882			if (PySet_Add(v, v2) == -1) {
 883                                Py_DECREF(v);
 884                                Py_DECREF(v2);
 885                                v = NULL;
 886                                break;
 887                        }
 888                        Py_DECREF(v2);
 889		}
 890		retval = v;
 891		break;
 892
 893	case TYPE_CODE:
 894		if (PyEval_GetRestricted()) {
 895			PyErr_SetString(PyExc_RuntimeError,
 896				"cannot unmarshal code objects in "
 897				"restricted execution mode");
 898			retval = NULL;
 899			break;
 900		}
 901		else {
 902			int argcount;
 903			int nlocals;
 904			int stacksize;
 905			int flags;
 906			PyObject *code = NULL;
 907			PyObject *consts = NULL;
 908			PyObject *names = NULL;
 909			PyObject *varnames = NULL;
 910			PyObject *freevars = NULL;
 911			PyObject *cellvars = NULL;
 912			PyObject *filename = NULL;
 913			PyObject *name = NULL;
 914			int firstlineno;
 915			PyObject *lnotab = NULL;
 916			
 917			v = NULL;
 918
 919                        /* XXX ignore long->int overflows for now */
 920			argcount = (int)r_long(p);
 921			nlocals = (int)r_long(p);
 922			stacksize = (int)r_long(p);
 923			flags = (int)r_long(p);
 924			code = r_object(p);
 925			if (code == NULL)
 926				goto code_error;
 927			consts = r_object(p);
 928			if (consts == NULL)
 929				goto code_error;
 930			names = r_object(p);
 931			if (names == NULL)
 932				goto code_error;
 933			varnames = r_object(p);
 934			if (varnames == NULL)
 935				goto code_error;
 936			freevars = r_object(p);
 937			if (freevars == NULL)
 938				goto code_error;
 939			cellvars = r_object(p);
 940			if (cellvars == NULL)
 941				goto code_error;
 942			filename = r_object(p);
 943			if (filename == NULL)
 944				goto code_error;
 945			name = r_object(p);
 946			if (name == NULL)
 947				goto code_error;
 948			firstlineno = (int)r_long(p);
 949			lnotab = r_object(p);
 950			if (lnotab == NULL)
 951				goto code_error;
 952
 953			v = (PyObject *) PyCode_New(
 954					argcount, nlocals, stacksize, flags,
 955					code, consts, names, varnames,
 956					freevars, cellvars, filename, name,
 957					firstlineno, lnotab);
 958
 959		  code_error:
 960			Py_XDECREF(code);
 961			Py_XDECREF(consts);
 962			Py_XDECREF(names);
 963			Py_XDECREF(varnames);
 964			Py_XDECREF(freevars);
 965			Py_XDECREF(cellvars);
 966			Py_XDECREF(filename);
 967			Py_XDECREF(name);
 968			Py_XDECREF(lnotab);
 969
 970		}
 971		retval = v;
 972		break;
 973
 974	default:
 975		/* Bogus data got written, which isn't ideal.
 976		   This will let you keep working and recover. */
 977		PyErr_SetString(PyExc_ValueError, "bad marshal data");
 978		retval = NULL;
 979		break;
 980
 981	}
 982	p->depth--;
 983	return retval;
 984}
 985
 986static PyObject *
 987read_object(RFILE *p)
 988{
 989	PyObject *v;
 990	if (PyErr_Occurred()) {
 991		fprintf(stderr, "XXX readobject called with exception set\n");
 992		return NULL;
 993	}
 994	v = r_object(p);
 995	if (v == NULL && !PyErr_Occurred())
 996		PyErr_SetString(PyExc_TypeError, "NULL object in marshal data");
 997	return v;
 998}
 999
1000int
1001PyMarshal_ReadShortFromFile(FILE *fp)
1002{
1003	RFILE rf;
1004	assert(fp);
1005	rf.fp = fp;
1006	rf.strings = NULL;
1007	rf.end = rf.ptr = NULL;
1008	return r_short(&rf);
1009}
1010
1011long
1012PyMarshal_ReadLongFromFile(FILE *fp)
1013{
1014	RFILE rf;
1015	rf.fp = fp;
1016	rf.strings = NULL;
1017	rf.ptr = rf.end = NULL;
1018	return r_long(&rf);
1019}
1020
1021#ifdef HAVE_FSTAT
1022/* Return size of file in bytes; < 0 if unknown. */
1023static off_t
1024getfilesize(FILE *fp)
1025{
1026	struct stat st;
1027	if (fstat(fileno(fp), &st) != 0)
1028		return -1;
1029	else
1030		return st.st_size;
1031}
1032#endif
1033
1034/* If we can get the size of the file up-front, and it's reasonably small,
1035 * read it in one gulp and delegate to ...FromString() instead.  Much quicker
1036 * than reading a byte at a time from file; speeds .pyc imports.
1037 * CAUTION:  since this may read the entire remainder of the file, don't
1038 * call it unless you know you're done with the file.
1039 */
1040PyObject *
1041PyMarshal_ReadLastObjectFromFile(FILE *fp)
1042{
1043/* 75% of 2.1's .pyc files can exploit SMALL_FILE_LIMIT.
1044 * REASONABLE_FILE_LIMIT is by defn something big enough for Tkinter.pyc.
1045 */
1046#define SMALL_FILE_LIMIT (1L << 14)
1047#define REASONABLE_FILE_LIMIT (1L << 18)
1048#ifdef HAVE_FSTAT
1049	off_t filesize;
1050#endif
1051#ifdef HAVE_FSTAT
1052	filesize = getfilesize(fp);
1053	if (filesize > 0) {
1054		char buf[SMALL_FILE_LIMIT];
1055		char* pBuf = NULL;
1056		if (filesize <= SMALL_FILE_LIMIT)
1057			pBuf = buf;
1058		else if (filesize <= REASONABLE_FILE_LIMIT)
1059			pBuf = (char *)PyMem_MALLOC(filesize);
1060		if (pBuf != NULL) {
1061			PyObject* v;
1062			size_t n;
1063			/* filesize must fit into an int, because it
1064			   is smaller than REASONABLE_FILE_LIMIT */
1065			n = fread(pBuf, 1, (int)filesize, fp);
1066			v = PyMarshal_ReadObjectFromString(pBuf, n);
1067			if (pBuf != buf)
1068				PyMem_FREE(pBuf);
1069			return v;
1070		}
1071
1072	}
1073#endif
1074	/* We don't have fstat, or we do but the file is larger than
1075	 * REASONABLE_FILE_LIMIT or malloc failed -- read a byte at a time.
1076	 */
1077	return PyMarshal_ReadObjectFromFile(fp);
1078
1079#undef SMALL_FILE_LIMIT
1080#undef REASONABLE_FILE_LIMIT
1081}
1082
1083PyObject *
1084PyMarshal_ReadObjectFromFile(FILE *fp)
1085{
1086	RFILE rf;
1087	PyObject *result;
1088	rf.fp = fp;
1089	rf.strings = PyList_New(0);
1090	rf.depth = 0;
1091	rf.ptr = rf.end = NULL;
1092	result = r_object(&rf);
1093	Py_DECREF(rf.strings);
1094	return result;
1095}
1096
1097PyObject *
1098PyMarshal_ReadObjectFromString(char *str, Py_ssize_t len)
1099{
1100	RFILE rf;
1101	PyObject *result;
1102	rf.fp = NULL;
1103	rf.ptr = str;
1104	rf.end = str + len;
1105	rf.strings = PyList_New(0);
1106	rf.depth = 0;
1107	result = r_object(&rf);
1108	Py_DECREF(rf.strings);
1109	return result;
1110}
1111
1112PyObject *
1113PyMarshal_WriteObjectToString(PyObject *x, int version)
1114{
1115	WFILE wf;
1116	wf.fp = NULL;
1117	wf.str = PyString_FromStringAndSize((char *)NULL, 50);
1118	if (wf.str == NULL)
1119		return NULL;
1120	wf.ptr = PyString_AS_STRING((PyStringObject *)wf.str);
1121	wf.end = wf.ptr + PyString_Size(wf.str);
1122	wf.error = 0;
1123	wf.depth = 0;
1124	wf.version = version;
1125	wf.strings = (version > 0) ? PyDict_New() : NULL;
1126	w_object(x, &wf);
1127	Py_XDECREF(wf.strings);
1128	if (wf.str != NULL) {
1129		char *base = PyString_AS_STRING((PyStringObject *)wf.str);
1130		if (wf.ptr - base > PY_SSIZE_T_MAX) {
1131			Py_DECREF(wf.str);
1132			PyErr_SetString(PyExc_OverflowError,
1133					"too much marshall data for a string");
1134			return NULL;
1135		}
1136		_PyString_Resize(&wf.str, (Py_ssize_t)(wf.ptr - base));
1137	}
1138	if (wf.error) {
1139		Py_XDECREF(wf.str);
1140		PyErr_SetString(PyExc_ValueError,
1141				(wf.error==1)?"unmarshallable object"
1142				:"object too deeply nested to marshal");
1143		return NULL;
1144	}
1145	return wf.str;
1146}
1147
1148/* And an interface for Python programs... */
1149
1150static PyObject *
1151marshal_dump(PyObject *self, PyObject *args)
1152{
1153	WFILE wf;
1154	PyObject *x;
1155	PyObject *f;
1156	int version = Py_MARSHAL_VERSION;
1157	if (!PyArg_ParseTuple(args, "OO|i:dump", &x, &f, &version))
1158		return NULL;
1159	if (!PyFile_Check(f)) {
1160		PyErr_SetString(PyExc_TypeError,
1161				"marshal.dump() 2nd arg must be file");
1162		return NULL;
1163	}
1164	wf.fp = PyFile_AsFile(f);
1165	wf.str = NULL;
1166	wf.ptr = wf.end = NULL;
1167	wf.error = 0;
1168	wf.depth = 0;
1169	wf.strings = (version > 0) ? PyDict_New() : 0;
1170	wf.version = version;
1171	w_object(x, &wf);
1172	Py_XDECREF(wf.strings);
1173	if (wf.error) {
1174		PyErr_SetString(PyExc_ValueError,
1175				(wf.error==1)?"unmarshallable object"
1176				:"object too deeply nested to marshal");
1177		return NULL;
1178	}
1179	Py_INCREF(Py_None);
1180	return Py_None;
1181}
1182
1183PyDoc_STRVAR(dump_doc,
1184"dump(value, file[, version])\n\
1185\n\
1186Write the value on the open file. The value must be a supported type.\n\
1187The file must be an open file object such as sys.stdout or returned by\n\
1188open() or os.popen(). It must be opened in binary mode ('wb' or 'w+b').\n\
1189\n\
1190If the value has (or contains an object that has) an unsupported type, a\n\
1191ValueError exception is raised — but garbage data will also be written\n\
1192to the file. The object will not be properly read back by load()\n\
1193\n\
1194New in version 2.4: The version argument indicates the data format that\n\
1195dump should use.");
1196
1197static PyObject *
1198marshal_load(PyObject *self, PyObject *f)
1199{
1200	RFILE rf;
1201	PyObject *result;
1202	if (!PyFile_Check(f)) {
1203		PyErr_SetString(PyExc_TypeError,
1204				"marshal.load() arg must be file");
1205		return NULL;
1206	}
1207	rf.fp = PyFile_AsFile(f);
1208	rf.strings = PyList_New(0);
1209	rf.depth = 0;
1210	result = read_object(&rf);
1211	Py_DECREF(rf.strings);
1212	return result;
1213}
1214
1215PyDoc_STRVAR(load_doc,
1216"load(file)\n\
1217\n\
1218Read one value from the open file and return it. If no valid value is\n\
1219read (e.g. because the data has a different Python version’s\n\
1220incompatible marshal format), raise EOFError, ValueError or TypeError.\n\
1221The file must be an open file object opened in binary mode ('rb' or\n\
1222'r+b').\n\
1223\n\
1224Note: If an object containing an unsupported type was marshalled with\n\
1225dump(), load() will substitute None for the unmarshallable type.");
1226
1227
1228static PyObject *
1229marshal_dumps(PyObject *self, PyObject *args)
1230{
1231	PyObject *x;
1232	int version = Py_MARSHAL_VERSION;
1233	if (!PyArg_ParseTuple(args, "O|i:dumps", &x, &version))
1234		return NULL;
1235	return PyMarshal_WriteObjectToString(x, version);
1236}
1237
1238PyDoc_STRVAR(dumps_doc,
1239"dumps(value[, version])\n\
1240\n\
1241Return the string that would be written to a file by dump(value, file).\n\
1242The value must be a supported type. Raise a ValueError exception if\n\
1243value has (or contains an object that has) an unsupported type.\n\
1244\n\
1245New in version 2.4: The version argument indicates the data format that\n\
1246dumps should use.");
1247
1248
1249static PyObject *
1250marshal_loads(PyObject *self, PyObject *args)
1251{
1252	RFILE rf;
1253	char *s;
1254	Py_ssize_t n;
1255	PyObject* result;
1256	if (!PyArg_ParseTuple(args, "s#:loads", &s, &n))
1257		return NULL;
1258	rf.fp = NULL;
1259	rf.ptr = s;
1260	rf.end = s + n;
1261	rf.strings = PyList_New(0);
1262	rf.depth = 0;
1263	result = read_object(&rf);
1264	Py_DECREF(rf.strings);
1265	return result;
1266}
1267
1268PyDoc_STRVAR(loads_doc,
1269"loads(string)\n\
1270\n\
1271Convert the string to a value. If no valid value is found, raise\n\
1272EOFError, ValueError or TypeError. Extra characters in the string are\n\
1273ignored.");
1274
1275static PyMethodDef marshal_methods[] = {
1276	{"dump",	marshal_dump,	METH_VARARGS,	dump_doc},
1277	{"load",	marshal_load,	METH_O,		load_doc},
1278	{"dumps",	marshal_dumps,	METH_VARARGS,	dumps_doc},
1279	{"loads",	marshal_loads,	METH_VARARGS,	loads_doc},
1280	{NULL,		NULL}		/* sentinel */
1281};
1282
1283PyDoc_STRVAR(marshal_doc,
1284"This module contains functions that can read and write Python values in\n\
1285a binary format. The format is specific to Python, but independent of\n\
1286machine architecture issues.\n\
1287\n\
1288Not all Python object types are supported; in general, only objects\n\
1289whose value is independent from a particular invocation of Python can be\n\
1290written and read by this module. The following types are supported:\n\
1291None, integers, long integers, floating point numbers, strings, Unicode\n\
1292objects, tuples, lists, sets, dictionaries, and code objects, where it\n\
1293should be understood that tuples, lists and dictionaries are only\n\
1294supported as long as the values contained therein are themselves\n\
1295supported; and recursive lists and dictionaries should not be written\n\
1296(they will cause infinite loops).\n\
1297\n\
1298Variables:\n\
1299\n\
1300version -- indicates the format that the module uses. Version 0 is the\n\
1301    historical format, version 1 (added in Python 2.4) shares interned\n\
1302    strings and version 2 (added in Python 2.5) uses a binary format for\n\
1303    floating point numbers. (New in version 2.4)\n\
1304\n\
1305Functions:\n\
1306\n\
1307dump() -- write value to a file\n\
1308load() -- read value from a file\n\
1309dumps() -- write value to a string\n\
1310loads() -- read value from a string");
1311
1312
1313PyMODINIT_FUNC
1314PyMarshal_Init(void)
1315{
1316	PyObject *mod = Py_InitModule3("marshal", marshal_methods,
1317		marshal_doc);
1318	if (mod == NULL)
1319		return;
1320	PyModule_AddIntConstant(mod, "version", Py_MARSHAL_VERSION);
1321}