/Objects/fileobject.c

Large files files are truncated, but you can click here to view the full file

   1/* File object implementation */
   2
   3#define PY_SSIZE_T_CLEAN
   4#include "Python.h"
   5#include "structmember.h"
   6
   7#ifdef HAVE_SYS_TYPES_H
   8#include <sys/types.h>
   9#endif /* HAVE_SYS_TYPES_H */
  10
  11#ifdef MS_WINDOWS
  12#define fileno _fileno
  13/* can simulate truncate with Win32 API functions; see file_truncate */
  14#define HAVE_FTRUNCATE
  15#define WIN32_LEAN_AND_MEAN
  16#include <windows.h>
  17#endif
  18
  19#ifdef _MSC_VER
  20/* Need GetVersion to see if on NT so safe to use _wfopen */
  21#define WIN32_LEAN_AND_MEAN
  22#include <windows.h>
  23#endif /* _MSC_VER */
  24
  25#if defined(PYOS_OS2) && defined(PYCC_GCC)
  26#include <io.h>
  27#endif
  28
  29#define BUF(v) PyString_AS_STRING((PyStringObject *)v)
  30
  31#ifndef DONT_HAVE_ERRNO_H
  32#include <errno.h>
  33#endif
  34
  35#ifdef HAVE_GETC_UNLOCKED
  36#define GETC(f) getc_unlocked(f)
  37#define FLOCKFILE(f) flockfile(f)
  38#define FUNLOCKFILE(f) funlockfile(f)
  39#else
  40#define GETC(f) getc(f)
  41#define FLOCKFILE(f)
  42#define FUNLOCKFILE(f)
  43#endif
  44
  45/* Bits in f_newlinetypes */
  46#define NEWLINE_UNKNOWN	0	/* No newline seen, yet */
  47#define NEWLINE_CR 1		/* \r newline seen */
  48#define NEWLINE_LF 2		/* \n newline seen */
  49#define NEWLINE_CRLF 4		/* \r\n newline seen */
  50
  51/*
  52 * These macros release the GIL while preventing the f_close() function being
  53 * called in the interval between them.  For that purpose, a running total of
  54 * the number of currently running unlocked code sections is kept in
  55 * the unlocked_count field of the PyFileObject. The close() method raises
  56 * an IOError if that field is non-zero.  See issue #815646, #595601.
  57 */
  58
  59#define FILE_BEGIN_ALLOW_THREADS(fobj) \
  60{ \
  61	fobj->unlocked_count++; \
  62	Py_BEGIN_ALLOW_THREADS
  63
  64#define FILE_END_ALLOW_THREADS(fobj) \
  65	Py_END_ALLOW_THREADS \
  66	fobj->unlocked_count--; \
  67	assert(fobj->unlocked_count >= 0); \
  68}
  69
  70#define FILE_ABORT_ALLOW_THREADS(fobj) \
  71	Py_BLOCK_THREADS \
  72	fobj->unlocked_count--; \
  73	assert(fobj->unlocked_count >= 0);
  74
  75#ifdef __cplusplus
  76extern "C" {
  77#endif
  78
  79FILE *
  80PyFile_AsFile(PyObject *f)
  81{
  82	if (f == NULL || !PyFile_Check(f))
  83		return NULL;
  84	else
  85		return ((PyFileObject *)f)->f_fp;
  86}
  87
  88void PyFile_IncUseCount(PyFileObject *fobj)
  89{
  90	fobj->unlocked_count++;
  91}
  92
  93void PyFile_DecUseCount(PyFileObject *fobj)
  94{
  95	fobj->unlocked_count--;
  96	assert(fobj->unlocked_count >= 0);
  97}
  98
  99PyObject *
 100PyFile_Name(PyObject *f)
 101{
 102	if (f == NULL || !PyFile_Check(f))
 103		return NULL;
 104	else
 105		return ((PyFileObject *)f)->f_name;
 106}
 107
 108/* This is a safe wrapper around PyObject_Print to print to the FILE
 109   of a PyFileObject. PyObject_Print releases the GIL but knows nothing
 110   about PyFileObject. */
 111static int
 112file_PyObject_Print(PyObject *op, PyFileObject *f, int flags)
 113{
 114	int result;
 115	PyFile_IncUseCount(f);
 116	result = PyObject_Print(op, f->f_fp, flags);
 117	PyFile_DecUseCount(f);
 118	return result;
 119}
 120
 121/* On Unix, fopen will succeed for directories.
 122   In Python, there should be no file objects referring to
 123   directories, so we need a check.  */
 124
 125static PyFileObject*
 126dircheck(PyFileObject* f)
 127{
 128#if defined(HAVE_FSTAT) && defined(S_IFDIR) && defined(EISDIR)
 129	struct stat buf;
 130	if (f->f_fp == NULL)
 131		return f;
 132	if (fstat(fileno(f->f_fp), &buf) == 0 &&
 133	    S_ISDIR(buf.st_mode)) {
 134		char *msg = strerror(EISDIR);
 135		PyObject *exc = PyObject_CallFunction(PyExc_IOError, "(isO)",
 136						      EISDIR, msg, f->f_name);
 137		PyErr_SetObject(PyExc_IOError, exc);
 138		Py_XDECREF(exc);
 139		return NULL;
 140	}
 141#endif
 142	return f;
 143}
 144
 145
 146static PyObject *
 147fill_file_fields(PyFileObject *f, FILE *fp, PyObject *name, char *mode,
 148		 int (*close)(FILE *))
 149{
 150	assert(name != NULL);
 151	assert(f != NULL);
 152	assert(PyFile_Check(f));
 153	assert(f->f_fp == NULL);
 154
 155	Py_DECREF(f->f_name);
 156	Py_DECREF(f->f_mode);
 157	Py_DECREF(f->f_encoding);
 158	Py_DECREF(f->f_errors);
 159
 160        Py_INCREF(name);
 161        f->f_name = name;
 162
 163	f->f_mode = PyString_FromString(mode);
 164
 165	f->f_close = close;
 166	f->f_softspace = 0;
 167	f->f_binary = strchr(mode,'b') != NULL;
 168	f->f_buf = NULL;
 169	f->f_univ_newline = (strchr(mode, 'U') != NULL);
 170	f->f_newlinetypes = NEWLINE_UNKNOWN;
 171	f->f_skipnextlf = 0;
 172	Py_INCREF(Py_None);
 173	f->f_encoding = Py_None;
 174	Py_INCREF(Py_None);
 175	f->f_errors = Py_None;
 176
 177	if (f->f_mode == NULL)
 178		return NULL;
 179	f->f_fp = fp;
 180        f = dircheck(f);
 181	return (PyObject *) f;
 182}
 183
 184/* check for known incorrect mode strings - problem is, platforms are
 185   free to accept any mode characters they like and are supposed to
 186   ignore stuff they don't understand... write or append mode with
 187   universal newline support is expressly forbidden by PEP 278.
 188   Additionally, remove the 'U' from the mode string as platforms
 189   won't know what it is. Non-zero return signals an exception */
 190int
 191_PyFile_SanitizeMode(char *mode)
 192{
 193	char *upos;
 194	size_t len = strlen(mode);
 195
 196	if (!len) {
 197		PyErr_SetString(PyExc_ValueError, "empty mode string");
 198		return -1;
 199	}
 200
 201	upos = strchr(mode, 'U');
 202	if (upos) {
 203		memmove(upos, upos+1, len-(upos-mode)); /* incl null char */
 204
 205		if (mode[0] == 'w' || mode[0] == 'a') {
 206			PyErr_Format(PyExc_ValueError, "universal newline "
 207			             "mode can only be used with modes "
 208				     "starting with 'r'");
 209			return -1;
 210		}
 211
 212		if (mode[0] != 'r') {
 213			memmove(mode+1, mode, strlen(mode)+1);
 214			mode[0] = 'r';
 215		}
 216
 217		if (!strchr(mode, 'b')) {
 218			memmove(mode+2, mode+1, strlen(mode));
 219			mode[1] = 'b';
 220		}
 221	} else if (mode[0] != 'r' && mode[0] != 'w' && mode[0] != 'a') {
 222		PyErr_Format(PyExc_ValueError, "mode string must begin with "
 223	        	    "one of 'r', 'w', 'a' or 'U', not '%.200s'", mode);
 224		return -1;
 225	}
 226
 227	return 0;
 228}
 229
 230static PyObject *
 231open_the_file(PyFileObject *f, char *name, char *mode)
 232{
 233	char *newmode;
 234	assert(f != NULL);
 235	assert(PyFile_Check(f));
 236#ifdef MS_WINDOWS
 237	/* windows ignores the passed name in order to support Unicode */
 238	assert(f->f_name != NULL);
 239#else
 240	assert(name != NULL);
 241#endif
 242	assert(mode != NULL);
 243	assert(f->f_fp == NULL);
 244
 245	/* probably need to replace 'U' by 'rb' */
 246	newmode = PyMem_MALLOC(strlen(mode) + 3);
 247	if (!newmode) {
 248		PyErr_NoMemory();
 249		return NULL;
 250	}
 251	strcpy(newmode, mode);
 252
 253	if (_PyFile_SanitizeMode(newmode)) {
 254		f = NULL;
 255		goto cleanup;
 256	}
 257
 258	/* rexec.py can't stop a user from getting the file() constructor --
 259	   all they have to do is get *any* file object f, and then do
 260	   type(f).  Here we prevent them from doing damage with it. */
 261	if (PyEval_GetRestricted()) {
 262		PyErr_SetString(PyExc_IOError,
 263		"file() constructor not accessible in restricted mode");
 264		f = NULL;
 265		goto cleanup;
 266	}
 267	errno = 0;
 268
 269#ifdef MS_WINDOWS
 270	if (PyUnicode_Check(f->f_name)) {
 271		PyObject *wmode;
 272		wmode = PyUnicode_DecodeASCII(newmode, strlen(newmode), NULL);
 273		if (f->f_name && wmode) {
 274			FILE_BEGIN_ALLOW_THREADS(f)
 275			/* PyUnicode_AS_UNICODE OK without thread
 276			   lock as it is a simple dereference. */
 277			f->f_fp = _wfopen(PyUnicode_AS_UNICODE(f->f_name),
 278					  PyUnicode_AS_UNICODE(wmode));
 279			FILE_END_ALLOW_THREADS(f)
 280		}
 281		Py_XDECREF(wmode);
 282	}
 283#endif
 284	if (NULL == f->f_fp && NULL != name) {
 285		FILE_BEGIN_ALLOW_THREADS(f)
 286		f->f_fp = fopen(name, newmode);
 287		FILE_END_ALLOW_THREADS(f)
 288	}
 289
 290	if (f->f_fp == NULL) {
 291#if defined  _MSC_VER && (_MSC_VER < 1400 || !defined(__STDC_SECURE_LIB__))
 292		/* MSVC 6 (Microsoft) leaves errno at 0 for bad mode strings,
 293		 * across all Windows flavors.  When it sets EINVAL varies
 294		 * across Windows flavors, the exact conditions aren't
 295		 * documented, and the answer lies in the OS's implementation
 296		 * of Win32's CreateFile function (whose source is secret).
 297		 * Seems the best we can do is map EINVAL to ENOENT.
 298		 * Starting with Visual Studio .NET 2005, EINVAL is correctly
 299		 * set by our CRT error handler (set in exceptions.c.)
 300		 */
 301		if (errno == 0)	/* bad mode string */
 302			errno = EINVAL;
 303		else if (errno == EINVAL) /* unknown, but not a mode string */
 304			errno = ENOENT;
 305#endif
 306                /* EINVAL is returned when an invalid filename or
 307                 * an invalid mode is supplied. */
 308		if (errno == EINVAL) {
 309			PyObject *v;
 310			char message[100];
 311			PyOS_snprintf(message, 100, 
 312			    "invalid mode ('%.50s') or filename", mode);
 313			v = Py_BuildValue("(isO)", errno, message, f->f_name);
 314			if (v != NULL) {
 315				PyErr_SetObject(PyExc_IOError, v);
 316				Py_DECREF(v);
 317			}
 318		}
 319		else
 320			PyErr_SetFromErrnoWithFilenameObject(PyExc_IOError, f->f_name);
 321		f = NULL;
 322	}
 323	if (f != NULL)
 324		f = dircheck(f);
 325
 326cleanup:
 327	PyMem_FREE(newmode);
 328
 329	return (PyObject *)f;
 330}
 331
 332static PyObject *
 333close_the_file(PyFileObject *f)
 334{
 335	int sts = 0;
 336	int (*local_close)(FILE *);
 337	FILE *local_fp = f->f_fp;
 338	if (local_fp != NULL) {
 339		local_close = f->f_close;
 340		if (local_close != NULL && f->unlocked_count > 0) {
 341			if (f->ob_refcnt > 0) {
 342				PyErr_SetString(PyExc_IOError,
 343					"close() called during concurrent "
 344					"operation on the same file object.");
 345			} else {
 346				/* This should not happen unless someone is
 347				 * carelessly playing with the PyFileObject
 348				 * struct fields and/or its associated FILE
 349				 * pointer. */
 350				PyErr_SetString(PyExc_SystemError,
 351					"PyFileObject locking error in "
 352					"destructor (refcnt <= 0 at close).");
 353			}
 354			return NULL;
 355		}
 356		/* NULL out the FILE pointer before releasing the GIL, because
 357		 * it will not be valid anymore after the close() function is
 358		 * called. */
 359		f->f_fp = NULL;
 360		if (local_close != NULL) {
 361			Py_BEGIN_ALLOW_THREADS
 362			errno = 0;
 363			sts = (*local_close)(local_fp);
 364			Py_END_ALLOW_THREADS
 365			if (sts == EOF)
 366				return PyErr_SetFromErrno(PyExc_IOError);
 367			if (sts != 0)
 368				return PyInt_FromLong((long)sts);
 369		}
 370	}
 371	Py_RETURN_NONE;
 372}
 373
 374PyObject *
 375PyFile_FromFile(FILE *fp, char *name, char *mode, int (*close)(FILE *))
 376{
 377	PyFileObject *f = (PyFileObject *)PyFile_Type.tp_new(&PyFile_Type,
 378							     NULL, NULL);
 379	if (f != NULL) {
 380		PyObject *o_name = PyString_FromString(name);
 381		if (o_name == NULL)
 382			return NULL;
 383		if (fill_file_fields(f, fp, o_name, mode, close) == NULL) {
 384			Py_DECREF(f);
 385			f = NULL;
 386		}
 387                Py_DECREF(o_name);
 388	}
 389	return (PyObject *) f;
 390}
 391
 392PyObject *
 393PyFile_FromString(char *name, char *mode)
 394{
 395	extern int fclose(FILE *);
 396	PyFileObject *f;
 397
 398	f = (PyFileObject *)PyFile_FromFile((FILE *)NULL, name, mode, fclose);
 399	if (f != NULL) {
 400		if (open_the_file(f, name, mode) == NULL) {
 401			Py_DECREF(f);
 402			f = NULL;
 403		}
 404	}
 405	return (PyObject *)f;
 406}
 407
 408void
 409PyFile_SetBufSize(PyObject *f, int bufsize)
 410{
 411	PyFileObject *file = (PyFileObject *)f;
 412	if (bufsize >= 0) {
 413		int type;
 414		switch (bufsize) {
 415		case 0:
 416			type = _IONBF;
 417			break;
 418#ifdef HAVE_SETVBUF
 419		case 1:
 420			type = _IOLBF;
 421			bufsize = BUFSIZ;
 422			break;
 423#endif
 424		default:
 425			type = _IOFBF;
 426#ifndef HAVE_SETVBUF
 427			bufsize = BUFSIZ;
 428#endif
 429			break;
 430		}
 431		fflush(file->f_fp);
 432		if (type == _IONBF) {
 433			PyMem_Free(file->f_setbuf);
 434			file->f_setbuf = NULL;
 435		} else {
 436			file->f_setbuf = (char *)PyMem_Realloc(file->f_setbuf, 
 437                                                                bufsize);
 438		}
 439#ifdef HAVE_SETVBUF
 440		setvbuf(file->f_fp, file->f_setbuf, type, bufsize);
 441#else /* !HAVE_SETVBUF */
 442		setbuf(file->f_fp, file->f_setbuf);
 443#endif /* !HAVE_SETVBUF */
 444	}
 445}
 446
 447/* Set the encoding used to output Unicode strings.
 448   Return 1 on success, 0 on failure. */
 449
 450int
 451PyFile_SetEncoding(PyObject *f, const char *enc)
 452{
 453	return PyFile_SetEncodingAndErrors(f, enc, NULL);
 454}
 455
 456int
 457PyFile_SetEncodingAndErrors(PyObject *f, const char *enc, char* errors)
 458{
 459	PyFileObject *file = (PyFileObject*)f;
 460	PyObject *str, *oerrors;
 461
 462	assert(PyFile_Check(f));
 463	str = PyString_FromString(enc);
 464	if (!str)
 465		return 0;
 466	if (errors) {
 467		oerrors = PyString_FromString(errors);
 468		if (!oerrors) {
 469			Py_DECREF(str);
 470			return 0;
 471		}
 472	} else {
 473		oerrors = Py_None;
 474		Py_INCREF(Py_None);
 475	}
 476	Py_DECREF(file->f_encoding);
 477	file->f_encoding = str;
 478	Py_DECREF(file->f_errors);
 479	file->f_errors = oerrors;
 480	return 1;
 481}
 482
 483static PyObject *
 484err_closed(void)
 485{
 486	PyErr_SetString(PyExc_ValueError, "I/O operation on closed file");
 487	return NULL;
 488}
 489
 490/* Refuse regular file I/O if there's data in the iteration-buffer.
 491 * Mixing them would cause data to arrive out of order, as the read*
 492 * methods don't use the iteration buffer. */
 493static PyObject *
 494err_iterbuffered(void)
 495{
 496	PyErr_SetString(PyExc_ValueError,
 497		"Mixing iteration and read methods would lose data");
 498	return NULL;
 499}
 500
 501static void drop_readahead(PyFileObject *);
 502
 503/* Methods */
 504
 505static void
 506file_dealloc(PyFileObject *f)
 507{
 508	PyObject *ret;
 509	if (f->weakreflist != NULL)
 510		PyObject_ClearWeakRefs((PyObject *) f);
 511	ret = close_the_file(f);
 512	if (!ret) {
 513		PySys_WriteStderr("close failed in file object destructor:\n");
 514		PyErr_Print();
 515	}
 516	else {
 517		Py_DECREF(ret);
 518	}
 519	PyMem_Free(f->f_setbuf);
 520	Py_XDECREF(f->f_name);
 521	Py_XDECREF(f->f_mode);
 522	Py_XDECREF(f->f_encoding);
 523	Py_XDECREF(f->f_errors);
 524	drop_readahead(f);
 525	Py_TYPE(f)->tp_free((PyObject *)f);
 526}
 527
 528static PyObject *
 529file_repr(PyFileObject *f)
 530{
 531	if (PyUnicode_Check(f->f_name)) {
 532#ifdef Py_USING_UNICODE
 533		PyObject *ret = NULL;
 534		PyObject *name = PyUnicode_AsUnicodeEscapeString(f->f_name);
 535		const char *name_str = name ? PyString_AsString(name) : "?";
 536		ret = PyString_FromFormat("<%s file u'%s', mode '%s' at %p>",
 537				   f->f_fp == NULL ? "closed" : "open",
 538				   name_str,
 539				   PyString_AsString(f->f_mode),
 540				   f);
 541		Py_XDECREF(name);
 542		return ret;
 543#endif
 544	} else {
 545		return PyString_FromFormat("<%s file '%s', mode '%s' at %p>",
 546				   f->f_fp == NULL ? "closed" : "open",
 547				   PyString_AsString(f->f_name),
 548				   PyString_AsString(f->f_mode),
 549				   f);
 550	}
 551}
 552
 553static PyObject *
 554file_close(PyFileObject *f)
 555{
 556	PyObject *sts = close_the_file(f);
 557	PyMem_Free(f->f_setbuf);
 558	f->f_setbuf = NULL;
 559	return sts;
 560}
 561
 562
 563/* Our very own off_t-like type, 64-bit if possible */
 564#if !defined(HAVE_LARGEFILE_SUPPORT)
 565typedef off_t Py_off_t;
 566#elif SIZEOF_OFF_T >= 8
 567typedef off_t Py_off_t;
 568#elif SIZEOF_FPOS_T >= 8
 569typedef fpos_t Py_off_t;
 570#else
 571#error "Large file support, but neither off_t nor fpos_t is large enough."
 572#endif
 573
 574
 575/* a portable fseek() function
 576   return 0 on success, non-zero on failure (with errno set) */
 577static int
 578_portable_fseek(FILE *fp, Py_off_t offset, int whence)
 579{
 580#if !defined(HAVE_LARGEFILE_SUPPORT)
 581	return fseek(fp, offset, whence);
 582#elif defined(HAVE_FSEEKO) && SIZEOF_OFF_T >= 8
 583	return fseeko(fp, offset, whence);
 584#elif defined(HAVE_FSEEK64)
 585	return fseek64(fp, offset, whence);
 586#elif defined(__BEOS__)
 587	return _fseek(fp, offset, whence);
 588#elif SIZEOF_FPOS_T >= 8
 589	/* lacking a 64-bit capable fseek(), use a 64-bit capable fsetpos()
 590	   and fgetpos() to implement fseek()*/
 591	fpos_t pos;
 592	switch (whence) {
 593	case SEEK_END:
 594#ifdef MS_WINDOWS
 595		fflush(fp);
 596		if (_lseeki64(fileno(fp), 0, 2) == -1)
 597			return -1;
 598#else
 599		if (fseek(fp, 0, SEEK_END) != 0)
 600			return -1;
 601#endif
 602		/* fall through */
 603	case SEEK_CUR:
 604		if (fgetpos(fp, &pos) != 0)
 605			return -1;
 606		offset += pos;
 607		break;
 608	/* case SEEK_SET: break; */
 609	}
 610	return fsetpos(fp, &offset);
 611#else
 612#error "Large file support, but no way to fseek."
 613#endif
 614}
 615
 616
 617/* a portable ftell() function
 618   Return -1 on failure with errno set appropriately, current file
 619   position on success */
 620static Py_off_t
 621_portable_ftell(FILE* fp)
 622{
 623#if !defined(HAVE_LARGEFILE_SUPPORT)
 624	return ftell(fp);
 625#elif defined(HAVE_FTELLO) && SIZEOF_OFF_T >= 8
 626	return ftello(fp);
 627#elif defined(HAVE_FTELL64)
 628	return ftell64(fp);
 629#elif SIZEOF_FPOS_T >= 8
 630	fpos_t pos;
 631	if (fgetpos(fp, &pos) != 0)
 632		return -1;
 633	return pos;
 634#else
 635#error "Large file support, but no way to ftell."
 636#endif
 637}
 638
 639
 640static PyObject *
 641file_seek(PyFileObject *f, PyObject *args)
 642{
 643	int whence;
 644	int ret;
 645	Py_off_t offset;
 646	PyObject *offobj, *off_index;
 647
 648	if (f->f_fp == NULL)
 649		return err_closed();
 650	drop_readahead(f);
 651	whence = 0;
 652	if (!PyArg_ParseTuple(args, "O|i:seek", &offobj, &whence))
 653		return NULL;
 654	off_index = PyNumber_Index(offobj);
 655	if (!off_index) {
 656		if (!PyFloat_Check(offobj))
 657			return NULL;
 658		/* Deprecated in 2.6 */
 659		PyErr_Clear();
 660		if (PyErr_WarnEx(PyExc_DeprecationWarning,
 661				 "integer argument expected, got float",
 662				 1) < 0)
 663			return NULL;
 664		off_index = offobj;
 665		Py_INCREF(offobj);
 666	}
 667#if !defined(HAVE_LARGEFILE_SUPPORT)
 668	offset = PyInt_AsLong(off_index);
 669#else
 670	offset = PyLong_Check(off_index) ?
 671		PyLong_AsLongLong(off_index) : PyInt_AsLong(off_index);
 672#endif
 673	Py_DECREF(off_index);
 674	if (PyErr_Occurred())
 675		return NULL;
 676
 677 	FILE_BEGIN_ALLOW_THREADS(f)
 678	errno = 0;
 679	ret = _portable_fseek(f->f_fp, offset, whence);
 680	FILE_END_ALLOW_THREADS(f)
 681
 682	if (ret != 0) {
 683		PyErr_SetFromErrno(PyExc_IOError);
 684		clearerr(f->f_fp);
 685		return NULL;
 686	}
 687	f->f_skipnextlf = 0;
 688	Py_INCREF(Py_None);
 689	return Py_None;
 690}
 691
 692
 693#ifdef HAVE_FTRUNCATE
 694static PyObject *
 695file_truncate(PyFileObject *f, PyObject *args)
 696{
 697	Py_off_t newsize;
 698	PyObject *newsizeobj = NULL;
 699	Py_off_t initialpos;
 700	int ret;
 701
 702	if (f->f_fp == NULL)
 703		return err_closed();
 704	if (!PyArg_UnpackTuple(args, "truncate", 0, 1, &newsizeobj))
 705		return NULL;
 706
 707	/* Get current file position.  If the file happens to be open for
 708	 * update and the last operation was an input operation, C doesn't
 709	 * define what the later fflush() will do, but we promise truncate()
 710	 * won't change the current position (and fflush() *does* change it
 711	 * then at least on Windows).  The easiest thing is to capture
 712	 * current pos now and seek back to it at the end.
 713	 */
 714	FILE_BEGIN_ALLOW_THREADS(f)
 715	errno = 0;
 716	initialpos = _portable_ftell(f->f_fp);
 717	FILE_END_ALLOW_THREADS(f)
 718	if (initialpos == -1)
 719		goto onioerror;
 720
 721	/* Set newsize to current postion if newsizeobj NULL, else to the
 722	 * specified value.
 723	 */
 724	if (newsizeobj != NULL) {
 725#if !defined(HAVE_LARGEFILE_SUPPORT)
 726		newsize = PyInt_AsLong(newsizeobj);
 727#else
 728		newsize = PyLong_Check(newsizeobj) ?
 729				PyLong_AsLongLong(newsizeobj) :
 730				PyInt_AsLong(newsizeobj);
 731#endif
 732		if (PyErr_Occurred())
 733			return NULL;
 734	}
 735	else /* default to current position */
 736		newsize = initialpos;
 737
 738	/* Flush the stream.  We're mixing stream-level I/O with lower-level
 739	 * I/O, and a flush may be necessary to synch both platform views
 740	 * of the current file state.
 741	 */
 742	FILE_BEGIN_ALLOW_THREADS(f)
 743	errno = 0;
 744	ret = fflush(f->f_fp);
 745	FILE_END_ALLOW_THREADS(f)
 746	if (ret != 0)
 747		goto onioerror;
 748
 749#ifdef MS_WINDOWS
 750	/* MS _chsize doesn't work if newsize doesn't fit in 32 bits,
 751	   so don't even try using it. */
 752	{
 753		HANDLE hFile;
 754
 755		/* Have to move current pos to desired endpoint on Windows. */
 756		FILE_BEGIN_ALLOW_THREADS(f)
 757		errno = 0;
 758		ret = _portable_fseek(f->f_fp, newsize, SEEK_SET) != 0;
 759		FILE_END_ALLOW_THREADS(f)
 760		if (ret)
 761			goto onioerror;
 762
 763		/* Truncate.  Note that this may grow the file! */
 764		FILE_BEGIN_ALLOW_THREADS(f)
 765		errno = 0;
 766		hFile = (HANDLE)_get_osfhandle(fileno(f->f_fp));
 767		ret = hFile == (HANDLE)-1;
 768		if (ret == 0) {
 769			ret = SetEndOfFile(hFile) == 0;
 770			if (ret)
 771				errno = EACCES;
 772		}
 773		FILE_END_ALLOW_THREADS(f)
 774		if (ret)
 775			goto onioerror;
 776	}
 777#else
 778	FILE_BEGIN_ALLOW_THREADS(f)
 779	errno = 0;
 780	ret = ftruncate(fileno(f->f_fp), newsize);
 781	FILE_END_ALLOW_THREADS(f)
 782	if (ret != 0)
 783		goto onioerror;
 784#endif /* !MS_WINDOWS */
 785
 786	/* Restore original file position. */
 787	FILE_BEGIN_ALLOW_THREADS(f)
 788	errno = 0;
 789	ret = _portable_fseek(f->f_fp, initialpos, SEEK_SET) != 0;
 790	FILE_END_ALLOW_THREADS(f)
 791	if (ret)
 792		goto onioerror;
 793
 794	Py_INCREF(Py_None);
 795	return Py_None;
 796
 797onioerror:
 798	PyErr_SetFromErrno(PyExc_IOError);
 799	clearerr(f->f_fp);
 800	return NULL;
 801}
 802#endif /* HAVE_FTRUNCATE */
 803
 804static PyObject *
 805file_tell(PyFileObject *f)
 806{
 807	Py_off_t pos;
 808
 809	if (f->f_fp == NULL)
 810		return err_closed();
 811	FILE_BEGIN_ALLOW_THREADS(f)
 812	errno = 0;
 813	pos = _portable_ftell(f->f_fp);
 814	FILE_END_ALLOW_THREADS(f)
 815
 816	if (pos == -1) {
 817		PyErr_SetFromErrno(PyExc_IOError);
 818		clearerr(f->f_fp);
 819		return NULL;
 820	}
 821	if (f->f_skipnextlf) {
 822		int c;
 823		c = GETC(f->f_fp);
 824		if (c == '\n') {
 825			f->f_newlinetypes |= NEWLINE_CRLF;
 826			pos++;
 827			f->f_skipnextlf = 0;
 828		} else if (c != EOF) ungetc(c, f->f_fp);
 829	}
 830#if !defined(HAVE_LARGEFILE_SUPPORT)
 831	return PyInt_FromLong(pos);
 832#else
 833	return PyLong_FromLongLong(pos);
 834#endif
 835}
 836
 837static PyObject *
 838file_fileno(PyFileObject *f)
 839{
 840	if (f->f_fp == NULL)
 841		return err_closed();
 842	return PyInt_FromLong((long) fileno(f->f_fp));
 843}
 844
 845static PyObject *
 846file_flush(PyFileObject *f)
 847{
 848	int res;
 849
 850	if (f->f_fp == NULL)
 851		return err_closed();
 852	FILE_BEGIN_ALLOW_THREADS(f)
 853	errno = 0;
 854	res = fflush(f->f_fp);
 855	FILE_END_ALLOW_THREADS(f)
 856	if (res != 0) {
 857		PyErr_SetFromErrno(PyExc_IOError);
 858		clearerr(f->f_fp);
 859		return NULL;
 860	}
 861	Py_INCREF(Py_None);
 862	return Py_None;
 863}
 864
 865static PyObject *
 866file_isatty(PyFileObject *f)
 867{
 868	long res;
 869	if (f->f_fp == NULL)
 870		return err_closed();
 871	FILE_BEGIN_ALLOW_THREADS(f)
 872	res = isatty((int)fileno(f->f_fp));
 873	FILE_END_ALLOW_THREADS(f)
 874	return PyBool_FromLong(res);
 875}
 876
 877
 878#if BUFSIZ < 8192
 879#define SMALLCHUNK 8192
 880#else
 881#define SMALLCHUNK BUFSIZ
 882#endif
 883
 884#if SIZEOF_INT < 4
 885#define BIGCHUNK  (512 * 32)
 886#else
 887#define BIGCHUNK  (512 * 1024)
 888#endif
 889
 890static size_t
 891new_buffersize(PyFileObject *f, size_t currentsize)
 892{
 893#ifdef HAVE_FSTAT
 894	off_t pos, end;
 895	struct stat st;
 896	if (fstat(fileno(f->f_fp), &st) == 0) {
 897		end = st.st_size;
 898		/* The following is not a bug: we really need to call lseek()
 899		   *and* ftell().  The reason is that some stdio libraries
 900		   mistakenly flush their buffer when ftell() is called and
 901		   the lseek() call it makes fails, thereby throwing away
 902		   data that cannot be recovered in any way.  To avoid this,
 903		   we first test lseek(), and only call ftell() if lseek()
 904		   works.  We can't use the lseek() value either, because we
 905		   need to take the amount of buffered data into account.
 906		   (Yet another reason why stdio stinks. :-) */
 907		pos = lseek(fileno(f->f_fp), 0L, SEEK_CUR);
 908		if (pos >= 0) {
 909			pos = ftell(f->f_fp);
 910		}
 911		if (pos < 0)
 912			clearerr(f->f_fp);
 913		if (end > pos && pos >= 0)
 914			return currentsize + end - pos + 1;
 915		/* Add 1 so if the file were to grow we'd notice. */
 916	}
 917#endif
 918	if (currentsize > SMALLCHUNK) {
 919		/* Keep doubling until we reach BIGCHUNK;
 920		   then keep adding BIGCHUNK. */
 921		if (currentsize <= BIGCHUNK)
 922			return currentsize + currentsize;
 923		else
 924			return currentsize + BIGCHUNK;
 925	}
 926	return currentsize + SMALLCHUNK;
 927}
 928
 929#if defined(EWOULDBLOCK) && defined(EAGAIN) && EWOULDBLOCK != EAGAIN
 930#define BLOCKED_ERRNO(x) ((x) == EWOULDBLOCK || (x) == EAGAIN)
 931#else
 932#ifdef EWOULDBLOCK
 933#define BLOCKED_ERRNO(x) ((x) == EWOULDBLOCK)
 934#else
 935#ifdef EAGAIN
 936#define BLOCKED_ERRNO(x) ((x) == EAGAIN)
 937#else
 938#define BLOCKED_ERRNO(x) 0
 939#endif
 940#endif
 941#endif
 942
 943static PyObject *
 944file_read(PyFileObject *f, PyObject *args)
 945{
 946	long bytesrequested = -1;
 947	size_t bytesread, buffersize, chunksize;
 948	PyObject *v;
 949
 950	if (f->f_fp == NULL)
 951		return err_closed();
 952	/* refuse to mix with f.next() */
 953	if (f->f_buf != NULL &&
 954	    (f->f_bufend - f->f_bufptr) > 0 &&
 955	    f->f_buf[0] != '\0')
 956		return err_iterbuffered();
 957	if (!PyArg_ParseTuple(args, "|l:read", &bytesrequested))
 958		return NULL;
 959	if (bytesrequested < 0)
 960		buffersize = new_buffersize(f, (size_t)0);
 961	else
 962		buffersize = bytesrequested;
 963	if (buffersize > PY_SSIZE_T_MAX) {
 964		PyErr_SetString(PyExc_OverflowError,
 965	"requested number of bytes is more than a Python string can hold");
 966		return NULL;
 967	}
 968	v = PyString_FromStringAndSize((char *)NULL, buffersize);
 969	if (v == NULL)
 970		return NULL;
 971	bytesread = 0;
 972	for (;;) {
 973		FILE_BEGIN_ALLOW_THREADS(f)
 974		errno = 0;
 975		chunksize = Py_UniversalNewlineFread(BUF(v) + bytesread,
 976			  buffersize - bytesread, f->f_fp, (PyObject *)f);
 977		FILE_END_ALLOW_THREADS(f)
 978		if (chunksize == 0) {
 979			if (!ferror(f->f_fp))
 980				break;
 981			clearerr(f->f_fp);
 982			/* When in non-blocking mode, data shouldn't
 983			 * be discarded if a blocking signal was
 984			 * received. That will also happen if
 985			 * chunksize != 0, but bytesread < buffersize. */
 986			if (bytesread > 0 && BLOCKED_ERRNO(errno))
 987				break;
 988			PyErr_SetFromErrno(PyExc_IOError);
 989			Py_DECREF(v);
 990			return NULL;
 991		}
 992		bytesread += chunksize;
 993		if (bytesread < buffersize) {
 994			clearerr(f->f_fp);
 995			break;
 996		}
 997		if (bytesrequested < 0) {
 998			buffersize = new_buffersize(f, buffersize);
 999			if (_PyString_Resize(&v, buffersize) < 0)
1000				return NULL;
1001		} else {
1002			/* Got what was requested. */
1003			break;
1004		}
1005	}
1006	if (bytesread != buffersize)
1007		_PyString_Resize(&v, bytesread);
1008	return v;
1009}
1010
1011static PyObject *
1012file_readinto(PyFileObject *f, PyObject *args)
1013{
1014	char *ptr;
1015	Py_ssize_t ntodo;
1016	Py_ssize_t ndone, nnow;
1017	Py_buffer pbuf;
1018
1019	if (f->f_fp == NULL)
1020		return err_closed();
1021	/* refuse to mix with f.next() */
1022	if (f->f_buf != NULL &&
1023	    (f->f_bufend - f->f_bufptr) > 0 &&
1024	    f->f_buf[0] != '\0')
1025		return err_iterbuffered();
1026	if (!PyArg_ParseTuple(args, "w*", &pbuf))
1027		return NULL;
1028	ptr = pbuf.buf;
1029	ntodo = pbuf.len;
1030	ndone = 0;
1031	while (ntodo > 0) {
1032		FILE_BEGIN_ALLOW_THREADS(f)
1033		errno = 0;
1034		nnow = Py_UniversalNewlineFread(ptr+ndone, ntodo, f->f_fp,
1035						(PyObject *)f);
1036		FILE_END_ALLOW_THREADS(f)
1037		if (nnow == 0) {
1038			if (!ferror(f->f_fp))
1039				break;
1040			PyErr_SetFromErrno(PyExc_IOError);
1041			clearerr(f->f_fp);
1042			PyBuffer_Release(&pbuf);
1043			return NULL;
1044		}
1045		ndone += nnow;
1046		ntodo -= nnow;
1047	}
1048	PyBuffer_Release(&pbuf);
1049	return PyInt_FromSsize_t(ndone);
1050}
1051
1052/**************************************************************************
1053Routine to get next line using platform fgets().
1054
1055Under MSVC 6:
1056
1057+ MS threadsafe getc is very slow (multiple layers of function calls before+
1058  after each character, to lock+unlock the stream).
1059+ The stream-locking functions are MS-internal -- can't access them from user
1060  code.
1061+ There's nothing Tim could find in the MS C or platform SDK libraries that
1062  can worm around this.
1063+ MS fgets locks/unlocks only once per line; it's the only hook we have.
1064
1065So we use fgets for speed(!), despite that it's painful.
1066
1067MS realloc is also slow.
1068
1069Reports from other platforms on this method vs getc_unlocked (which MS doesn't
1070have):
1071	Linux		a wash
1072	Solaris		a wash
1073	Tru64 Unix	getline_via_fgets significantly faster
1074
1075CAUTION:  The C std isn't clear about this:  in those cases where fgets
1076writes something into the buffer, can it write into any position beyond the
1077required trailing null byte?  MSVC 6 fgets does not, and no platform is (yet)
1078known on which it does; and it would be a strange way to code fgets. Still,
1079getline_via_fgets may not work correctly if it does.  The std test
1080test_bufio.py should fail if platform fgets() routinely writes beyond the
1081trailing null byte.  #define DONT_USE_FGETS_IN_GETLINE to disable this code.
1082**************************************************************************/
1083
1084/* Use this routine if told to, or by default on non-get_unlocked()
1085 * platforms unless told not to.  Yikes!  Let's spell that out:
1086 * On a platform with getc_unlocked():
1087 *     By default, use getc_unlocked().
1088 *     If you want to use fgets() instead, #define USE_FGETS_IN_GETLINE.
1089 * On a platform without getc_unlocked():
1090 *     By default, use fgets().
1091 *     If you don't want to use fgets(), #define DONT_USE_FGETS_IN_GETLINE.
1092 */
1093#if !defined(USE_FGETS_IN_GETLINE) && !defined(HAVE_GETC_UNLOCKED)
1094#define USE_FGETS_IN_GETLINE
1095#endif
1096
1097#if defined(DONT_USE_FGETS_IN_GETLINE) && defined(USE_FGETS_IN_GETLINE)
1098#undef USE_FGETS_IN_GETLINE
1099#endif
1100
1101#ifdef USE_FGETS_IN_GETLINE
1102static PyObject*
1103getline_via_fgets(PyFileObject *f, FILE *fp)
1104{
1105/* INITBUFSIZE is the maximum line length that lets us get away with the fast
1106 * no-realloc, one-fgets()-call path.  Boosting it isn't free, because we have
1107 * to fill this much of the buffer with a known value in order to figure out
1108 * how much of the buffer fgets() overwrites.  So if INITBUFSIZE is larger
1109 * than "most" lines, we waste time filling unused buffer slots.  100 is
1110 * surely adequate for most peoples' email archives, chewing over source code,
1111 * etc -- "regular old text files".
1112 * MAXBUFSIZE is the maximum line length that lets us get away with the less
1113 * fast (but still zippy) no-realloc, two-fgets()-call path.  See above for
1114 * cautions about boosting that.  300 was chosen because the worst real-life
1115 * text-crunching job reported on Python-Dev was a mail-log crawler where over
1116 * half the lines were 254 chars.
1117 */
1118#define INITBUFSIZE 100
1119#define MAXBUFSIZE 300
1120	char* p;	/* temp */
1121	char buf[MAXBUFSIZE];
1122	PyObject* v;	/* the string object result */
1123	char* pvfree;	/* address of next free slot */
1124	char* pvend;    /* address one beyond last free slot */
1125	size_t nfree;	/* # of free buffer slots; pvend-pvfree */
1126	size_t total_v_size;  /* total # of slots in buffer */
1127	size_t increment;	/* amount to increment the buffer */
1128	size_t prev_v_size;
1129
1130	/* Optimize for normal case:  avoid _PyString_Resize if at all
1131	 * possible via first reading into stack buffer "buf".
1132	 */
1133	total_v_size = INITBUFSIZE;	/* start small and pray */
1134	pvfree = buf;
1135	for (;;) {
1136		FILE_BEGIN_ALLOW_THREADS(f)
1137		pvend = buf + total_v_size;
1138		nfree = pvend - pvfree;
1139		memset(pvfree, '\n', nfree);
1140		assert(nfree < INT_MAX); /* Should be atmost MAXBUFSIZE */
1141		p = fgets(pvfree, (int)nfree, fp);
1142		FILE_END_ALLOW_THREADS(f)
1143
1144		if (p == NULL) {
1145			clearerr(fp);
1146			if (PyErr_CheckSignals())
1147				return NULL;
1148			v = PyString_FromStringAndSize(buf, pvfree - buf);
1149			return v;
1150		}
1151		/* fgets read *something* */
1152		p = memchr(pvfree, '\n', nfree);
1153		if (p != NULL) {
1154			/* Did the \n come from fgets or from us?
1155			 * Since fgets stops at the first \n, and then writes
1156			 * \0, if it's from fgets a \0 must be next.  But if
1157			 * that's so, it could not have come from us, since
1158			 * the \n's we filled the buffer with have only more
1159			 * \n's to the right.
1160			 */
1161			if (p+1 < pvend && *(p+1) == '\0') {
1162				/* It's from fgets:  we win!  In particular,
1163				 * we haven't done any mallocs yet, and can
1164				 * build the final result on the first try.
1165				 */
1166				++p;	/* include \n from fgets */
1167			}
1168			else {
1169				/* Must be from us:  fgets didn't fill the
1170				 * buffer and didn't find a newline, so it
1171				 * must be the last and newline-free line of
1172				 * the file.
1173				 */
1174				assert(p > pvfree && *(p-1) == '\0');
1175				--p;	/* don't include \0 from fgets */
1176			}
1177			v = PyString_FromStringAndSize(buf, p - buf);
1178			return v;
1179		}
1180		/* yuck:  fgets overwrote all the newlines, i.e. the entire
1181		 * buffer.  So this line isn't over yet, or maybe it is but
1182		 * we're exactly at EOF.  If we haven't already, try using the
1183		 * rest of the stack buffer.
1184		 */
1185		assert(*(pvend-1) == '\0');
1186		if (pvfree == buf) {
1187			pvfree = pvend - 1;	/* overwrite trailing null */
1188			total_v_size = MAXBUFSIZE;
1189		}
1190		else
1191			break;
1192	}
1193
1194	/* The stack buffer isn't big enough; malloc a string object and read
1195	 * into its buffer.
1196	 */
1197	total_v_size = MAXBUFSIZE << 1;
1198	v = PyString_FromStringAndSize((char*)NULL, (int)total_v_size);
1199	if (v == NULL)
1200		return v;
1201	/* copy over everything except the last null byte */
1202	memcpy(BUF(v), buf, MAXBUFSIZE-1);
1203	pvfree = BUF(v) + MAXBUFSIZE - 1;
1204
1205	/* Keep reading stuff into v; if it ever ends successfully, break
1206	 * after setting p one beyond the end of the line.  The code here is
1207	 * very much like the code above, except reads into v's buffer; see
1208	 * the code above for detailed comments about the logic.
1209	 */
1210	for (;;) {
1211		FILE_BEGIN_ALLOW_THREADS(f)
1212		pvend = BUF(v) + total_v_size;
1213		nfree = pvend - pvfree;
1214		memset(pvfree, '\n', nfree);
1215		assert(nfree < INT_MAX);
1216		p = fgets(pvfree, (int)nfree, fp);
1217		FILE_END_ALLOW_THREADS(f)
1218
1219		if (p == NULL) {
1220			clearerr(fp);
1221			if (PyErr_CheckSignals()) {
1222				Py_DECREF(v);
1223				return NULL;
1224			}
1225			p = pvfree;
1226			break;
1227		}
1228		p = memchr(pvfree, '\n', nfree);
1229		if (p != NULL) {
1230			if (p+1 < pvend && *(p+1) == '\0') {
1231				/* \n came from fgets */
1232				++p;
1233				break;
1234			}
1235			/* \n came from us; last line of file, no newline */
1236			assert(p > pvfree && *(p-1) == '\0');
1237			--p;
1238			break;
1239		}
1240		/* expand buffer and try again */
1241		assert(*(pvend-1) == '\0');
1242		increment = total_v_size >> 2;	/* mild exponential growth */
1243		prev_v_size = total_v_size;
1244		total_v_size += increment;
1245		/* check for overflow */
1246		if (total_v_size <= prev_v_size ||
1247		    total_v_size > PY_SSIZE_T_MAX) {
1248			PyErr_SetString(PyExc_OverflowError,
1249			    "line is longer than a Python string can hold");
1250			Py_DECREF(v);
1251			return NULL;
1252		}
1253		if (_PyString_Resize(&v, (int)total_v_size) < 0)
1254			return NULL;
1255		/* overwrite the trailing null byte */
1256		pvfree = BUF(v) + (prev_v_size - 1);
1257	}
1258	if (BUF(v) + total_v_size != p)
1259		_PyString_Resize(&v, p - BUF(v));
1260	return v;
1261#undef INITBUFSIZE
1262#undef MAXBUFSIZE
1263}
1264#endif	/* ifdef USE_FGETS_IN_GETLINE */
1265
1266/* Internal routine to get a line.
1267   Size argument interpretation:
1268   > 0: max length;
1269   <= 0: read arbitrary line
1270*/
1271
1272static PyObject *
1273get_line(PyFileObject *f, int n)
1274{
1275	FILE *fp = f->f_fp;
1276	int c;
1277	char *buf, *end;
1278	size_t total_v_size;	/* total # of slots in buffer */
1279	size_t used_v_size;	/* # used slots in buffer */
1280	size_t increment;       /* amount to increment the buffer */
1281	PyObject *v;
1282	int newlinetypes = f->f_newlinetypes;
1283	int skipnextlf = f->f_skipnextlf;
1284	int univ_newline = f->f_univ_newline;
1285
1286#if defined(USE_FGETS_IN_GETLINE)
1287	if (n <= 0 && !univ_newline )
1288		return getline_via_fgets(f, fp);
1289#endif
1290	total_v_size = n > 0 ? n : 100;
1291	v = PyString_FromStringAndSize((char *)NULL, total_v_size);
1292	if (v == NULL)
1293		return NULL;
1294	buf = BUF(v);
1295	end = buf + total_v_size;
1296
1297	for (;;) {
1298		FILE_BEGIN_ALLOW_THREADS(f)
1299		FLOCKFILE(fp);
1300		if (univ_newline) {
1301			c = 'x'; /* Shut up gcc warning */
1302			while ( buf != end && (c = GETC(fp)) != EOF ) {
1303				if (skipnextlf ) {
1304					skipnextlf = 0;
1305					if (c == '\n') {
1306						/* Seeing a \n here with
1307						 * skipnextlf true means we
1308						 * saw a \r before.
1309						 */
1310						newlinetypes |= NEWLINE_CRLF;
1311						c = GETC(fp);
1312						if (c == EOF) break;
1313					} else {
1314						newlinetypes |= NEWLINE_CR;
1315					}
1316				}
1317				if (c == '\r') {
1318					skipnextlf = 1;
1319					c = '\n';
1320				} else if ( c == '\n')
1321					newlinetypes |= NEWLINE_LF;
1322				*buf++ = c;
1323				if (c == '\n') break;
1324			}
1325			if ( c == EOF && skipnextlf )
1326				newlinetypes |= NEWLINE_CR;
1327		} else /* If not universal newlines use the normal loop */
1328		while ((c = GETC(fp)) != EOF &&
1329		       (*buf++ = c) != '\n' &&
1330			buf != end)
1331			;
1332		FUNLOCKFILE(fp);
1333		FILE_END_ALLOW_THREADS(f)
1334		f->f_newlinetypes = newlinetypes;
1335		f->f_skipnextlf = skipnextlf;
1336		if (c == '\n')
1337			break;
1338		if (c == EOF) {
1339			if (ferror(fp)) {
1340				PyErr_SetFromErrno(PyExc_IOError);
1341				clearerr(fp);
1342				Py_DECREF(v);
1343				return NULL;
1344			}
1345			clearerr(fp);
1346			if (PyErr_CheckSignals()) {
1347				Py_DECREF(v);
1348				return NULL;
1349			}
1350			break;
1351		}
1352		/* Must be because buf == end */
1353		if (n > 0)
1354			break;
1355		used_v_size = total_v_size;
1356		increment = total_v_size >> 2; /* mild exponential growth */
1357		total_v_size += increment;
1358		if (total_v_size > PY_SSIZE_T_MAX) {
1359			PyErr_SetString(PyExc_OverflowError,
1360			    "line is longer than a Python string can hold");
1361			Py_DECREF(v);
1362			return NULL;
1363		}
1364		if (_PyString_Resize(&v, total_v_size) < 0)
1365			return NULL;
1366		buf = BUF(v) + used_v_size;
1367		end = BUF(v) + total_v_size;
1368	}
1369
1370	used_v_size = buf - BUF(v);
1371	if (used_v_size != total_v_size)
1372		_PyString_Resize(&v, used_v_size);
1373	return v;
1374}
1375
1376/* External C interface */
1377
1378PyObject *
1379PyFile_GetLine(PyObject *f, int n)
1380{
1381	PyObject *result;
1382
1383	if (f == NULL) {
1384		PyErr_BadInternalCall();
1385		return NULL;
1386	}
1387
1388	if (PyFile_Check(f)) {
1389		PyFileObject *fo = (PyFileObject *)f;
1390		if (fo->f_fp == NULL)
1391			return err_closed();
1392		/* refuse to mix with f.next() */
1393		if (fo->f_buf != NULL &&
1394		    (fo->f_bufend - fo->f_bufptr) > 0 &&
1395		    fo->f_buf[0] != '\0')
1396			return err_iterbuffered();
1397		result = get_line(fo, n);
1398	}
1399	else {
1400		PyObject *reader;
1401		PyObject *args;
1402
1403		reader = PyObject_GetAttrString(f, "readline");
1404		if (reader == NULL)
1405			return NULL;
1406		if (n <= 0)
1407			args = PyTuple_New(0);
1408		else
1409			args = Py_BuildValue("(i)", n);
1410		if (args == NULL) {
1411			Py_DECREF(reader);
1412			return NULL;
1413		}
1414		result = PyEval_CallObject(reader, args);
1415		Py_DECREF(reader);
1416		Py_DECREF(args);
1417		if (result != NULL && !PyString_Check(result) &&
1418		    !PyUnicode_Check(result)) {
1419			Py_DECREF(result);
1420			result = NULL;
1421			PyErr_SetString(PyExc_TypeError,
1422				   "object.readline() returned non-string");
1423		}
1424	}
1425
1426	if (n < 0 && result != NULL && PyString_Check(result)) {
1427		char *s = PyString_AS_STRING(result);
1428		Py_ssize_t len = PyString_GET_SIZE(result);
1429		if (len == 0) {
1430			Py_DECREF(result);
1431			result = NULL;
1432			PyErr_SetString(PyExc_EOFError,
1433					"EOF when reading a line");
1434		}
1435		else if (s[len-1] == '\n') {
1436			if (result->ob_refcnt == 1)
1437				_PyString_Resize(&result, len-1);
1438			else {
1439				PyObject *v;
1440				v = PyString_FromStringAndSize(s, len-1);
1441				Py_DECREF(result);
1442				result = v;
1443			}
1444		}
1445	}
1446#ifdef Py_USING_UNICODE
1447	if (n < 0 && result != NULL && PyUnicode_Check(result)) {
1448		Py_UNICODE *s = PyUnicode_AS_UNICODE(result);
1449		Py_ssize_t len = PyUnicode_GET_SIZE(result);
1450		if (len == 0) {
1451			Py_DECREF(result);
1452			result = NULL;
1453			PyErr_SetString(PyExc_EOFError,
1454					"EOF when reading a line");
1455		}
1456		else if (s[len-1] == '\n') {
1457			if (result->ob_refcnt == 1)
1458				PyUnicode_Resize(&result, len-1);
1459			else {
1460				PyObject *v;
1461				v = PyUnicode_FromUnicode(s, len-1);
1462				Py_DECREF(result);
1463				result = v;
1464			}
1465		}
1466	}
1467#endif
1468	return result;
1469}
1470
1471/* Python method */
1472
1473static PyObject *
1474file_readline(PyFileObject *f, PyObject *args)
1475{
1476	int n = -1;
1477
1478	if (f->f_fp == NULL)
1479		return err_closed();
1480	/* refuse to mix with f.next() */
1481	if (f->f_buf != NULL &&
1482	    (f->f_bufend - f->f_bufptr) > 0 &&
1483	    f->f_buf[0] != '\0')
1484		return err_iterbuffered();
1485	if (!PyArg_ParseTuple(args, "|i:readline", &n))
1486		return NULL;
1487	if (n == 0)
1488		return PyString_FromString("");
1489	if (n < 0)
1490		n = 0;
1491	return get_line(f, n);
1492}
1493
1494static PyObject *
1495file_readlines(PyFileObject *f, PyObject *args)
1496{
1497	long sizehint = 0;
1498	PyObject *list = NULL;
1499	PyObject *line;
1500	char small_buffer[SMALLCHUNK];
1501	char *buffer = small_buffer;
1502	size_t buffersize = SMALLCHUNK;
1503	PyObject *big_buffer = NULL;
1504	size_t nfilled = 0;
1505	size_t nread;
1506	size_t totalread = 0;
1507	char *p, *q, *end;
1508	int err;
1509	int shortread = 0;
1510
1511	if (f->f_fp == NULL)
1512		return err_closed();
1513	/* refuse to mix with f.next() */
1514	if (f->f_buf != NULL &&
1515	    (f->f_bufend - f->f_bufptr) > 0 &&
1516	    f->f_buf[0] != '\0')
1517		return err_iterbuffered();
1518	if (!PyArg_ParseTuple(args, "|l:readlines", &sizehint))
1519		return NULL;
1520	if ((list = PyList_New(0)) == NULL)
1521		return NULL;
1522	for (;;) {
1523		if (shortread)
1524			nread = 0;
1525		else {
1526			FILE_BEGIN_ALLOW_THREADS(f)
1527			errno = 0;
1528			nread = Py_UniversalNewlineFread(buffer+nfilled,
1529				buffersize-nfilled, f->f_fp, (PyObject *)f);
1530			FILE_END_ALLOW_THREADS(f)
1531			shortread = (nread < buffersize-nfilled);
1532		}
1533		if (nread == 0) {
1534			sizehint = 0;
1535			if (!ferror(f->f_fp))
1536				break;
1537			PyErr_SetFromErrno(PyExc_IOError);
1538			clearerr(f->f_fp);
1539			goto error;
1540		}
1541		totalread += nread;
1542		p = (char *)memchr(buffer+nfilled, '\n', nread);
1543		if (p == NULL) {
1544			/* Need a larger buffer to fit this line */
1545			nfilled += nread;
1546			buffersize *= 2;
1547			if (buffersize > PY_SSIZE_T_MAX) {
1548				PyErr_SetString(PyExc_OverflowError,
1549			    "line is longer than a Python string can hold");
1550				goto error;
1551			}
1552			if (big_buffer == NULL) {
1553				/* Create the big buffer */
1554				big_buffer = PyString_FromStringAndSize(
1555					NULL, buffersize);
1556				if (big_buffer == NULL)
1557					goto error;
1558				buffer = PyString_AS_STRING(big_buffer);
1559				memcpy(buffer, small_buffer, nfilled);
1560			}
1561			else {
1562				/* Grow the big buffer */
1563				if ( _PyString_Resize(&big_buffer, buffersize) < 0 )
1564					goto error;
1565				buffer = PyString_AS_STRING(big_buffer);
1566			}
1567			continue;
1568		}
1569		end = buffer+nfilled+nread;
1570		q = buffer;
1571		do {
1572			/* Process complete lines */
1573			p++;
1574			line = PyString_FromStringAndSize(q, p-q);
1575			if (line == NULL)
1576				goto error;
1577			err = PyList_Append(list, line);
1578			Py_DECREF(line);
1579			if (err != 0)
1580				goto error;
1581			q = p;
1582			p = (char *)memchr(q, '\n', end-q);
1583		} while (p != NULL);
1584		/* Move the remaining incomplete line to the start */
1585		nfilled = end-q;
1586		memmove(buffer, q, nfilled);
1587		if (sizehint > 0)
1588			if (totalread >= (size_t)sizehint)
1589				break;
1590	}
1591	if (nfilled != 0) {
1592		/* Partial last line */
1593		line = PyString_FromStringAndSize(buffer, nfilled);
1594		if (line == NULL)
1595			goto error;
1596		if (sizehint > 0) {
1597			/* Need to complete the last line */
1598			PyObject *rest = get_line(f, 0);
1599			if (rest == NULL) {
1600				Py_DECREF(line);
1601				goto error;
1602			}
1603			PyString_Concat(&line, rest);
1604			Py_DECREF(rest);
1605			if (line == NULL)
1606				goto error;
1607		}
1608		err = PyList_Append(list, line);
1609		Py_DECREF(line);
1610		if (err != 0)
1611			goto error;
1612	}
1613
1614cleanup:
1615	Py_XDECREF(big_buffer);
1616	return list;
1617
1618error:
1619	Py_CLEAR(list);
1620	goto cleanup;
1621}
1622
1623static PyObject *
1624file_write(PyFileObject *f, PyObject *args)
1625{
1626	Py_buffer pbuf;
1627	char *s;
1628	Py_ssize_t n, n2;
1629	if (f->f_fp == NULL)
1630		return err_closed();
1631	if (f->f_binary) {
1632		if (!PyArg_ParseTuple(args, "s*", &pbuf))
1633			return NULL;
1634		s = pbuf.buf;
1635		n = pbuf.len;
1636	} else
1637		if (!PyArg_ParseTuple(args, "t#", &s, &n))
1638		return NULL;
1639	f->f_softspace = 0;
1640	FILE_BEGIN_ALLOW_THREADS(f)
1641	errno = 0;
1642	n2 = fwrite(s, 1, n, f->f_fp);
1643	FILE_END_ALLOW_THREADS(f)
1644	if (f->f_binary)
1645		PyBuffer_Release(&pbuf);
1646	if (n2 != n) {
1647		PyErr_SetFromErrno(PyExc_IOError);
1648		clearerr(f->f_fp);
1649		return NULL;
1650	}
1651	Py_INCREF(Py_None);
1652	return Py_None;
1653}
1654
1655static PyObject *
1656file_writelines(PyFileObject *f, PyObject *seq)
1657{
1658#define CHUNKSIZE 1000
1659	PyObject *list, *line;
1660	PyObject *it;	/* iter(seq) */
1661	PyObject *result;
1662	int index, islist;
1663	Py_ssize_t i, j, nwritten, len;
1664
1665	assert(seq != NULL);
1666	if (f->f_fp == NULL)
1667		return err_closed();
1668
1669	result = NULL;
1670	list = NULL;
1671	islist = PyList_Check(seq);
1672	if  (islist)
1673		it = NULL;
1674	else {
1675		it = PyObject_GetIter(seq);
1676		if (it == NULL) {
1677			PyErr_SetString(PyExc_TypeError,
1678				"writelines() requires an iterable argument");
1679			return NULL;
1680		}
1681		/* From here on, fail by going to error, to reclaim "it". */
1682		list = PyList_New(CHUNKSIZE);
1683		if (list == NULL)
1684			goto error;
1685	}
1686
1687	/* Strategy: slurp CHUNKSIZE lines into a private list,
1688	   checking that they are all strings, then write that list
1689	   without holding the interpreter lock, then come back for more. */
1690	for (index = 0; ; index += CHUNKSIZE) {
1691		if (islist) {
1692			Py_XDECREF(list);
1693			list = PyList_GetSlice(seq, index, index+CHUNKSIZE);
1694			if (list == NULL)
1695				goto error;
1696			j = PyList_GET_SIZE(list);
1697		}
1698		else {
1699			for (j = 0; j < CHUNKSIZE; j++) {
1700				line = PyIter_Next(it);
1701				if (line == NULL) {
1702					if (PyErr_Occurred())
1703						goto error;
1704					break;
1705				}
1706				PyList_SetItem(list, j, line);
1707			}
1708		}
1709		if (j == 0)
1710			break;
1711
1712		/* Check that all entries are indeed strings. If not,
1713		   apply the same rules as for file.write() and
1714		   convert the results to strings. This is slow, but
1715		   seems to be the only way since all conversion APIs
1716		   could potentially execute Python code. */
1717		for (i = 0; i < j; i++) {
1718			PyObject *v = PyList_GET_ITEM(list, i);
1719			if (!PyString_Check(v)) {
1720			    	const char *buffer;
1721				if (((f->f_binary &&
1722				      PyObject_AsReadBuffer(v,
1723					      (const void**)&buffer,
1724							    &len)) ||
1725				     PyObject_AsCharBuffer(v,
1726							   &buffer,
1727							   &len))) {
1728					PyErr_SetString(PyExc_TypeError,
1729			"writelines() argument must be a sequence of strings");
1730					goto error;
1731				}
1732				line = PyString_FromStringAndSize(buffer,
1733								  len);
1734				if (line == NULL)
1735					goto error;
1736				Py_DECREF(v);
1737				PyList_SET_ITEM(list, i, line);
1738			}
1739		}
1740
1741		/* Since we are releasing the global lock, the
1742		   following code may *not* execute Python code. */
1743		f->f_softspace = 0;
1744		FILE_BEGIN_ALLOW_THREADS(f)
1745		errno = 0;
1746		for (i = 0; i < j; i++) {
1747		    	line = PyList_GET_ITEM(list, i);
1748			len = PyString_GET_SIZE(line);
1749			nwritten = fwrite(PyString_AS_STRING(line),
1750					  1, len, f->f_fp);
1751			if (nwritten != len) {
1752				FILE_ABORT_ALLOW_THREADS(f)
1753				PyErr_SetFromErrno(PyExc_IOError);
1754				clearerr(f->f_fp);
1755				goto error;
1756			}
1757		}
1758		FILE_END_ALLOW_THREADS(f)
1759
1760		if (j < CHUNKSIZE)
1761			break;
1762	}
1763
1764	Py_INCREF(Py_None);
1765	result = Py_None;
1766  error:
1767	Py_XDECREF(list);
1768  	Py_XDECREF(it);
1769	return result;
1770#undef CHUNKSIZE
1771}
1772
1773static PyObject *
1774file_self(PyFileObject *f)
1775{
1776	if (f->f_fp == NULL)
1777		return err_closed();
1778	Py_INCREF(f);
1779	return (PyObject *)f;
1780}
1781
1782static PyObject *
1783file_xreadlines(PyFileObject *f)
1784{
1785	if (PyErr_WarnPy3k("f.xreadlines() not supported in 3.x, "
1786			   "try 'for line in f' instead", 1) < 0)
1787	       return NULL;
1788	return file_self(f);
1789}
1790
1791static PyObject *
1792file_exit(PyObject *f, PyObject *args)
1793{
1794	PyObject *ret = PyObject_CallMethod(f, "close", NULL);
1795	if (!ret)
1796		/* If error occurred, pass through */
1797		return NULL;
1798	Py_DECREF(ret);
1799	/* We cannot return the result of close since a true
1800	 * value will be interpreted as "yes, swallow the
1801	 * exception if one was raised inside the with block". */
1802	Py_RETURN_NONE;
1803}
1804
1805PyDoc_STRVAR(readline_doc,
1806"readline([size]) -> next line from the file, as a string.\n"
1807"\n"
1808"Retain newline.  A non-negative size argument limits the maximum\n"
1809"number of bytes to return (an incomplete line may be returned then).\n"
1810"Return an empty string at EOF.");
1811
1812PyDoc_STRVAR(read_doc,
1813"read([size]) -> read at most size bytes, returned as a string.\n"
1814"\n"
1815"If the size argument is negative or omitted, read until EOF is reached.\n"
1816"Notice that when in non-blocking mode, less data than what was requested\n"
1817"may be returned, even if no size parameter was given.");
1818
1819PyDoc_STRVAR(write_doc,
1820"write(str) -> None.  Write string str to file.\n"
1821"\n"
1822"Note that due to buffering, flush() or close() may be needed before\n"
1823"the file on disk reflects the data written.");
1824
1825PyDoc_STRVAR(fileno_doc,
1826"fileno() -> integer \"file descriptor\".\n"
1827"\n"
1828"This is needed for lower-level file interfaces, such os.read().");
1829
1830PyDoc_STRVAR(seek_doc,
1831"seek(offset[, whence]) -> None.  Move to new file position.\n"
1832"\n"
1833"Argument offset is a byte count.  Optional argument whence defaults to\n"
1834"0 (offset from start of file, offset should be >= 0); other values are 1\n"
1835"(move relative to current position, positive or negative), and 2 (move\n"
1836"relative to end of file, usually negative, although many platforms allow\n"
1837"seeking beyond the end of a file).  If the file is opened in text mode,\n"
1838"only offsets returned by tell() are legal.  Use of other offsets causes\n"
1839"undefined behavior."
1840"\n"
1841"Note that not all file objects are seekable.");
1842
1843#ifdef HAVE_FTRUNCATE
1844PyDoc_STRVAR(truncate_doc,
1845"truncate([size]) -> None.  Truncate the file to at most size bytes.\n"
1846"\n"
1847"Size defaults to the current file position, as returned by tell().");
1848#endif
1849
1850PyDoc_STRVAR(tell_doc,
1851"tell() -> current file position, an integer (may be a long integer).");
1852
1853PyDoc_STRVAR(readinto_doc,
1854"readinto() -> Undocumented.  Don't use this; it may go away.");
1855
1856PyDoc_STRVAR(readlines_doc,
1857"readlines([size]) -> list of strings, each a line from the file.\n"
1858"\n"
1859"Call readline() repeatedly and return a list of the lines so read.\n"
1860"The optional size argument, if given, is an approximate bound on the\n"
1861"total number of bytes in the lines returned.");
1862
1863PyDoc_STRVAR(xreadlines_doc,
1864"xreadlines() -> returns self.\n"
1865"\n"
1866"For backward compatibility. File objects now include the performance\n"
1867"optimizations previously implemented in the xreadlines module.");
1868
1869PyDoc_STRVAR(writelines_doc,
1870"writelines(sequence_of_strings) -> None.  Write the strings to the file.\n"
1871"\n"
1872"Note that newlines are not added.  The sequence can be any iterable object\n"
1873"producing strings. This is equivalent to calling write() for each string.");
1874
1875PyDoc_STRVAR(flush_doc,
1876"flush() -> None.  Flush the internal I/O buffer.");
1877
1878PyDoc_STRVAR(close_doc,
1879"close() -> None or (perhaps) an integer.  Close the file.\n"
1880"\n"
1881"Sets data attribute .closed to True.  A closed file cannot be used for\n"
1882"further I/O operations.  close() may be called more than once without\n"
1883"error.  Some kinds of file objects (for example, opened by popen())\n"
1884"may return an exit status upon closing.");
1885
1886PyDoc_STRVAR(isatty_doc,
1887"isatty() -> true or false.  True if the file is connected to a tty device.");
1888
1889PyDoc_STRVAR(enter_doc,
1890	     "__enter__() -> self.");
1891
1892PyDoc_STRVAR(exit_doc,
1893	     "__exit__(*excinfo) -> None.  Closes the file.");
1894
1895static PyMethodDef file_methods[] = {
1896	{"readline",  (PyCFunction)file_readline, METH_VARARGS, readline_doc},
1897	{"read",      (PyCFunction)file_read,     METH_VARARGS, read_doc},
1898	{"write",     (PyCFunction)file_write,    METH_VARARGS, write_doc},
1899	{"fileno",    (PyCFunction)file_fileno,   METH_NOARGS,  fileno_doc},
1900	{"seek",      (PyCFunction)file_seek,     METH_VARARGS, seek_doc},
1901#ifdef HAVE_FTRUNCATE
1902	{"truncate",  (PyCFunction)file_truncate, METH_VARARGS, truncate_doc},
1903#endif
1904	{"tell",      (PyCFunction)file_tell,     METH_NOARGS,  tell_doc},
1905	{"readinto",  (PyCFunction)file_readinto, METH_VARARGS, readinto_doc},
1906	{"readlines", (PyCFunction)file_readlines, METH_VARARGS, readlines_doc},
1907	{"xreadlines",(PyCFunction)file_xreadlines, METH_NOARGS, xreadlines_doc},
1908	{"writelines",(PyCFunction)file_writelines, METH_O,     writelines_doc},
1909	{"flush",     (PyCFunction)file_flush,    METH_NOARGS,  flush_doc},
1910	{"close",     (PyCFunction)file_close,    METH_NOARGS,  close_doc},
1911	{"isatty",    (PyCFunction)file_isatty,   METH_NOARGS,  isatty_doc},
1912	{"__enter__", (PyCFunction)file_self,     METH_NOARGS,  enter_doc},
1913	{"__exit__",  (PyCFunction)file_exit,     METH_VARARGS, exit_doc},
1914	{NULL,	      NULL}		/* sentinel */
1915};
1916
1917#define OFF(x) offsetof(PyFileObject, x)
1918
1919static PyMemberDef file_memberlist[] = {
1920	{"mode",	T_OBJECT,	OFF(f_mode),	RO,
1921	 "file mode ('r', 'U', 'w', 'a', possibly with 'b' or '+' added)"},
1922	{"name",	T_OBJECT,	OFF(f_name),	RO,
1923	 "file name"},
1924	{"encoding",	T_OBJECT,	OFF(f_encoding),	RO,
1925	 "file encoding"},
1926	{"errors",	T_OBJECT,	OFF(f_errors),	RO,
1927	 "Unicode error handler"},
1928	/* getattr(f, "closed") is implemented without this table */
1929	{NULL}	/* Sentinel */
1930};
1931
1932static PyObject *
1933get_closed(PyFileObject *f, void *closure)
1934{
1935	return PyBool_FromLong((long)(f->f_fp == 0));
1936}
1937static PyObject *
1938get_newlines(PyFileObject *f, void *closure)
1939{
1940	switch (f->f_newlinetypes) {
1941	case NEWLINE_UNKNOWN:
1942		Py_INCREF(Py_None);
1943		return Py_None;
1944	case NEWLINE_CR:
1945		return PyString_FromString("\r");
1946	case N…