/Objects/fileobject.c
C | 2665 lines | 2144 code | 208 blank | 313 comment | 604 complexity | ea0f032e46e8ca655c62abe0c0f4ba9b MD5 | raw file
Large files files are truncated, but you can click here to view the full file
1/* File object implementation */ 2 3#define PY_SSIZE_T_CLEAN 4#include "Python.h" 5#include "structmember.h" 6 7#ifdef HAVE_SYS_TYPES_H 8#include <sys/types.h> 9#endif /* HAVE_SYS_TYPES_H */ 10 11#ifdef MS_WINDOWS 12#define fileno _fileno 13/* can simulate truncate with Win32 API functions; see file_truncate */ 14#define HAVE_FTRUNCATE 15#define WIN32_LEAN_AND_MEAN 16#include <windows.h> 17#endif 18 19#ifdef _MSC_VER 20/* Need GetVersion to see if on NT so safe to use _wfopen */ 21#define WIN32_LEAN_AND_MEAN 22#include <windows.h> 23#endif /* _MSC_VER */ 24 25#if defined(PYOS_OS2) && defined(PYCC_GCC) 26#include <io.h> 27#endif 28 29#define BUF(v) PyString_AS_STRING((PyStringObject *)v) 30 31#ifndef DONT_HAVE_ERRNO_H 32#include <errno.h> 33#endif 34 35#ifdef HAVE_GETC_UNLOCKED 36#define GETC(f) getc_unlocked(f) 37#define FLOCKFILE(f) flockfile(f) 38#define FUNLOCKFILE(f) funlockfile(f) 39#else 40#define GETC(f) getc(f) 41#define FLOCKFILE(f) 42#define FUNLOCKFILE(f) 43#endif 44 45/* Bits in f_newlinetypes */ 46#define NEWLINE_UNKNOWN 0 /* No newline seen, yet */ 47#define NEWLINE_CR 1 /* \r newline seen */ 48#define NEWLINE_LF 2 /* \n newline seen */ 49#define NEWLINE_CRLF 4 /* \r\n newline seen */ 50 51/* 52 * These macros release the GIL while preventing the f_close() function being 53 * called in the interval between them. For that purpose, a running total of 54 * the number of currently running unlocked code sections is kept in 55 * the unlocked_count field of the PyFileObject. The close() method raises 56 * an IOError if that field is non-zero. See issue #815646, #595601. 57 */ 58 59#define FILE_BEGIN_ALLOW_THREADS(fobj) \ 60{ \ 61 fobj->unlocked_count++; \ 62 Py_BEGIN_ALLOW_THREADS 63 64#define FILE_END_ALLOW_THREADS(fobj) \ 65 Py_END_ALLOW_THREADS \ 66 fobj->unlocked_count--; \ 67 assert(fobj->unlocked_count >= 0); \ 68} 69 70#define FILE_ABORT_ALLOW_THREADS(fobj) \ 71 Py_BLOCK_THREADS \ 72 fobj->unlocked_count--; \ 73 assert(fobj->unlocked_count >= 0); 74 75#ifdef __cplusplus 76extern "C" { 77#endif 78 79FILE * 80PyFile_AsFile(PyObject *f) 81{ 82 if (f == NULL || !PyFile_Check(f)) 83 return NULL; 84 else 85 return ((PyFileObject *)f)->f_fp; 86} 87 88void PyFile_IncUseCount(PyFileObject *fobj) 89{ 90 fobj->unlocked_count++; 91} 92 93void PyFile_DecUseCount(PyFileObject *fobj) 94{ 95 fobj->unlocked_count--; 96 assert(fobj->unlocked_count >= 0); 97} 98 99PyObject * 100PyFile_Name(PyObject *f) 101{ 102 if (f == NULL || !PyFile_Check(f)) 103 return NULL; 104 else 105 return ((PyFileObject *)f)->f_name; 106} 107 108/* This is a safe wrapper around PyObject_Print to print to the FILE 109 of a PyFileObject. PyObject_Print releases the GIL but knows nothing 110 about PyFileObject. */ 111static int 112file_PyObject_Print(PyObject *op, PyFileObject *f, int flags) 113{ 114 int result; 115 PyFile_IncUseCount(f); 116 result = PyObject_Print(op, f->f_fp, flags); 117 PyFile_DecUseCount(f); 118 return result; 119} 120 121/* On Unix, fopen will succeed for directories. 122 In Python, there should be no file objects referring to 123 directories, so we need a check. */ 124 125static PyFileObject* 126dircheck(PyFileObject* f) 127{ 128#if defined(HAVE_FSTAT) && defined(S_IFDIR) && defined(EISDIR) 129 struct stat buf; 130 if (f->f_fp == NULL) 131 return f; 132 if (fstat(fileno(f->f_fp), &buf) == 0 && 133 S_ISDIR(buf.st_mode)) { 134 char *msg = strerror(EISDIR); 135 PyObject *exc = PyObject_CallFunction(PyExc_IOError, "(isO)", 136 EISDIR, msg, f->f_name); 137 PyErr_SetObject(PyExc_IOError, exc); 138 Py_XDECREF(exc); 139 return NULL; 140 } 141#endif 142 return f; 143} 144 145 146static PyObject * 147fill_file_fields(PyFileObject *f, FILE *fp, PyObject *name, char *mode, 148 int (*close)(FILE *)) 149{ 150 assert(name != NULL); 151 assert(f != NULL); 152 assert(PyFile_Check(f)); 153 assert(f->f_fp == NULL); 154 155 Py_DECREF(f->f_name); 156 Py_DECREF(f->f_mode); 157 Py_DECREF(f->f_encoding); 158 Py_DECREF(f->f_errors); 159 160 Py_INCREF(name); 161 f->f_name = name; 162 163 f->f_mode = PyString_FromString(mode); 164 165 f->f_close = close; 166 f->f_softspace = 0; 167 f->f_binary = strchr(mode,'b') != NULL; 168 f->f_buf = NULL; 169 f->f_univ_newline = (strchr(mode, 'U') != NULL); 170 f->f_newlinetypes = NEWLINE_UNKNOWN; 171 f->f_skipnextlf = 0; 172 Py_INCREF(Py_None); 173 f->f_encoding = Py_None; 174 Py_INCREF(Py_None); 175 f->f_errors = Py_None; 176 177 if (f->f_mode == NULL) 178 return NULL; 179 f->f_fp = fp; 180 f = dircheck(f); 181 return (PyObject *) f; 182} 183 184/* check for known incorrect mode strings - problem is, platforms are 185 free to accept any mode characters they like and are supposed to 186 ignore stuff they don't understand... write or append mode with 187 universal newline support is expressly forbidden by PEP 278. 188 Additionally, remove the 'U' from the mode string as platforms 189 won't know what it is. Non-zero return signals an exception */ 190int 191_PyFile_SanitizeMode(char *mode) 192{ 193 char *upos; 194 size_t len = strlen(mode); 195 196 if (!len) { 197 PyErr_SetString(PyExc_ValueError, "empty mode string"); 198 return -1; 199 } 200 201 upos = strchr(mode, 'U'); 202 if (upos) { 203 memmove(upos, upos+1, len-(upos-mode)); /* incl null char */ 204 205 if (mode[0] == 'w' || mode[0] == 'a') { 206 PyErr_Format(PyExc_ValueError, "universal newline " 207 "mode can only be used with modes " 208 "starting with 'r'"); 209 return -1; 210 } 211 212 if (mode[0] != 'r') { 213 memmove(mode+1, mode, strlen(mode)+1); 214 mode[0] = 'r'; 215 } 216 217 if (!strchr(mode, 'b')) { 218 memmove(mode+2, mode+1, strlen(mode)); 219 mode[1] = 'b'; 220 } 221 } else if (mode[0] != 'r' && mode[0] != 'w' && mode[0] != 'a') { 222 PyErr_Format(PyExc_ValueError, "mode string must begin with " 223 "one of 'r', 'w', 'a' or 'U', not '%.200s'", mode); 224 return -1; 225 } 226 227 return 0; 228} 229 230static PyObject * 231open_the_file(PyFileObject *f, char *name, char *mode) 232{ 233 char *newmode; 234 assert(f != NULL); 235 assert(PyFile_Check(f)); 236#ifdef MS_WINDOWS 237 /* windows ignores the passed name in order to support Unicode */ 238 assert(f->f_name != NULL); 239#else 240 assert(name != NULL); 241#endif 242 assert(mode != NULL); 243 assert(f->f_fp == NULL); 244 245 /* probably need to replace 'U' by 'rb' */ 246 newmode = PyMem_MALLOC(strlen(mode) + 3); 247 if (!newmode) { 248 PyErr_NoMemory(); 249 return NULL; 250 } 251 strcpy(newmode, mode); 252 253 if (_PyFile_SanitizeMode(newmode)) { 254 f = NULL; 255 goto cleanup; 256 } 257 258 /* rexec.py can't stop a user from getting the file() constructor -- 259 all they have to do is get *any* file object f, and then do 260 type(f). Here we prevent them from doing damage with it. */ 261 if (PyEval_GetRestricted()) { 262 PyErr_SetString(PyExc_IOError, 263 "file() constructor not accessible in restricted mode"); 264 f = NULL; 265 goto cleanup; 266 } 267 errno = 0; 268 269#ifdef MS_WINDOWS 270 if (PyUnicode_Check(f->f_name)) { 271 PyObject *wmode; 272 wmode = PyUnicode_DecodeASCII(newmode, strlen(newmode), NULL); 273 if (f->f_name && wmode) { 274 FILE_BEGIN_ALLOW_THREADS(f) 275 /* PyUnicode_AS_UNICODE OK without thread 276 lock as it is a simple dereference. */ 277 f->f_fp = _wfopen(PyUnicode_AS_UNICODE(f->f_name), 278 PyUnicode_AS_UNICODE(wmode)); 279 FILE_END_ALLOW_THREADS(f) 280 } 281 Py_XDECREF(wmode); 282 } 283#endif 284 if (NULL == f->f_fp && NULL != name) { 285 FILE_BEGIN_ALLOW_THREADS(f) 286 f->f_fp = fopen(name, newmode); 287 FILE_END_ALLOW_THREADS(f) 288 } 289 290 if (f->f_fp == NULL) { 291#if defined _MSC_VER && (_MSC_VER < 1400 || !defined(__STDC_SECURE_LIB__)) 292 /* MSVC 6 (Microsoft) leaves errno at 0 for bad mode strings, 293 * across all Windows flavors. When it sets EINVAL varies 294 * across Windows flavors, the exact conditions aren't 295 * documented, and the answer lies in the OS's implementation 296 * of Win32's CreateFile function (whose source is secret). 297 * Seems the best we can do is map EINVAL to ENOENT. 298 * Starting with Visual Studio .NET 2005, EINVAL is correctly 299 * set by our CRT error handler (set in exceptions.c.) 300 */ 301 if (errno == 0) /* bad mode string */ 302 errno = EINVAL; 303 else if (errno == EINVAL) /* unknown, but not a mode string */ 304 errno = ENOENT; 305#endif 306 /* EINVAL is returned when an invalid filename or 307 * an invalid mode is supplied. */ 308 if (errno == EINVAL) { 309 PyObject *v; 310 char message[100]; 311 PyOS_snprintf(message, 100, 312 "invalid mode ('%.50s') or filename", mode); 313 v = Py_BuildValue("(isO)", errno, message, f->f_name); 314 if (v != NULL) { 315 PyErr_SetObject(PyExc_IOError, v); 316 Py_DECREF(v); 317 } 318 } 319 else 320 PyErr_SetFromErrnoWithFilenameObject(PyExc_IOError, f->f_name); 321 f = NULL; 322 } 323 if (f != NULL) 324 f = dircheck(f); 325 326cleanup: 327 PyMem_FREE(newmode); 328 329 return (PyObject *)f; 330} 331 332static PyObject * 333close_the_file(PyFileObject *f) 334{ 335 int sts = 0; 336 int (*local_close)(FILE *); 337 FILE *local_fp = f->f_fp; 338 if (local_fp != NULL) { 339 local_close = f->f_close; 340 if (local_close != NULL && f->unlocked_count > 0) { 341 if (f->ob_refcnt > 0) { 342 PyErr_SetString(PyExc_IOError, 343 "close() called during concurrent " 344 "operation on the same file object."); 345 } else { 346 /* This should not happen unless someone is 347 * carelessly playing with the PyFileObject 348 * struct fields and/or its associated FILE 349 * pointer. */ 350 PyErr_SetString(PyExc_SystemError, 351 "PyFileObject locking error in " 352 "destructor (refcnt <= 0 at close)."); 353 } 354 return NULL; 355 } 356 /* NULL out the FILE pointer before releasing the GIL, because 357 * it will not be valid anymore after the close() function is 358 * called. */ 359 f->f_fp = NULL; 360 if (local_close != NULL) { 361 Py_BEGIN_ALLOW_THREADS 362 errno = 0; 363 sts = (*local_close)(local_fp); 364 Py_END_ALLOW_THREADS 365 if (sts == EOF) 366 return PyErr_SetFromErrno(PyExc_IOError); 367 if (sts != 0) 368 return PyInt_FromLong((long)sts); 369 } 370 } 371 Py_RETURN_NONE; 372} 373 374PyObject * 375PyFile_FromFile(FILE *fp, char *name, char *mode, int (*close)(FILE *)) 376{ 377 PyFileObject *f = (PyFileObject *)PyFile_Type.tp_new(&PyFile_Type, 378 NULL, NULL); 379 if (f != NULL) { 380 PyObject *o_name = PyString_FromString(name); 381 if (o_name == NULL) 382 return NULL; 383 if (fill_file_fields(f, fp, o_name, mode, close) == NULL) { 384 Py_DECREF(f); 385 f = NULL; 386 } 387 Py_DECREF(o_name); 388 } 389 return (PyObject *) f; 390} 391 392PyObject * 393PyFile_FromString(char *name, char *mode) 394{ 395 extern int fclose(FILE *); 396 PyFileObject *f; 397 398 f = (PyFileObject *)PyFile_FromFile((FILE *)NULL, name, mode, fclose); 399 if (f != NULL) { 400 if (open_the_file(f, name, mode) == NULL) { 401 Py_DECREF(f); 402 f = NULL; 403 } 404 } 405 return (PyObject *)f; 406} 407 408void 409PyFile_SetBufSize(PyObject *f, int bufsize) 410{ 411 PyFileObject *file = (PyFileObject *)f; 412 if (bufsize >= 0) { 413 int type; 414 switch (bufsize) { 415 case 0: 416 type = _IONBF; 417 break; 418#ifdef HAVE_SETVBUF 419 case 1: 420 type = _IOLBF; 421 bufsize = BUFSIZ; 422 break; 423#endif 424 default: 425 type = _IOFBF; 426#ifndef HAVE_SETVBUF 427 bufsize = BUFSIZ; 428#endif 429 break; 430 } 431 fflush(file->f_fp); 432 if (type == _IONBF) { 433 PyMem_Free(file->f_setbuf); 434 file->f_setbuf = NULL; 435 } else { 436 file->f_setbuf = (char *)PyMem_Realloc(file->f_setbuf, 437 bufsize); 438 } 439#ifdef HAVE_SETVBUF 440 setvbuf(file->f_fp, file->f_setbuf, type, bufsize); 441#else /* !HAVE_SETVBUF */ 442 setbuf(file->f_fp, file->f_setbuf); 443#endif /* !HAVE_SETVBUF */ 444 } 445} 446 447/* Set the encoding used to output Unicode strings. 448 Return 1 on success, 0 on failure. */ 449 450int 451PyFile_SetEncoding(PyObject *f, const char *enc) 452{ 453 return PyFile_SetEncodingAndErrors(f, enc, NULL); 454} 455 456int 457PyFile_SetEncodingAndErrors(PyObject *f, const char *enc, char* errors) 458{ 459 PyFileObject *file = (PyFileObject*)f; 460 PyObject *str, *oerrors; 461 462 assert(PyFile_Check(f)); 463 str = PyString_FromString(enc); 464 if (!str) 465 return 0; 466 if (errors) { 467 oerrors = PyString_FromString(errors); 468 if (!oerrors) { 469 Py_DECREF(str); 470 return 0; 471 } 472 } else { 473 oerrors = Py_None; 474 Py_INCREF(Py_None); 475 } 476 Py_DECREF(file->f_encoding); 477 file->f_encoding = str; 478 Py_DECREF(file->f_errors); 479 file->f_errors = oerrors; 480 return 1; 481} 482 483static PyObject * 484err_closed(void) 485{ 486 PyErr_SetString(PyExc_ValueError, "I/O operation on closed file"); 487 return NULL; 488} 489 490/* Refuse regular file I/O if there's data in the iteration-buffer. 491 * Mixing them would cause data to arrive out of order, as the read* 492 * methods don't use the iteration buffer. */ 493static PyObject * 494err_iterbuffered(void) 495{ 496 PyErr_SetString(PyExc_ValueError, 497 "Mixing iteration and read methods would lose data"); 498 return NULL; 499} 500 501static void drop_readahead(PyFileObject *); 502 503/* Methods */ 504 505static void 506file_dealloc(PyFileObject *f) 507{ 508 PyObject *ret; 509 if (f->weakreflist != NULL) 510 PyObject_ClearWeakRefs((PyObject *) f); 511 ret = close_the_file(f); 512 if (!ret) { 513 PySys_WriteStderr("close failed in file object destructor:\n"); 514 PyErr_Print(); 515 } 516 else { 517 Py_DECREF(ret); 518 } 519 PyMem_Free(f->f_setbuf); 520 Py_XDECREF(f->f_name); 521 Py_XDECREF(f->f_mode); 522 Py_XDECREF(f->f_encoding); 523 Py_XDECREF(f->f_errors); 524 drop_readahead(f); 525 Py_TYPE(f)->tp_free((PyObject *)f); 526} 527 528static PyObject * 529file_repr(PyFileObject *f) 530{ 531 if (PyUnicode_Check(f->f_name)) { 532#ifdef Py_USING_UNICODE 533 PyObject *ret = NULL; 534 PyObject *name = PyUnicode_AsUnicodeEscapeString(f->f_name); 535 const char *name_str = name ? PyString_AsString(name) : "?"; 536 ret = PyString_FromFormat("<%s file u'%s', mode '%s' at %p>", 537 f->f_fp == NULL ? "closed" : "open", 538 name_str, 539 PyString_AsString(f->f_mode), 540 f); 541 Py_XDECREF(name); 542 return ret; 543#endif 544 } else { 545 return PyString_FromFormat("<%s file '%s', mode '%s' at %p>", 546 f->f_fp == NULL ? "closed" : "open", 547 PyString_AsString(f->f_name), 548 PyString_AsString(f->f_mode), 549 f); 550 } 551} 552 553static PyObject * 554file_close(PyFileObject *f) 555{ 556 PyObject *sts = close_the_file(f); 557 PyMem_Free(f->f_setbuf); 558 f->f_setbuf = NULL; 559 return sts; 560} 561 562 563/* Our very own off_t-like type, 64-bit if possible */ 564#if !defined(HAVE_LARGEFILE_SUPPORT) 565typedef off_t Py_off_t; 566#elif SIZEOF_OFF_T >= 8 567typedef off_t Py_off_t; 568#elif SIZEOF_FPOS_T >= 8 569typedef fpos_t Py_off_t; 570#else 571#error "Large file support, but neither off_t nor fpos_t is large enough." 572#endif 573 574 575/* a portable fseek() function 576 return 0 on success, non-zero on failure (with errno set) */ 577static int 578_portable_fseek(FILE *fp, Py_off_t offset, int whence) 579{ 580#if !defined(HAVE_LARGEFILE_SUPPORT) 581 return fseek(fp, offset, whence); 582#elif defined(HAVE_FSEEKO) && SIZEOF_OFF_T >= 8 583 return fseeko(fp, offset, whence); 584#elif defined(HAVE_FSEEK64) 585 return fseek64(fp, offset, whence); 586#elif defined(__BEOS__) 587 return _fseek(fp, offset, whence); 588#elif SIZEOF_FPOS_T >= 8 589 /* lacking a 64-bit capable fseek(), use a 64-bit capable fsetpos() 590 and fgetpos() to implement fseek()*/ 591 fpos_t pos; 592 switch (whence) { 593 case SEEK_END: 594#ifdef MS_WINDOWS 595 fflush(fp); 596 if (_lseeki64(fileno(fp), 0, 2) == -1) 597 return -1; 598#else 599 if (fseek(fp, 0, SEEK_END) != 0) 600 return -1; 601#endif 602 /* fall through */ 603 case SEEK_CUR: 604 if (fgetpos(fp, &pos) != 0) 605 return -1; 606 offset += pos; 607 break; 608 /* case SEEK_SET: break; */ 609 } 610 return fsetpos(fp, &offset); 611#else 612#error "Large file support, but no way to fseek." 613#endif 614} 615 616 617/* a portable ftell() function 618 Return -1 on failure with errno set appropriately, current file 619 position on success */ 620static Py_off_t 621_portable_ftell(FILE* fp) 622{ 623#if !defined(HAVE_LARGEFILE_SUPPORT) 624 return ftell(fp); 625#elif defined(HAVE_FTELLO) && SIZEOF_OFF_T >= 8 626 return ftello(fp); 627#elif defined(HAVE_FTELL64) 628 return ftell64(fp); 629#elif SIZEOF_FPOS_T >= 8 630 fpos_t pos; 631 if (fgetpos(fp, &pos) != 0) 632 return -1; 633 return pos; 634#else 635#error "Large file support, but no way to ftell." 636#endif 637} 638 639 640static PyObject * 641file_seek(PyFileObject *f, PyObject *args) 642{ 643 int whence; 644 int ret; 645 Py_off_t offset; 646 PyObject *offobj, *off_index; 647 648 if (f->f_fp == NULL) 649 return err_closed(); 650 drop_readahead(f); 651 whence = 0; 652 if (!PyArg_ParseTuple(args, "O|i:seek", &offobj, &whence)) 653 return NULL; 654 off_index = PyNumber_Index(offobj); 655 if (!off_index) { 656 if (!PyFloat_Check(offobj)) 657 return NULL; 658 /* Deprecated in 2.6 */ 659 PyErr_Clear(); 660 if (PyErr_WarnEx(PyExc_DeprecationWarning, 661 "integer argument expected, got float", 662 1) < 0) 663 return NULL; 664 off_index = offobj; 665 Py_INCREF(offobj); 666 } 667#if !defined(HAVE_LARGEFILE_SUPPORT) 668 offset = PyInt_AsLong(off_index); 669#else 670 offset = PyLong_Check(off_index) ? 671 PyLong_AsLongLong(off_index) : PyInt_AsLong(off_index); 672#endif 673 Py_DECREF(off_index); 674 if (PyErr_Occurred()) 675 return NULL; 676 677 FILE_BEGIN_ALLOW_THREADS(f) 678 errno = 0; 679 ret = _portable_fseek(f->f_fp, offset, whence); 680 FILE_END_ALLOW_THREADS(f) 681 682 if (ret != 0) { 683 PyErr_SetFromErrno(PyExc_IOError); 684 clearerr(f->f_fp); 685 return NULL; 686 } 687 f->f_skipnextlf = 0; 688 Py_INCREF(Py_None); 689 return Py_None; 690} 691 692 693#ifdef HAVE_FTRUNCATE 694static PyObject * 695file_truncate(PyFileObject *f, PyObject *args) 696{ 697 Py_off_t newsize; 698 PyObject *newsizeobj = NULL; 699 Py_off_t initialpos; 700 int ret; 701 702 if (f->f_fp == NULL) 703 return err_closed(); 704 if (!PyArg_UnpackTuple(args, "truncate", 0, 1, &newsizeobj)) 705 return NULL; 706 707 /* Get current file position. If the file happens to be open for 708 * update and the last operation was an input operation, C doesn't 709 * define what the later fflush() will do, but we promise truncate() 710 * won't change the current position (and fflush() *does* change it 711 * then at least on Windows). The easiest thing is to capture 712 * current pos now and seek back to it at the end. 713 */ 714 FILE_BEGIN_ALLOW_THREADS(f) 715 errno = 0; 716 initialpos = _portable_ftell(f->f_fp); 717 FILE_END_ALLOW_THREADS(f) 718 if (initialpos == -1) 719 goto onioerror; 720 721 /* Set newsize to current postion if newsizeobj NULL, else to the 722 * specified value. 723 */ 724 if (newsizeobj != NULL) { 725#if !defined(HAVE_LARGEFILE_SUPPORT) 726 newsize = PyInt_AsLong(newsizeobj); 727#else 728 newsize = PyLong_Check(newsizeobj) ? 729 PyLong_AsLongLong(newsizeobj) : 730 PyInt_AsLong(newsizeobj); 731#endif 732 if (PyErr_Occurred()) 733 return NULL; 734 } 735 else /* default to current position */ 736 newsize = initialpos; 737 738 /* Flush the stream. We're mixing stream-level I/O with lower-level 739 * I/O, and a flush may be necessary to synch both platform views 740 * of the current file state. 741 */ 742 FILE_BEGIN_ALLOW_THREADS(f) 743 errno = 0; 744 ret = fflush(f->f_fp); 745 FILE_END_ALLOW_THREADS(f) 746 if (ret != 0) 747 goto onioerror; 748 749#ifdef MS_WINDOWS 750 /* MS _chsize doesn't work if newsize doesn't fit in 32 bits, 751 so don't even try using it. */ 752 { 753 HANDLE hFile; 754 755 /* Have to move current pos to desired endpoint on Windows. */ 756 FILE_BEGIN_ALLOW_THREADS(f) 757 errno = 0; 758 ret = _portable_fseek(f->f_fp, newsize, SEEK_SET) != 0; 759 FILE_END_ALLOW_THREADS(f) 760 if (ret) 761 goto onioerror; 762 763 /* Truncate. Note that this may grow the file! */ 764 FILE_BEGIN_ALLOW_THREADS(f) 765 errno = 0; 766 hFile = (HANDLE)_get_osfhandle(fileno(f->f_fp)); 767 ret = hFile == (HANDLE)-1; 768 if (ret == 0) { 769 ret = SetEndOfFile(hFile) == 0; 770 if (ret) 771 errno = EACCES; 772 } 773 FILE_END_ALLOW_THREADS(f) 774 if (ret) 775 goto onioerror; 776 } 777#else 778 FILE_BEGIN_ALLOW_THREADS(f) 779 errno = 0; 780 ret = ftruncate(fileno(f->f_fp), newsize); 781 FILE_END_ALLOW_THREADS(f) 782 if (ret != 0) 783 goto onioerror; 784#endif /* !MS_WINDOWS */ 785 786 /* Restore original file position. */ 787 FILE_BEGIN_ALLOW_THREADS(f) 788 errno = 0; 789 ret = _portable_fseek(f->f_fp, initialpos, SEEK_SET) != 0; 790 FILE_END_ALLOW_THREADS(f) 791 if (ret) 792 goto onioerror; 793 794 Py_INCREF(Py_None); 795 return Py_None; 796 797onioerror: 798 PyErr_SetFromErrno(PyExc_IOError); 799 clearerr(f->f_fp); 800 return NULL; 801} 802#endif /* HAVE_FTRUNCATE */ 803 804static PyObject * 805file_tell(PyFileObject *f) 806{ 807 Py_off_t pos; 808 809 if (f->f_fp == NULL) 810 return err_closed(); 811 FILE_BEGIN_ALLOW_THREADS(f) 812 errno = 0; 813 pos = _portable_ftell(f->f_fp); 814 FILE_END_ALLOW_THREADS(f) 815 816 if (pos == -1) { 817 PyErr_SetFromErrno(PyExc_IOError); 818 clearerr(f->f_fp); 819 return NULL; 820 } 821 if (f->f_skipnextlf) { 822 int c; 823 c = GETC(f->f_fp); 824 if (c == '\n') { 825 f->f_newlinetypes |= NEWLINE_CRLF; 826 pos++; 827 f->f_skipnextlf = 0; 828 } else if (c != EOF) ungetc(c, f->f_fp); 829 } 830#if !defined(HAVE_LARGEFILE_SUPPORT) 831 return PyInt_FromLong(pos); 832#else 833 return PyLong_FromLongLong(pos); 834#endif 835} 836 837static PyObject * 838file_fileno(PyFileObject *f) 839{ 840 if (f->f_fp == NULL) 841 return err_closed(); 842 return PyInt_FromLong((long) fileno(f->f_fp)); 843} 844 845static PyObject * 846file_flush(PyFileObject *f) 847{ 848 int res; 849 850 if (f->f_fp == NULL) 851 return err_closed(); 852 FILE_BEGIN_ALLOW_THREADS(f) 853 errno = 0; 854 res = fflush(f->f_fp); 855 FILE_END_ALLOW_THREADS(f) 856 if (res != 0) { 857 PyErr_SetFromErrno(PyExc_IOError); 858 clearerr(f->f_fp); 859 return NULL; 860 } 861 Py_INCREF(Py_None); 862 return Py_None; 863} 864 865static PyObject * 866file_isatty(PyFileObject *f) 867{ 868 long res; 869 if (f->f_fp == NULL) 870 return err_closed(); 871 FILE_BEGIN_ALLOW_THREADS(f) 872 res = isatty((int)fileno(f->f_fp)); 873 FILE_END_ALLOW_THREADS(f) 874 return PyBool_FromLong(res); 875} 876 877 878#if BUFSIZ < 8192 879#define SMALLCHUNK 8192 880#else 881#define SMALLCHUNK BUFSIZ 882#endif 883 884#if SIZEOF_INT < 4 885#define BIGCHUNK (512 * 32) 886#else 887#define BIGCHUNK (512 * 1024) 888#endif 889 890static size_t 891new_buffersize(PyFileObject *f, size_t currentsize) 892{ 893#ifdef HAVE_FSTAT 894 off_t pos, end; 895 struct stat st; 896 if (fstat(fileno(f->f_fp), &st) == 0) { 897 end = st.st_size; 898 /* The following is not a bug: we really need to call lseek() 899 *and* ftell(). The reason is that some stdio libraries 900 mistakenly flush their buffer when ftell() is called and 901 the lseek() call it makes fails, thereby throwing away 902 data that cannot be recovered in any way. To avoid this, 903 we first test lseek(), and only call ftell() if lseek() 904 works. We can't use the lseek() value either, because we 905 need to take the amount of buffered data into account. 906 (Yet another reason why stdio stinks. :-) */ 907 pos = lseek(fileno(f->f_fp), 0L, SEEK_CUR); 908 if (pos >= 0) { 909 pos = ftell(f->f_fp); 910 } 911 if (pos < 0) 912 clearerr(f->f_fp); 913 if (end > pos && pos >= 0) 914 return currentsize + end - pos + 1; 915 /* Add 1 so if the file were to grow we'd notice. */ 916 } 917#endif 918 if (currentsize > SMALLCHUNK) { 919 /* Keep doubling until we reach BIGCHUNK; 920 then keep adding BIGCHUNK. */ 921 if (currentsize <= BIGCHUNK) 922 return currentsize + currentsize; 923 else 924 return currentsize + BIGCHUNK; 925 } 926 return currentsize + SMALLCHUNK; 927} 928 929#if defined(EWOULDBLOCK) && defined(EAGAIN) && EWOULDBLOCK != EAGAIN 930#define BLOCKED_ERRNO(x) ((x) == EWOULDBLOCK || (x) == EAGAIN) 931#else 932#ifdef EWOULDBLOCK 933#define BLOCKED_ERRNO(x) ((x) == EWOULDBLOCK) 934#else 935#ifdef EAGAIN 936#define BLOCKED_ERRNO(x) ((x) == EAGAIN) 937#else 938#define BLOCKED_ERRNO(x) 0 939#endif 940#endif 941#endif 942 943static PyObject * 944file_read(PyFileObject *f, PyObject *args) 945{ 946 long bytesrequested = -1; 947 size_t bytesread, buffersize, chunksize; 948 PyObject *v; 949 950 if (f->f_fp == NULL) 951 return err_closed(); 952 /* refuse to mix with f.next() */ 953 if (f->f_buf != NULL && 954 (f->f_bufend - f->f_bufptr) > 0 && 955 f->f_buf[0] != '\0') 956 return err_iterbuffered(); 957 if (!PyArg_ParseTuple(args, "|l:read", &bytesrequested)) 958 return NULL; 959 if (bytesrequested < 0) 960 buffersize = new_buffersize(f, (size_t)0); 961 else 962 buffersize = bytesrequested; 963 if (buffersize > PY_SSIZE_T_MAX) { 964 PyErr_SetString(PyExc_OverflowError, 965 "requested number of bytes is more than a Python string can hold"); 966 return NULL; 967 } 968 v = PyString_FromStringAndSize((char *)NULL, buffersize); 969 if (v == NULL) 970 return NULL; 971 bytesread = 0; 972 for (;;) { 973 FILE_BEGIN_ALLOW_THREADS(f) 974 errno = 0; 975 chunksize = Py_UniversalNewlineFread(BUF(v) + bytesread, 976 buffersize - bytesread, f->f_fp, (PyObject *)f); 977 FILE_END_ALLOW_THREADS(f) 978 if (chunksize == 0) { 979 if (!ferror(f->f_fp)) 980 break; 981 clearerr(f->f_fp); 982 /* When in non-blocking mode, data shouldn't 983 * be discarded if a blocking signal was 984 * received. That will also happen if 985 * chunksize != 0, but bytesread < buffersize. */ 986 if (bytesread > 0 && BLOCKED_ERRNO(errno)) 987 break; 988 PyErr_SetFromErrno(PyExc_IOError); 989 Py_DECREF(v); 990 return NULL; 991 } 992 bytesread += chunksize; 993 if (bytesread < buffersize) { 994 clearerr(f->f_fp); 995 break; 996 } 997 if (bytesrequested < 0) { 998 buffersize = new_buffersize(f, buffersize); 999 if (_PyString_Resize(&v, buffersize) < 0) 1000 return NULL; 1001 } else { 1002 /* Got what was requested. */ 1003 break; 1004 } 1005 } 1006 if (bytesread != buffersize) 1007 _PyString_Resize(&v, bytesread); 1008 return v; 1009} 1010 1011static PyObject * 1012file_readinto(PyFileObject *f, PyObject *args) 1013{ 1014 char *ptr; 1015 Py_ssize_t ntodo; 1016 Py_ssize_t ndone, nnow; 1017 Py_buffer pbuf; 1018 1019 if (f->f_fp == NULL) 1020 return err_closed(); 1021 /* refuse to mix with f.next() */ 1022 if (f->f_buf != NULL && 1023 (f->f_bufend - f->f_bufptr) > 0 && 1024 f->f_buf[0] != '\0') 1025 return err_iterbuffered(); 1026 if (!PyArg_ParseTuple(args, "w*", &pbuf)) 1027 return NULL; 1028 ptr = pbuf.buf; 1029 ntodo = pbuf.len; 1030 ndone = 0; 1031 while (ntodo > 0) { 1032 FILE_BEGIN_ALLOW_THREADS(f) 1033 errno = 0; 1034 nnow = Py_UniversalNewlineFread(ptr+ndone, ntodo, f->f_fp, 1035 (PyObject *)f); 1036 FILE_END_ALLOW_THREADS(f) 1037 if (nnow == 0) { 1038 if (!ferror(f->f_fp)) 1039 break; 1040 PyErr_SetFromErrno(PyExc_IOError); 1041 clearerr(f->f_fp); 1042 PyBuffer_Release(&pbuf); 1043 return NULL; 1044 } 1045 ndone += nnow; 1046 ntodo -= nnow; 1047 } 1048 PyBuffer_Release(&pbuf); 1049 return PyInt_FromSsize_t(ndone); 1050} 1051 1052/************************************************************************** 1053Routine to get next line using platform fgets(). 1054 1055Under MSVC 6: 1056 1057+ MS threadsafe getc is very slow (multiple layers of function calls before+ 1058 after each character, to lock+unlock the stream). 1059+ The stream-locking functions are MS-internal -- can't access them from user 1060 code. 1061+ There's nothing Tim could find in the MS C or platform SDK libraries that 1062 can worm around this. 1063+ MS fgets locks/unlocks only once per line; it's the only hook we have. 1064 1065So we use fgets for speed(!), despite that it's painful. 1066 1067MS realloc is also slow. 1068 1069Reports from other platforms on this method vs getc_unlocked (which MS doesn't 1070have): 1071 Linux a wash 1072 Solaris a wash 1073 Tru64 Unix getline_via_fgets significantly faster 1074 1075CAUTION: The C std isn't clear about this: in those cases where fgets 1076writes something into the buffer, can it write into any position beyond the 1077required trailing null byte? MSVC 6 fgets does not, and no platform is (yet) 1078known on which it does; and it would be a strange way to code fgets. Still, 1079getline_via_fgets may not work correctly if it does. The std test 1080test_bufio.py should fail if platform fgets() routinely writes beyond the 1081trailing null byte. #define DONT_USE_FGETS_IN_GETLINE to disable this code. 1082**************************************************************************/ 1083 1084/* Use this routine if told to, or by default on non-get_unlocked() 1085 * platforms unless told not to. Yikes! Let's spell that out: 1086 * On a platform with getc_unlocked(): 1087 * By default, use getc_unlocked(). 1088 * If you want to use fgets() instead, #define USE_FGETS_IN_GETLINE. 1089 * On a platform without getc_unlocked(): 1090 * By default, use fgets(). 1091 * If you don't want to use fgets(), #define DONT_USE_FGETS_IN_GETLINE. 1092 */ 1093#if !defined(USE_FGETS_IN_GETLINE) && !defined(HAVE_GETC_UNLOCKED) 1094#define USE_FGETS_IN_GETLINE 1095#endif 1096 1097#if defined(DONT_USE_FGETS_IN_GETLINE) && defined(USE_FGETS_IN_GETLINE) 1098#undef USE_FGETS_IN_GETLINE 1099#endif 1100 1101#ifdef USE_FGETS_IN_GETLINE 1102static PyObject* 1103getline_via_fgets(PyFileObject *f, FILE *fp) 1104{ 1105/* INITBUFSIZE is the maximum line length that lets us get away with the fast 1106 * no-realloc, one-fgets()-call path. Boosting it isn't free, because we have 1107 * to fill this much of the buffer with a known value in order to figure out 1108 * how much of the buffer fgets() overwrites. So if INITBUFSIZE is larger 1109 * than "most" lines, we waste time filling unused buffer slots. 100 is 1110 * surely adequate for most peoples' email archives, chewing over source code, 1111 * etc -- "regular old text files". 1112 * MAXBUFSIZE is the maximum line length that lets us get away with the less 1113 * fast (but still zippy) no-realloc, two-fgets()-call path. See above for 1114 * cautions about boosting that. 300 was chosen because the worst real-life 1115 * text-crunching job reported on Python-Dev was a mail-log crawler where over 1116 * half the lines were 254 chars. 1117 */ 1118#define INITBUFSIZE 100 1119#define MAXBUFSIZE 300 1120 char* p; /* temp */ 1121 char buf[MAXBUFSIZE]; 1122 PyObject* v; /* the string object result */ 1123 char* pvfree; /* address of next free slot */ 1124 char* pvend; /* address one beyond last free slot */ 1125 size_t nfree; /* # of free buffer slots; pvend-pvfree */ 1126 size_t total_v_size; /* total # of slots in buffer */ 1127 size_t increment; /* amount to increment the buffer */ 1128 size_t prev_v_size; 1129 1130 /* Optimize for normal case: avoid _PyString_Resize if at all 1131 * possible via first reading into stack buffer "buf". 1132 */ 1133 total_v_size = INITBUFSIZE; /* start small and pray */ 1134 pvfree = buf; 1135 for (;;) { 1136 FILE_BEGIN_ALLOW_THREADS(f) 1137 pvend = buf + total_v_size; 1138 nfree = pvend - pvfree; 1139 memset(pvfree, '\n', nfree); 1140 assert(nfree < INT_MAX); /* Should be atmost MAXBUFSIZE */ 1141 p = fgets(pvfree, (int)nfree, fp); 1142 FILE_END_ALLOW_THREADS(f) 1143 1144 if (p == NULL) { 1145 clearerr(fp); 1146 if (PyErr_CheckSignals()) 1147 return NULL; 1148 v = PyString_FromStringAndSize(buf, pvfree - buf); 1149 return v; 1150 } 1151 /* fgets read *something* */ 1152 p = memchr(pvfree, '\n', nfree); 1153 if (p != NULL) { 1154 /* Did the \n come from fgets or from us? 1155 * Since fgets stops at the first \n, and then writes 1156 * \0, if it's from fgets a \0 must be next. But if 1157 * that's so, it could not have come from us, since 1158 * the \n's we filled the buffer with have only more 1159 * \n's to the right. 1160 */ 1161 if (p+1 < pvend && *(p+1) == '\0') { 1162 /* It's from fgets: we win! In particular, 1163 * we haven't done any mallocs yet, and can 1164 * build the final result on the first try. 1165 */ 1166 ++p; /* include \n from fgets */ 1167 } 1168 else { 1169 /* Must be from us: fgets didn't fill the 1170 * buffer and didn't find a newline, so it 1171 * must be the last and newline-free line of 1172 * the file. 1173 */ 1174 assert(p > pvfree && *(p-1) == '\0'); 1175 --p; /* don't include \0 from fgets */ 1176 } 1177 v = PyString_FromStringAndSize(buf, p - buf); 1178 return v; 1179 } 1180 /* yuck: fgets overwrote all the newlines, i.e. the entire 1181 * buffer. So this line isn't over yet, or maybe it is but 1182 * we're exactly at EOF. If we haven't already, try using the 1183 * rest of the stack buffer. 1184 */ 1185 assert(*(pvend-1) == '\0'); 1186 if (pvfree == buf) { 1187 pvfree = pvend - 1; /* overwrite trailing null */ 1188 total_v_size = MAXBUFSIZE; 1189 } 1190 else 1191 break; 1192 } 1193 1194 /* The stack buffer isn't big enough; malloc a string object and read 1195 * into its buffer. 1196 */ 1197 total_v_size = MAXBUFSIZE << 1; 1198 v = PyString_FromStringAndSize((char*)NULL, (int)total_v_size); 1199 if (v == NULL) 1200 return v; 1201 /* copy over everything except the last null byte */ 1202 memcpy(BUF(v), buf, MAXBUFSIZE-1); 1203 pvfree = BUF(v) + MAXBUFSIZE - 1; 1204 1205 /* Keep reading stuff into v; if it ever ends successfully, break 1206 * after setting p one beyond the end of the line. The code here is 1207 * very much like the code above, except reads into v's buffer; see 1208 * the code above for detailed comments about the logic. 1209 */ 1210 for (;;) { 1211 FILE_BEGIN_ALLOW_THREADS(f) 1212 pvend = BUF(v) + total_v_size; 1213 nfree = pvend - pvfree; 1214 memset(pvfree, '\n', nfree); 1215 assert(nfree < INT_MAX); 1216 p = fgets(pvfree, (int)nfree, fp); 1217 FILE_END_ALLOW_THREADS(f) 1218 1219 if (p == NULL) { 1220 clearerr(fp); 1221 if (PyErr_CheckSignals()) { 1222 Py_DECREF(v); 1223 return NULL; 1224 } 1225 p = pvfree; 1226 break; 1227 } 1228 p = memchr(pvfree, '\n', nfree); 1229 if (p != NULL) { 1230 if (p+1 < pvend && *(p+1) == '\0') { 1231 /* \n came from fgets */ 1232 ++p; 1233 break; 1234 } 1235 /* \n came from us; last line of file, no newline */ 1236 assert(p > pvfree && *(p-1) == '\0'); 1237 --p; 1238 break; 1239 } 1240 /* expand buffer and try again */ 1241 assert(*(pvend-1) == '\0'); 1242 increment = total_v_size >> 2; /* mild exponential growth */ 1243 prev_v_size = total_v_size; 1244 total_v_size += increment; 1245 /* check for overflow */ 1246 if (total_v_size <= prev_v_size || 1247 total_v_size > PY_SSIZE_T_MAX) { 1248 PyErr_SetString(PyExc_OverflowError, 1249 "line is longer than a Python string can hold"); 1250 Py_DECREF(v); 1251 return NULL; 1252 } 1253 if (_PyString_Resize(&v, (int)total_v_size) < 0) 1254 return NULL; 1255 /* overwrite the trailing null byte */ 1256 pvfree = BUF(v) + (prev_v_size - 1); 1257 } 1258 if (BUF(v) + total_v_size != p) 1259 _PyString_Resize(&v, p - BUF(v)); 1260 return v; 1261#undef INITBUFSIZE 1262#undef MAXBUFSIZE 1263} 1264#endif /* ifdef USE_FGETS_IN_GETLINE */ 1265 1266/* Internal routine to get a line. 1267 Size argument interpretation: 1268 > 0: max length; 1269 <= 0: read arbitrary line 1270*/ 1271 1272static PyObject * 1273get_line(PyFileObject *f, int n) 1274{ 1275 FILE *fp = f->f_fp; 1276 int c; 1277 char *buf, *end; 1278 size_t total_v_size; /* total # of slots in buffer */ 1279 size_t used_v_size; /* # used slots in buffer */ 1280 size_t increment; /* amount to increment the buffer */ 1281 PyObject *v; 1282 int newlinetypes = f->f_newlinetypes; 1283 int skipnextlf = f->f_skipnextlf; 1284 int univ_newline = f->f_univ_newline; 1285 1286#if defined(USE_FGETS_IN_GETLINE) 1287 if (n <= 0 && !univ_newline ) 1288 return getline_via_fgets(f, fp); 1289#endif 1290 total_v_size = n > 0 ? n : 100; 1291 v = PyString_FromStringAndSize((char *)NULL, total_v_size); 1292 if (v == NULL) 1293 return NULL; 1294 buf = BUF(v); 1295 end = buf + total_v_size; 1296 1297 for (;;) { 1298 FILE_BEGIN_ALLOW_THREADS(f) 1299 FLOCKFILE(fp); 1300 if (univ_newline) { 1301 c = 'x'; /* Shut up gcc warning */ 1302 while ( buf != end && (c = GETC(fp)) != EOF ) { 1303 if (skipnextlf ) { 1304 skipnextlf = 0; 1305 if (c == '\n') { 1306 /* Seeing a \n here with 1307 * skipnextlf true means we 1308 * saw a \r before. 1309 */ 1310 newlinetypes |= NEWLINE_CRLF; 1311 c = GETC(fp); 1312 if (c == EOF) break; 1313 } else { 1314 newlinetypes |= NEWLINE_CR; 1315 } 1316 } 1317 if (c == '\r') { 1318 skipnextlf = 1; 1319 c = '\n'; 1320 } else if ( c == '\n') 1321 newlinetypes |= NEWLINE_LF; 1322 *buf++ = c; 1323 if (c == '\n') break; 1324 } 1325 if ( c == EOF && skipnextlf ) 1326 newlinetypes |= NEWLINE_CR; 1327 } else /* If not universal newlines use the normal loop */ 1328 while ((c = GETC(fp)) != EOF && 1329 (*buf++ = c) != '\n' && 1330 buf != end) 1331 ; 1332 FUNLOCKFILE(fp); 1333 FILE_END_ALLOW_THREADS(f) 1334 f->f_newlinetypes = newlinetypes; 1335 f->f_skipnextlf = skipnextlf; 1336 if (c == '\n') 1337 break; 1338 if (c == EOF) { 1339 if (ferror(fp)) { 1340 PyErr_SetFromErrno(PyExc_IOError); 1341 clearerr(fp); 1342 Py_DECREF(v); 1343 return NULL; 1344 } 1345 clearerr(fp); 1346 if (PyErr_CheckSignals()) { 1347 Py_DECREF(v); 1348 return NULL; 1349 } 1350 break; 1351 } 1352 /* Must be because buf == end */ 1353 if (n > 0) 1354 break; 1355 used_v_size = total_v_size; 1356 increment = total_v_size >> 2; /* mild exponential growth */ 1357 total_v_size += increment; 1358 if (total_v_size > PY_SSIZE_T_MAX) { 1359 PyErr_SetString(PyExc_OverflowError, 1360 "line is longer than a Python string can hold"); 1361 Py_DECREF(v); 1362 return NULL; 1363 } 1364 if (_PyString_Resize(&v, total_v_size) < 0) 1365 return NULL; 1366 buf = BUF(v) + used_v_size; 1367 end = BUF(v) + total_v_size; 1368 } 1369 1370 used_v_size = buf - BUF(v); 1371 if (used_v_size != total_v_size) 1372 _PyString_Resize(&v, used_v_size); 1373 return v; 1374} 1375 1376/* External C interface */ 1377 1378PyObject * 1379PyFile_GetLine(PyObject *f, int n) 1380{ 1381 PyObject *result; 1382 1383 if (f == NULL) { 1384 PyErr_BadInternalCall(); 1385 return NULL; 1386 } 1387 1388 if (PyFile_Check(f)) { 1389 PyFileObject *fo = (PyFileObject *)f; 1390 if (fo->f_fp == NULL) 1391 return err_closed(); 1392 /* refuse to mix with f.next() */ 1393 if (fo->f_buf != NULL && 1394 (fo->f_bufend - fo->f_bufptr) > 0 && 1395 fo->f_buf[0] != '\0') 1396 return err_iterbuffered(); 1397 result = get_line(fo, n); 1398 } 1399 else { 1400 PyObject *reader; 1401 PyObject *args; 1402 1403 reader = PyObject_GetAttrString(f, "readline"); 1404 if (reader == NULL) 1405 return NULL; 1406 if (n <= 0) 1407 args = PyTuple_New(0); 1408 else 1409 args = Py_BuildValue("(i)", n); 1410 if (args == NULL) { 1411 Py_DECREF(reader); 1412 return NULL; 1413 } 1414 result = PyEval_CallObject(reader, args); 1415 Py_DECREF(reader); 1416 Py_DECREF(args); 1417 if (result != NULL && !PyString_Check(result) && 1418 !PyUnicode_Check(result)) { 1419 Py_DECREF(result); 1420 result = NULL; 1421 PyErr_SetString(PyExc_TypeError, 1422 "object.readline() returned non-string"); 1423 } 1424 } 1425 1426 if (n < 0 && result != NULL && PyString_Check(result)) { 1427 char *s = PyString_AS_STRING(result); 1428 Py_ssize_t len = PyString_GET_SIZE(result); 1429 if (len == 0) { 1430 Py_DECREF(result); 1431 result = NULL; 1432 PyErr_SetString(PyExc_EOFError, 1433 "EOF when reading a line"); 1434 } 1435 else if (s[len-1] == '\n') { 1436 if (result->ob_refcnt == 1) 1437 _PyString_Resize(&result, len-1); 1438 else { 1439 PyObject *v; 1440 v = PyString_FromStringAndSize(s, len-1); 1441 Py_DECREF(result); 1442 result = v; 1443 } 1444 } 1445 } 1446#ifdef Py_USING_UNICODE 1447 if (n < 0 && result != NULL && PyUnicode_Check(result)) { 1448 Py_UNICODE *s = PyUnicode_AS_UNICODE(result); 1449 Py_ssize_t len = PyUnicode_GET_SIZE(result); 1450 if (len == 0) { 1451 Py_DECREF(result); 1452 result = NULL; 1453 PyErr_SetString(PyExc_EOFError, 1454 "EOF when reading a line"); 1455 } 1456 else if (s[len-1] == '\n') { 1457 if (result->ob_refcnt == 1) 1458 PyUnicode_Resize(&result, len-1); 1459 else { 1460 PyObject *v; 1461 v = PyUnicode_FromUnicode(s, len-1); 1462 Py_DECREF(result); 1463 result = v; 1464 } 1465 } 1466 } 1467#endif 1468 return result; 1469} 1470 1471/* Python method */ 1472 1473static PyObject * 1474file_readline(PyFileObject *f, PyObject *args) 1475{ 1476 int n = -1; 1477 1478 if (f->f_fp == NULL) 1479 return err_closed(); 1480 /* refuse to mix with f.next() */ 1481 if (f->f_buf != NULL && 1482 (f->f_bufend - f->f_bufptr) > 0 && 1483 f->f_buf[0] != '\0') 1484 return err_iterbuffered(); 1485 if (!PyArg_ParseTuple(args, "|i:readline", &n)) 1486 return NULL; 1487 if (n == 0) 1488 return PyString_FromString(""); 1489 if (n < 0) 1490 n = 0; 1491 return get_line(f, n); 1492} 1493 1494static PyObject * 1495file_readlines(PyFileObject *f, PyObject *args) 1496{ 1497 long sizehint = 0; 1498 PyObject *list = NULL; 1499 PyObject *line; 1500 char small_buffer[SMALLCHUNK]; 1501 char *buffer = small_buffer; 1502 size_t buffersize = SMALLCHUNK; 1503 PyObject *big_buffer = NULL; 1504 size_t nfilled = 0; 1505 size_t nread; 1506 size_t totalread = 0; 1507 char *p, *q, *end; 1508 int err; 1509 int shortread = 0; 1510 1511 if (f->f_fp == NULL) 1512 return err_closed(); 1513 /* refuse to mix with f.next() */ 1514 if (f->f_buf != NULL && 1515 (f->f_bufend - f->f_bufptr) > 0 && 1516 f->f_buf[0] != '\0') 1517 return err_iterbuffered(); 1518 if (!PyArg_ParseTuple(args, "|l:readlines", &sizehint)) 1519 return NULL; 1520 if ((list = PyList_New(0)) == NULL) 1521 return NULL; 1522 for (;;) { 1523 if (shortread) 1524 nread = 0; 1525 else { 1526 FILE_BEGIN_ALLOW_THREADS(f) 1527 errno = 0; 1528 nread = Py_UniversalNewlineFread(buffer+nfilled, 1529 buffersize-nfilled, f->f_fp, (PyObject *)f); 1530 FILE_END_ALLOW_THREADS(f) 1531 shortread = (nread < buffersize-nfilled); 1532 } 1533 if (nread == 0) { 1534 sizehint = 0; 1535 if (!ferror(f->f_fp)) 1536 break; 1537 PyErr_SetFromErrno(PyExc_IOError); 1538 clearerr(f->f_fp); 1539 goto error; 1540 } 1541 totalread += nread; 1542 p = (char *)memchr(buffer+nfilled, '\n', nread); 1543 if (p == NULL) { 1544 /* Need a larger buffer to fit this line */ 1545 nfilled += nread; 1546 buffersize *= 2; 1547 if (buffersize > PY_SSIZE_T_MAX) { 1548 PyErr_SetString(PyExc_OverflowError, 1549 "line is longer than a Python string can hold"); 1550 goto error; 1551 } 1552 if (big_buffer == NULL) { 1553 /* Create the big buffer */ 1554 big_buffer = PyString_FromStringAndSize( 1555 NULL, buffersize); 1556 if (big_buffer == NULL) 1557 goto error; 1558 buffer = PyString_AS_STRING(big_buffer); 1559 memcpy(buffer, small_buffer, nfilled); 1560 } 1561 else { 1562 /* Grow the big buffer */ 1563 if ( _PyString_Resize(&big_buffer, buffersize) < 0 ) 1564 goto error; 1565 buffer = PyString_AS_STRING(big_buffer); 1566 } 1567 continue; 1568 } 1569 end = buffer+nfilled+nread; 1570 q = buffer; 1571 do { 1572 /* Process complete lines */ 1573 p++; 1574 line = PyString_FromStringAndSize(q, p-q); 1575 if (line == NULL) 1576 goto error; 1577 err = PyList_Append(list, line); 1578 Py_DECREF(line); 1579 if (err != 0) 1580 goto error; 1581 q = p; 1582 p = (char *)memchr(q, '\n', end-q); 1583 } while (p != NULL); 1584 /* Move the remaining incomplete line to the start */ 1585 nfilled = end-q; 1586 memmove(buffer, q, nfilled); 1587 if (sizehint > 0) 1588 if (totalread >= (size_t)sizehint) 1589 break; 1590 } 1591 if (nfilled != 0) { 1592 /* Partial last line */ 1593 line = PyString_FromStringAndSize(buffer, nfilled); 1594 if (line == NULL) 1595 goto error; 1596 if (sizehint > 0) { 1597 /* Need to complete the last line */ 1598 PyObject *rest = get_line(f, 0); 1599 if (rest == NULL) { 1600 Py_DECREF(line); 1601 goto error; 1602 } 1603 PyString_Concat(&line, rest); 1604 Py_DECREF(rest); 1605 if (line == NULL) 1606 goto error; 1607 } 1608 err = PyList_Append(list, line); 1609 Py_DECREF(line); 1610 if (err != 0) 1611 goto error; 1612 } 1613 1614cleanup: 1615 Py_XDECREF(big_buffer); 1616 return list; 1617 1618error: 1619 Py_CLEAR(list); 1620 goto cleanup; 1621} 1622 1623static PyObject * 1624file_write(PyFileObject *f, PyObject *args) 1625{ 1626 Py_buffer pbuf; 1627 char *s; 1628 Py_ssize_t n, n2; 1629 if (f->f_fp == NULL) 1630 return err_closed(); 1631 if (f->f_binary) { 1632 if (!PyArg_ParseTuple(args, "s*", &pbuf)) 1633 return NULL; 1634 s = pbuf.buf; 1635 n = pbuf.len; 1636 } else 1637 if (!PyArg_ParseTuple(args, "t#", &s, &n)) 1638 return NULL; 1639 f->f_softspace = 0; 1640 FILE_BEGIN_ALLOW_THREADS(f) 1641 errno = 0; 1642 n2 = fwrite(s, 1, n, f->f_fp); 1643 FILE_END_ALLOW_THREADS(f) 1644 if (f->f_binary) 1645 PyBuffer_Release(&pbuf); 1646 if (n2 != n) { 1647 PyErr_SetFromErrno(PyExc_IOError); 1648 clearerr(f->f_fp); 1649 return NULL; 1650 } 1651 Py_INCREF(Py_None); 1652 return Py_None; 1653} 1654 1655static PyObject * 1656file_writelines(PyFileObject *f, PyObject *seq) 1657{ 1658#define CHUNKSIZE 1000 1659 PyObject *list, *line; 1660 PyObject *it; /* iter(seq) */ 1661 PyObject *result; 1662 int index, islist; 1663 Py_ssize_t i, j, nwritten, len; 1664 1665 assert(seq != NULL); 1666 if (f->f_fp == NULL) 1667 return err_closed(); 1668 1669 result = NULL; 1670 list = NULL; 1671 islist = PyList_Check(seq); 1672 if (islist) 1673 it = NULL; 1674 else { 1675 it = PyObject_GetIter(seq); 1676 if (it == NULL) { 1677 PyErr_SetString(PyExc_TypeError, 1678 "writelines() requires an iterable argument"); 1679 return NULL; 1680 } 1681 /* From here on, fail by going to error, to reclaim "it". */ 1682 list = PyList_New(CHUNKSIZE); 1683 if (list == NULL) 1684 goto error; 1685 } 1686 1687 /* Strategy: slurp CHUNKSIZE lines into a private list, 1688 checking that they are all strings, then write that list 1689 without holding the interpreter lock, then come back for more. */ 1690 for (index = 0; ; index += CHUNKSIZE) { 1691 if (islist) { 1692 Py_XDECREF(list); 1693 list = PyList_GetSlice(seq, index, index+CHUNKSIZE); 1694 if (list == NULL) 1695 goto error; 1696 j = PyList_GET_SIZE(list); 1697 } 1698 else { 1699 for (j = 0; j < CHUNKSIZE; j++) { 1700 line = PyIter_Next(it); 1701 if (line == NULL) { 1702 if (PyErr_Occurred()) 1703 goto error; 1704 break; 1705 } 1706 PyList_SetItem(list, j, line); 1707 } 1708 } 1709 if (j == 0) 1710 break; 1711 1712 /* Check that all entries are indeed strings. If not, 1713 apply the same rules as for file.write() and 1714 convert the results to strings. This is slow, but 1715 seems to be the only way since all conversion APIs 1716 could potentially execute Python code. */ 1717 for (i = 0; i < j; i++) { 1718 PyObject *v = PyList_GET_ITEM(list, i); 1719 if (!PyString_Check(v)) { 1720 const char *buffer; 1721 if (((f->f_binary && 1722 PyObject_AsReadBuffer(v, 1723 (const void**)&buffer, 1724 &len)) || 1725 PyObject_AsCharBuffer(v, 1726 &buffer, 1727 &len))) { 1728 PyErr_SetString(PyExc_TypeError, 1729 "writelines() argument must be a sequence of strings"); 1730 goto error; 1731 } 1732 line = PyString_FromStringAndSize(buffer, 1733 len); 1734 if (line == NULL) 1735 goto error; 1736 Py_DECREF(v); 1737 PyList_SET_ITEM(list, i, line); 1738 } 1739 } 1740 1741 /* Since we are releasing the global lock, the 1742 following code may *not* execute Python code. */ 1743 f->f_softspace = 0; 1744 FILE_BEGIN_ALLOW_THREADS(f) 1745 errno = 0; 1746 for (i = 0; i < j; i++) { 1747 line = PyList_GET_ITEM(list, i); 1748 len = PyString_GET_SIZE(line); 1749 nwritten = fwrite(PyString_AS_STRING(line), 1750 1, len, f->f_fp); 1751 if (nwritten != len) { 1752 FILE_ABORT_ALLOW_THREADS(f) 1753 PyErr_SetFromErrno(PyExc_IOError); 1754 clearerr(f->f_fp); 1755 goto error; 1756 } 1757 } 1758 FILE_END_ALLOW_THREADS(f) 1759 1760 if (j < CHUNKSIZE) 1761 break; 1762 } 1763 1764 Py_INCREF(Py_None); 1765 result = Py_None; 1766 error: 1767 Py_XDECREF(list); 1768 Py_XDECREF(it); 1769 return result; 1770#undef CHUNKSIZE 1771} 1772 1773static PyObject * 1774file_self(PyFileObject *f) 1775{ 1776 if (f->f_fp == NULL) 1777 return err_closed(); 1778 Py_INCREF(f); 1779 return (PyObject *)f; 1780} 1781 1782static PyObject * 1783file_xreadlines(PyFileObject *f) 1784{ 1785 if (PyErr_WarnPy3k("f.xreadlines() not supported in 3.x, " 1786 "try 'for line in f' instead", 1) < 0) 1787 return NULL; 1788 return file_self(f); 1789} 1790 1791static PyObject * 1792file_exit(PyObject *f, PyObject *args) 1793{ 1794 PyObject *ret = PyObject_CallMethod(f, "close", NULL); 1795 if (!ret) 1796 /* If error occurred, pass through */ 1797 return NULL; 1798 Py_DECREF(ret); 1799 /* We cannot return the result of close since a true 1800 * value will be interpreted as "yes, swallow the 1801 * exception if one was raised inside the with block". */ 1802 Py_RETURN_NONE; 1803} 1804 1805PyDoc_STRVAR(readline_doc, 1806"readline([size]) -> next line from the file, as a string.\n" 1807"\n" 1808"Retain newline. A non-negative size argument limits the maximum\n" 1809"number of bytes to return (an incomplete line may be returned then).\n" 1810"Return an empty string at EOF."); 1811 1812PyDoc_STRVAR(read_doc, 1813"read([size]) -> read at most size bytes, returned as a string.\n" 1814"\n" 1815"If the size argument is negative or omitted, read until EOF is reached.\n" 1816"Notice that when in non-blocking mode, less data than what was requested\n" 1817"may be returned, even if no size parameter was given."); 1818 1819PyDoc_STRVAR(write_doc, 1820"write(str) -> None. Write string str to file.\n" 1821"\n" 1822"Note that due to buffering, flush() or close() may be needed before\n" 1823"the file on disk reflects the data written."); 1824 1825PyDoc_STRVAR(fileno_doc, 1826"fileno() -> integer \"file descriptor\".\n" 1827"\n" 1828"This is needed for lower-level file interfaces, such os.read()."); 1829 1830PyDoc_STRVAR(seek_doc, 1831"seek(offset[, whence]) -> None. Move to new file position.\n" 1832"\n" 1833"Argument offset is a byte count. Optional argument whence defaults to\n" 1834"0 (offset from start of file, offset should be >= 0); other values are 1\n" 1835"(move relative to current position, positive or negative), and 2 (move\n" 1836"relative to end of file, usually negative, although many platforms allow\n" 1837"seeking beyond the end of a file). If the file is opened in text mode,\n" 1838"only offsets returned by tell() are legal. Use of other offsets causes\n" 1839"undefined behavior." 1840"\n" 1841"Note that not all file objects are seekable."); 1842 1843#ifdef HAVE_FTRUNCATE 1844PyDoc_STRVAR(truncate_doc, 1845"truncate([size]) -> None. Truncate the file to at most size bytes.\n" 1846"\n" 1847"Size defaults to the current file position, as returned by tell()."); 1848#endif 1849 1850PyDoc_STRVAR(tell_doc, 1851"tell() -> current file position, an integer (may be a long integer)."); 1852 1853PyDoc_STRVAR(readinto_doc, 1854"readinto() -> Undocumented. Don't use this; it may go away."); 1855 1856PyDoc_STRVAR(readlines_doc, 1857"readlines([size]) -> list of strings, each a line from the file.\n" 1858"\n" 1859"Call readline() repeatedly and return a list of the lines so read.\n" 1860"The optional size argument, if given, is an approximate bound on the\n" 1861"total number of bytes in the lines returned."); 1862 1863PyDoc_STRVAR(xreadlines_doc, 1864"xreadlines() -> returns self.\n" 1865"\n" 1866"For backward compatibility. File objects now include the performance\n" 1867"optimizations previously implemented in the xreadlines module."); 1868 1869PyDoc_STRVAR(writelines_doc, 1870"writelines(sequence_of_strings) -> None. Write the strings to the file.\n" 1871"\n" 1872"Note that newlines are not added. The sequence can be any iterable object\n" 1873"producing strings. This is equivalent to calling write() for each string."); 1874 1875PyDoc_STRVAR(flush_doc, 1876"flush() -> None. Flush the internal I/O buffer."); 1877 1878PyDoc_STRVAR(close_doc, 1879"close() -> None or (perhaps) an integer. Close the file.\n" 1880"\n" 1881"Sets data attribute .closed to True. A closed file cannot be used for\n" 1882"further I/O operations. close() may be called more than once without\n" 1883"error. Some kinds of file objects (for example, opened by popen())\n" 1884"may return an exit status upon closing."); 1885 1886PyDoc_STRVAR(isatty_doc, 1887"isatty() -> true or false. True if the file is connected to a tty device."); 1888 1889PyDoc_STRVAR(enter_doc, 1890 "__enter__() -> self."); 1891 1892PyDoc_STRVAR(exit_doc, 1893 "__exit__(*excinfo) -> None. Closes the file."); 1894 1895static PyMethodDef file_methods[] = { 1896 {"readline", (PyCFunction)file_readline, METH_VARARGS, readline_doc}, 1897 {"read", (PyCFunction)file_read, METH_VARARGS, read_doc}, 1898 {"write", (PyCFunction)file_write, METH_VARARGS, write_doc}, 1899 {"fileno", (PyCFunction)file_fileno, METH_NOARGS, fileno_doc}, 1900 {"seek", (PyCFunction)file_seek, METH_VARARGS, seek_doc}, 1901#ifdef HAVE_FTRUNCATE 1902 {"truncate", (PyCFunction)file_truncate, METH_VARARGS, truncate_doc}, 1903#endif 1904 {"tell", (PyCFunction)file_tell, METH_NOARGS, tell_doc}, 1905 {"readinto", (PyCFunction)file_readinto, METH_VARARGS, readinto_doc}, 1906 {"readlines", (PyCFunction)file_readlines, METH_VARARGS, readlines_doc}, 1907 {"xreadlines",(PyCFunction)file_xreadlines, METH_NOARGS, xreadlines_doc}, 1908 {"writelines",(PyCFunction)file_writelines, METH_O, writelines_doc}, 1909 {"flush", (PyCFunction)file_flush, METH_NOARGS, flush_doc}, 1910 {"close", (PyCFunction)file_close, METH_NOARGS, close_doc}, 1911 {"isatty", (PyCFunction)file_isatty, METH_NOARGS, isatty_doc}, 1912 {"__enter__", (PyCFunction)file_self, METH_NOARGS, enter_doc}, 1913 {"__exit__", (PyCFunction)file_exit, METH_VARARGS, exit_doc}, 1914 {NULL, NULL} /* sentinel */ 1915}; 1916 1917#define OFF(x) offsetof(PyFileObject, x) 1918 1919static PyMemberDef file_memberlist[] = { 1920 {"mode", T_OBJECT, OFF(f_mode), RO, 1921 "file mode ('r', 'U', 'w', 'a', possibly with 'b' or '+' added)"}, 1922 {"name", T_OBJECT, OFF(f_name), RO, 1923 "file name"}, 1924 {"encoding", T_OBJECT, OFF(f_encoding), RO, 1925 "file encoding"}, 1926 {"errors", T_OBJECT, OFF(f_errors), RO, 1927 "Unicode error handler"}, 1928 /* getattr(f, "closed") is implemented without this table */ 1929 {NULL} /* Sentinel */ 1930}; 1931 1932static PyObject * 1933get_closed(PyFileObject *f, void *closure) 1934{ 1935 return PyBool_FromLong((long)(f->f_fp == 0)); 1936} 1937static PyObject * 1938get_newlines(PyFileObject *f, void *closure) 1939{ 1940 switch (f->f_newlinetypes) { 1941 case NEWLINE_UNKNOWN: 1942 Py_INCREF(Py_None); 1943 return Py_None; 1944 case NEWLINE_CR: 1945 return PyString_FromString("\r"); 1946 case N…
Large files files are truncated, but you can click here to view the full file