/External.LCA_RESTRICTED/Languages/IronPython/27/Lib/_pyio.py
Python | 2037 lines | 1937 code | 46 blank | 54 comment | 82 complexity | 371c7fa6cc7fb206d779cce337d3bdc8 MD5 | raw file
Possible License(s): CPL-1.0, BSD-3-Clause, ISC, GPL-2.0, MPL-2.0-no-copyleft-exception
Large files files are truncated, but you can click here to view the full file
- """
- Python implementation of the io module.
- """
- from __future__ import (print_function, unicode_literals)
- import os
- import abc
- import codecs
- import sys
- import warnings
- import errno
- # Import thread instead of threading to reduce startup cost
- try:
- from thread import allocate_lock as Lock
- except ImportError:
- from dummy_thread import allocate_lock as Lock
- import io
- from io import (__all__, SEEK_SET, SEEK_CUR, SEEK_END)
- from errno import EINTR
- __metaclass__ = type
- # open() uses st_blksize whenever we can
- DEFAULT_BUFFER_SIZE = 8 * 1024 # bytes
- # NOTE: Base classes defined here are registered with the "official" ABCs
- # defined in io.py. We don't use real inheritance though, because we don't want
- # to inherit the C implementations.
- class BlockingIOError(IOError):
- """Exception raised when I/O would block on a non-blocking I/O stream."""
- def __init__(self, errno, strerror, characters_written=0):
- super(IOError, self).__init__(errno, strerror)
- if not isinstance(characters_written, (int, long)):
- raise TypeError("characters_written must be a integer")
- self.characters_written = characters_written
- def open(file, mode="r", buffering=-1,
- encoding=None, errors=None,
- newline=None, closefd=True):
- r"""Open file and return a stream. Raise IOError upon failure.
- file is either a text or byte string giving the name (and the path
- if the file isn't in the current working directory) of the file to
- be opened or an integer file descriptor of the file to be
- wrapped. (If a file descriptor is given, it is closed when the
- returned I/O object is closed, unless closefd is set to False.)
- mode is an optional string that specifies the mode in which the file
- is opened. It defaults to 'r' which means open for reading in text
- mode. Other common values are 'w' for writing (truncating the file if
- it already exists), and 'a' for appending (which on some Unix systems,
- means that all writes append to the end of the file regardless of the
- current seek position). In text mode, if encoding is not specified the
- encoding used is platform dependent. (For reading and writing raw
- bytes use binary mode and leave encoding unspecified.) The available
- modes are:
- ========= ===============================================================
- Character Meaning
- --------- ---------------------------------------------------------------
- 'r' open for reading (default)
- 'w' open for writing, truncating the file first
- 'a' open for writing, appending to the end of the file if it exists
- 'b' binary mode
- 't' text mode (default)
- '+' open a disk file for updating (reading and writing)
- 'U' universal newline mode (for backwards compatibility; unneeded
- for new code)
- ========= ===============================================================
- The default mode is 'rt' (open for reading text). For binary random
- access, the mode 'w+b' opens and truncates the file to 0 bytes, while
- 'r+b' opens the file without truncation.
- Python distinguishes between files opened in binary and text modes,
- even when the underlying operating system doesn't. Files opened in
- binary mode (appending 'b' to the mode argument) return contents as
- bytes objects without any decoding. In text mode (the default, or when
- 't' is appended to the mode argument), the contents of the file are
- returned as strings, the bytes having been first decoded using a
- platform-dependent encoding or using the specified encoding if given.
- buffering is an optional integer used to set the buffering policy.
- Pass 0 to switch buffering off (only allowed in binary mode), 1 to select
- line buffering (only usable in text mode), and an integer > 1 to indicate
- the size of a fixed-size chunk buffer. When no buffering argument is
- given, the default buffering policy works as follows:
- * Binary files are buffered in fixed-size chunks; the size of the buffer
- is chosen using a heuristic trying to determine the underlying device's
- "block size" and falling back on `io.DEFAULT_BUFFER_SIZE`.
- On many systems, the buffer will typically be 4096 or 8192 bytes long.
- * "Interactive" text files (files for which isatty() returns True)
- use line buffering. Other text files use the policy described above
- for binary files.
- encoding is the name of the encoding used to decode or encode the
- file. This should only be used in text mode. The default encoding is
- platform dependent, but any encoding supported by Python can be
- passed. See the codecs module for the list of supported encodings.
- errors is an optional string that specifies how encoding errors are to
- be handled---this argument should not be used in binary mode. Pass
- 'strict' to raise a ValueError exception if there is an encoding error
- (the default of None has the same effect), or pass 'ignore' to ignore
- errors. (Note that ignoring encoding errors can lead to data loss.)
- See the documentation for codecs.register for a list of the permitted
- encoding error strings.
- newline controls how universal newlines works (it only applies to text
- mode). It can be None, '', '\n', '\r', and '\r\n'. It works as
- follows:
- * On input, if newline is None, universal newlines mode is
- enabled. Lines in the input can end in '\n', '\r', or '\r\n', and
- these are translated into '\n' before being returned to the
- caller. If it is '', universal newline mode is enabled, but line
- endings are returned to the caller untranslated. If it has any of
- the other legal values, input lines are only terminated by the given
- string, and the line ending is returned to the caller untranslated.
- * On output, if newline is None, any '\n' characters written are
- translated to the system default line separator, os.linesep. If
- newline is '', no translation takes place. If newline is any of the
- other legal values, any '\n' characters written are translated to
- the given string.
- If closefd is False, the underlying file descriptor will be kept open
- when the file is closed. This does not work when a file name is given
- and must be True in that case.
- open() returns a file object whose type depends on the mode, and
- through which the standard file operations such as reading and writing
- are performed. When open() is used to open a file in a text mode ('w',
- 'r', 'wt', 'rt', etc.), it returns a TextIOWrapper. When used to open
- a file in a binary mode, the returned class varies: in read binary
- mode, it returns a BufferedReader; in write binary and append binary
- modes, it returns a BufferedWriter, and in read/write mode, it returns
- a BufferedRandom.
- It is also possible to use a string or bytearray as a file for both
- reading and writing. For strings StringIO can be used like a file
- opened in a text mode, and for bytes a BytesIO can be used like a file
- opened in a binary mode.
- """
- if not isinstance(file, (basestring, int, long)):
- raise TypeError("invalid file: %r" % file)
- if not isinstance(mode, basestring):
- raise TypeError("invalid mode: %r" % mode)
- if not isinstance(buffering, (int, long)):
- raise TypeError("invalid buffering: %r" % buffering)
- if encoding is not None and not isinstance(encoding, basestring):
- raise TypeError("invalid encoding: %r" % encoding)
- if errors is not None and not isinstance(errors, basestring):
- raise TypeError("invalid errors: %r" % errors)
- modes = set(mode)
- if modes - set("arwb+tU") or len(mode) > len(modes):
- raise ValueError("invalid mode: %r" % mode)
- reading = "r" in modes
- writing = "w" in modes
- appending = "a" in modes
- updating = "+" in modes
- text = "t" in modes
- binary = "b" in modes
- if "U" in modes:
- if writing or appending:
- raise ValueError("can't use U and writing mode at once")
- reading = True
- if text and binary:
- raise ValueError("can't have text and binary mode at once")
- if reading + writing + appending > 1:
- raise ValueError("can't have read/write/append mode at once")
- if not (reading or writing or appending):
- raise ValueError("must have exactly one of read/write/append mode")
- if binary and encoding is not None:
- raise ValueError("binary mode doesn't take an encoding argument")
- if binary and errors is not None:
- raise ValueError("binary mode doesn't take an errors argument")
- if binary and newline is not None:
- raise ValueError("binary mode doesn't take a newline argument")
- raw = FileIO(file,
- (reading and "r" or "") +
- (writing and "w" or "") +
- (appending and "a" or "") +
- (updating and "+" or ""),
- closefd)
- result = raw
- try:
- line_buffering = False
- if buffering == 1 or buffering < 0 and raw.isatty():
- buffering = -1
- line_buffering = True
- if buffering < 0:
- buffering = DEFAULT_BUFFER_SIZE
- try:
- bs = os.fstat(raw.fileno()).st_blksize
- except (os.error, AttributeError):
- pass
- else:
- if bs > 1:
- buffering = bs
- if buffering < 0:
- raise ValueError("invalid buffering size")
- if buffering == 0:
- if binary:
- return result
- raise ValueError("can't have unbuffered text I/O")
- if updating:
- buffer = BufferedRandom(raw, buffering)
- elif writing or appending:
- buffer = BufferedWriter(raw, buffering)
- elif reading:
- buffer = BufferedReader(raw, buffering)
- else:
- raise ValueError("unknown mode: %r" % mode)
- result = buffer
- if binary:
- return result
- text = TextIOWrapper(buffer, encoding, errors, newline, line_buffering)
- result = text
- text.mode = mode
- return result
- except:
- result.close()
- raise
- class DocDescriptor:
- """Helper for builtins.open.__doc__
- """
- def __get__(self, obj, typ):
- return (
- "open(file, mode='r', buffering=-1, encoding=None, "
- "errors=None, newline=None, closefd=True)\n\n" +
- open.__doc__)
- class OpenWrapper:
- """Wrapper for builtins.open
- Trick so that open won't become a bound method when stored
- as a class variable (as dbm.dumb does).
- See initstdio() in Python/pythonrun.c.
- """
- __doc__ = DocDescriptor()
- def __new__(cls, *args, **kwargs):
- return open(*args, **kwargs)
- class UnsupportedOperation(ValueError, IOError):
- pass
- class IOBase:
- __metaclass__ = abc.ABCMeta
- """The abstract base class for all I/O classes, acting on streams of
- bytes. There is no public constructor.
- This class provides dummy implementations for many methods that
- derived classes can override selectively; the default implementations
- represent a file that cannot be read, written or seeked.
- Even though IOBase does not declare read, readinto, or write because
- their signatures will vary, implementations and clients should
- consider those methods part of the interface. Also, implementations
- may raise a IOError when operations they do not support are called.
- The basic type used for binary data read from or written to a file is
- the bytes type. Method arguments may also be bytearray or memoryview of
- arrays of bytes. In some cases, such as readinto, a writable object such
- as bytearray is required. Text I/O classes work with unicode data.
- Note that calling any method (even inquiries) on a closed stream is
- undefined. Implementations may raise IOError in this case.
- IOBase (and its subclasses) support the iterator protocol, meaning
- that an IOBase object can be iterated over yielding the lines in a
- stream.
- IOBase also supports the :keyword:`with` statement. In this example,
- fp is closed after the suite of the with statement is complete:
- with open('spam.txt', 'r') as fp:
- fp.write('Spam and eggs!')
- """
- ### Internal ###
- def _unsupported(self, name):
- """Internal: raise an exception for unsupported operations."""
- raise UnsupportedOperation("%s.%s() not supported" %
- (self.__class__.__name__, name))
- ### Positioning ###
- def seek(self, pos, whence=0):
- """Change stream position.
- Change the stream position to byte offset pos. Argument pos is
- interpreted relative to the position indicated by whence. Values
- for whence are:
- * 0 -- start of stream (the default); offset should be zero or positive
- * 1 -- current stream position; offset may be negative
- * 2 -- end of stream; offset is usually negative
- Return the new absolute position.
- """
- self._unsupported("seek")
- def tell(self):
- """Return current stream position."""
- return self.seek(0, 1)
- def truncate(self, pos=None):
- """Truncate file to size bytes.
- Size defaults to the current IO position as reported by tell(). Return
- the new size.
- """
- self._unsupported("truncate")
- ### Flush and close ###
- def flush(self):
- """Flush write buffers, if applicable.
- This is not implemented for read-only and non-blocking streams.
- """
- self._checkClosed()
- # XXX Should this return the number of bytes written???
- __closed = False
- def close(self):
- """Flush and close the IO object.
- This method has no effect if the file is already closed.
- """
- if not self.__closed:
- try:
- self.flush()
- finally:
- self.__closed = True
- def __del__(self):
- """Destructor. Calls close()."""
- # The try/except block is in case this is called at program
- # exit time, when it's possible that globals have already been
- # deleted, and then the close() call might fail. Since
- # there's nothing we can do about such failures and they annoy
- # the end users, we suppress the traceback.
- try:
- self.close()
- except:
- pass
- ### Inquiries ###
- def seekable(self):
- """Return whether object supports random access.
- If False, seek(), tell() and truncate() will raise IOError.
- This method may need to do a test seek().
- """
- return False
- def _checkSeekable(self, msg=None):
- """Internal: raise an IOError if file is not seekable
- """
- if not self.seekable():
- raise IOError("File or stream is not seekable."
- if msg is None else msg)
- def readable(self):
- """Return whether object was opened for reading.
- If False, read() will raise IOError.
- """
- return False
- def _checkReadable(self, msg=None):
- """Internal: raise an IOError if file is not readable
- """
- if not self.readable():
- raise IOError("File or stream is not readable."
- if msg is None else msg)
- def writable(self):
- """Return whether object was opened for writing.
- If False, write() and truncate() will raise IOError.
- """
- return False
- def _checkWritable(self, msg=None):
- """Internal: raise an IOError if file is not writable
- """
- if not self.writable():
- raise IOError("File or stream is not writable."
- if msg is None else msg)
- @property
- def closed(self):
- """closed: bool. True iff the file has been closed.
- For backwards compatibility, this is a property, not a predicate.
- """
- return self.__closed
- def _checkClosed(self, msg=None):
- """Internal: raise a ValueError if file is closed
- """
- if self.closed:
- raise ValueError("I/O operation on closed file."
- if msg is None else msg)
- ### Context manager ###
- def __enter__(self):
- """Context management protocol. Returns self."""
- self._checkClosed()
- return self
- def __exit__(self, *args):
- """Context management protocol. Calls close()"""
- self.close()
- ### Lower-level APIs ###
- # XXX Should these be present even if unimplemented?
- def fileno(self):
- """Returns underlying file descriptor if one exists.
- An IOError is raised if the IO object does not use a file descriptor.
- """
- self._unsupported("fileno")
- def isatty(self):
- """Return whether this is an 'interactive' stream.
- Return False if it can't be determined.
- """
- self._checkClosed()
- return False
- ### Readline[s] and writelines ###
- def readline(self, limit=-1):
- r"""Read and return a line from the stream.
- If limit is specified, at most limit bytes will be read.
- The line terminator is always b'\n' for binary files; for text
- files, the newlines argument to open can be used to select the line
- terminator(s) recognized.
- """
- # For backwards compatibility, a (slowish) readline().
- if hasattr(self, "peek"):
- def nreadahead():
- readahead = self.peek(1)
- if not readahead:
- return 1
- n = (readahead.find(b"\n") + 1) or len(readahead)
- if limit >= 0:
- n = min(n, limit)
- return n
- else:
- def nreadahead():
- return 1
- if limit is None:
- limit = -1
- elif not isinstance(limit, (int, long)):
- raise TypeError("limit must be an integer")
- res = bytearray()
- while limit < 0 or len(res) < limit:
- b = self.read(nreadahead())
- if not b:
- break
- res += b
- if res.endswith(b"\n"):
- break
- return bytes(res)
- def __iter__(self):
- self._checkClosed()
- return self
- def next(self):
- line = self.readline()
- if not line:
- raise StopIteration
- return line
- def readlines(self, hint=None):
- """Return a list of lines from the stream.
- hint can be specified to control the number of lines read: no more
- lines will be read if the total size (in bytes/characters) of all
- lines so far exceeds hint.
- """
- if hint is not None and not isinstance(hint, (int, long)):
- raise TypeError("integer or None expected")
- if hint is None or hint <= 0:
- return list(self)
- n = 0
- lines = []
- for line in self:
- lines.append(line)
- n += len(line)
- if n >= hint:
- break
- return lines
- def writelines(self, lines):
- self._checkClosed()
- for line in lines:
- self.write(line)
- io.IOBase.register(IOBase)
- class RawIOBase(IOBase):
- """Base class for raw binary I/O."""
- # The read() method is implemented by calling readinto(); derived
- # classes that want to support read() only need to implement
- # readinto() as a primitive operation. In general, readinto() can be
- # more efficient than read().
- # (It would be tempting to also provide an implementation of
- # readinto() in terms of read(), in case the latter is a more suitable
- # primitive operation, but that would lead to nasty recursion in case
- # a subclass doesn't implement either.)
- def read(self, n=-1):
- """Read and return up to n bytes.
- Returns an empty bytes object on EOF, or None if the object is
- set not to block and has no data to read.
- """
- if n is None:
- n = -1
- if n < 0:
- return self.readall()
- b = bytearray(n.__index__())
- n = self.readinto(b)
- if n is None:
- return None
- del b[n:]
- return bytes(b)
- def readall(self):
- """Read until EOF, using multiple read() call."""
- res = bytearray()
- while True:
- data = self.read(DEFAULT_BUFFER_SIZE)
- if not data:
- break
- res += data
- if res:
- return bytes(res)
- else:
- # b'' or None
- return data
- def readinto(self, b):
- """Read up to len(b) bytes into b.
- Returns number of bytes read (0 for EOF), or None if the object
- is set not to block and has no data to read.
- """
- self._unsupported("readinto")
- def write(self, b):
- """Write the given buffer to the IO stream.
- Returns the number of bytes written, which may be less than len(b).
- """
- self._unsupported("write")
- io.RawIOBase.register(RawIOBase)
- from _io import FileIO
- RawIOBase.register(FileIO)
- class BufferedIOBase(IOBase):
- """Base class for buffered IO objects.
- The main difference with RawIOBase is that the read() method
- supports omitting the size argument, and does not have a default
- implementation that defers to readinto().
- In addition, read(), readinto() and write() may raise
- BlockingIOError if the underlying raw stream is in non-blocking
- mode and not ready; unlike their raw counterparts, they will never
- return None.
- A typical implementation should not inherit from a RawIOBase
- implementation, but wrap one.
- """
- def read(self, n=None):
- """Read and return up to n bytes.
- If the argument is omitted, None, or negative, reads and
- returns all data until EOF.
- If the argument is positive, and the underlying raw stream is
- not 'interactive', multiple raw reads may be issued to satisfy
- the byte count (unless EOF is reached first). But for
- interactive raw streams (XXX and for pipes?), at most one raw
- read will be issued, and a short result does not imply that
- EOF is imminent.
- Returns an empty bytes array on EOF.
- Raises BlockingIOError if the underlying raw stream has no
- data at the moment.
- """
- self._unsupported("read")
- def read1(self, n=None):
- """Read up to n bytes with at most one read() system call."""
- self._unsupported("read1")
- def readinto(self, b):
- """Read up to len(b) bytes into b.
- Like read(), this may issue multiple reads to the underlying raw
- stream, unless the latter is 'interactive'.
- Returns the number of bytes read (0 for EOF).
- Raises BlockingIOError if the underlying raw stream has no
- data at the moment.
- """
- data = self.read(len(b))
- n = len(data)
- try:
- b[:n] = data
- except TypeError as err:
- import array
- if not isinstance(b, array.array):
- raise err
- b[:n] = array.array(b'b', data)
- return n
- def write(self, b):
- """Write the given buffer to the IO stream.
- Return the number of bytes written, which is always len(b).
- Raises BlockingIOError if the buffer is full and the
- underlying raw stream cannot accept more data at the moment.
- """
- self._unsupported("write")
- def detach(self):
- """
- Separate the underlying raw stream from the buffer and return it.
- After the raw stream has been detached, the buffer is in an unusable
- state.
- """
- self._unsupported("detach")
- io.BufferedIOBase.register(BufferedIOBase)
- class _BufferedIOMixin(BufferedIOBase):
- """A mixin implementation of BufferedIOBase with an underlying raw stream.
- This passes most requests on to the underlying raw stream. It
- does *not* provide implementations of read(), readinto() or
- write().
- """
- def __init__(self, raw):
- self._raw = raw
- ### Positioning ###
- def seek(self, pos, whence=0):
- new_position = self.raw.seek(pos, whence)
- if new_position < 0:
- raise IOError("seek() returned an invalid position")
- return new_position
- def tell(self):
- pos = self.raw.tell()
- if pos < 0:
- raise IOError("tell() returned an invalid position")
- return pos
- def truncate(self, pos=None):
- # Flush the stream. We're mixing buffered I/O with lower-level I/O,
- # and a flush may be necessary to synch both views of the current
- # file state.
- self.flush()
- if pos is None:
- pos = self.tell()
- # XXX: Should seek() be used, instead of passing the position
- # XXX directly to truncate?
- return self.raw.truncate(pos)
- ### Flush and close ###
- def flush(self):
- if self.closed:
- raise ValueError("flush of closed file")
- self.raw.flush()
- def close(self):
- if self.raw is not None and not self.closed:
- try:
- # may raise BlockingIOError or BrokenPipeError etc
- self.flush()
- finally:
- self.raw.close()
- def detach(self):
- if self.raw is None:
- raise ValueError("raw stream already detached")
- self.flush()
- raw = self._raw
- self._raw = None
- return raw
- ### Inquiries ###
- def seekable(self):
- return self.raw.seekable()
- def readable(self):
- return self.raw.readable()
- def writable(self):
- return self.raw.writable()
- @property
- def raw(self):
- return self._raw
- @property
- def closed(self):
- return self.raw.closed
- @property
- def name(self):
- return self.raw.name
- @property
- def mode(self):
- return self.raw.mode
- def __repr__(self):
- clsname = self.__class__.__name__
- try:
- name = self.name
- except Exception:
- return "<_pyio.{0}>".format(clsname)
- else:
- return "<_pyio.{0} name={1!r}>".format(clsname, name)
- ### Lower-level APIs ###
- def fileno(self):
- return self.raw.fileno()
- def isatty(self):
- return self.raw.isatty()
- class BytesIO(BufferedIOBase):
- """Buffered I/O implementation using an in-memory bytes buffer."""
- def __init__(self, initial_bytes=None):
- buf = bytearray()
- if initial_bytes is not None:
- buf.extend(initial_bytes)
- self._buffer = buf
- self._pos = 0
- def __getstate__(self):
- if self.closed:
- raise ValueError("__getstate__ on closed file")
- return self.__dict__.copy()
- def getvalue(self):
- """Return the bytes value (contents) of the buffer
- """
- if self.closed:
- raise ValueError("getvalue on closed file")
- return bytes(self._buffer)
- def read(self, n=None):
- if self.closed:
- raise ValueError("read from closed file")
- if n is None:
- n = -1
- if not isinstance(n, (int, long)):
- raise TypeError("integer argument expected, got {0!r}".format(
- type(n)))
- if n < 0:
- n = len(self._buffer)
- if len(self._buffer) <= self._pos:
- return b""
- newpos = min(len(self._buffer), self._pos + n)
- b = self._buffer[self._pos : newpos]
- self._pos = newpos
- return bytes(b)
- def read1(self, n):
- """This is the same as read.
- """
- return self.read(n)
- def write(self, b):
- if self.closed:
- raise ValueError("write to closed file")
- if isinstance(b, unicode):
- raise TypeError("can't write unicode to binary stream")
- n = len(b)
- if n == 0:
- return 0
- pos = self._pos
- if pos > len(self._buffer):
- # Inserts null bytes between the current end of the file
- # and the new write position.
- padding = b'\x00' * (pos - len(self._buffer))
- self._buffer += padding
- self._buffer[pos:pos + n] = b
- self._pos += n
- return n
- def seek(self, pos, whence=0):
- if self.closed:
- raise ValueError("seek on closed file")
- try:
- pos.__index__
- except AttributeError:
- raise TypeError("an integer is required")
- if whence == 0:
- if pos < 0:
- raise ValueError("negative seek position %r" % (pos,))
- self._pos = pos
- elif whence == 1:
- self._pos = max(0, self._pos + pos)
- elif whence == 2:
- self._pos = max(0, len(self._buffer) + pos)
- else:
- raise ValueError("invalid whence value")
- return self._pos
- def tell(self):
- if self.closed:
- raise ValueError("tell on closed file")
- return self._pos
- def truncate(self, pos=None):
- if self.closed:
- raise ValueError("truncate on closed file")
- if pos is None:
- pos = self._pos
- else:
- try:
- pos.__index__
- except AttributeError:
- raise TypeError("an integer is required")
- if pos < 0:
- raise ValueError("negative truncate position %r" % (pos,))
- del self._buffer[pos:]
- return pos
- def readable(self):
- if self.closed:
- raise ValueError("I/O operation on closed file.")
- return True
- def writable(self):
- if self.closed:
- raise ValueError("I/O operation on closed file.")
- return True
- def seekable(self):
- if self.closed:
- raise ValueError("I/O operation on closed file.")
- return True
- class BufferedReader(_BufferedIOMixin):
- """BufferedReader(raw[, buffer_size])
- A buffer for a readable, sequential BaseRawIO object.
- The constructor creates a BufferedReader for the given readable raw
- stream and buffer_size. If buffer_size is omitted, DEFAULT_BUFFER_SIZE
- is used.
- """
- def __init__(self, raw, buffer_size=DEFAULT_BUFFER_SIZE):
- """Create a new buffered reader using the given readable raw IO object.
- """
- if not raw.readable():
- raise IOError('"raw" argument must be readable.')
- _BufferedIOMixin.__init__(self, raw)
- if buffer_size <= 0:
- raise ValueError("invalid buffer size")
- self.buffer_size = buffer_size
- self._reset_read_buf()
- self._read_lock = Lock()
- def _reset_read_buf(self):
- self._read_buf = b""
- self._read_pos = 0
- def read(self, n=None):
- """Read n bytes.
- Returns exactly n bytes of data unless the underlying raw IO
- stream reaches EOF or if the call would block in non-blocking
- mode. If n is negative, read until EOF or until read() would
- block.
- """
- if n is not None and n < -1:
- raise ValueError("invalid number of bytes to read")
- with self._read_lock:
- return self._read_unlocked(n)
- def _read_unlocked(self, n=None):
- nodata_val = b""
- empty_values = (b"", None)
- buf = self._read_buf
- pos = self._read_pos
- # Special case for when the number of bytes to read is unspecified.
- if n is None or n == -1:
- self._reset_read_buf()
- chunks = [buf[pos:]] # Strip the consumed bytes.
- current_size = 0
- while True:
- # Read until EOF or until read() would block.
- try:
- chunk = self.raw.read()
- except IOError as e:
- if e.errno != EINTR:
- raise
- continue
- if chunk in empty_values:
- nodata_val = chunk
- break
- current_size += len(chunk)
- chunks.append(chunk)
- return b"".join(chunks) or nodata_val
- # The number of bytes to read is specified, return at most n bytes.
- avail = len(buf) - pos # Length of the available buffered data.
- if n <= avail:
- # Fast path: the data to read is fully buffered.
- self._read_pos += n
- return buf[pos:pos+n]
- # Slow path: read from the stream until enough bytes are read,
- # or until an EOF occurs or until read() would block.
- chunks = [buf[pos:]]
- wanted = max(self.buffer_size, n)
- while avail < n:
- try:
- chunk = self.raw.read(wanted)
- except IOError as e:
- if e.errno != EINTR:
- raise
- continue
- if chunk in empty_values:
- nodata_val = chunk
- break
- avail += len(chunk)
- chunks.append(chunk)
- # n is more than avail only when an EOF occurred or when
- # read() would have blocked.
- n = min(n, avail)
- out = b"".join(chunks)
- self._read_buf = out[n:] # Save the extra data in the buffer.
- self._read_pos = 0
- return out[:n] if out else nodata_val
- def peek(self, n=0):
- """Returns buffered bytes without advancing the position.
- The argument indicates a desired minimal number of bytes; we
- do at most one raw read to satisfy it. We never return more
- than self.buffer_size.
- """
- with self._read_lock:
- return self._peek_unlocked(n)
- def _peek_unlocked(self, n=0):
- want = min(n, self.buffer_size)
- have = len(self._read_buf) - self._read_pos
- if have < want or have <= 0:
- to_read = self.buffer_size - have
- while True:
- try:
- current = self.raw.read(to_read)
- except IOError as e:
- if e.errno != EINTR:
- raise
- continue
- break
- if current:
- self._read_buf = self._read_buf[self._read_pos:] + current
- self._read_pos = 0
- return self._read_buf[self._read_pos:]
- def read1(self, n):
- """Reads up to n bytes, with at most one read() system call."""
- # Returns up to n bytes. If at least one byte is buffered, we
- # only return buffered bytes. Otherwise, we do one raw read.
- if n < 0:
- raise ValueError("number of bytes to read must be positive")
- if n == 0:
- return b""
- with self._read_lock:
- self._peek_unlocked(1)
- return self._read_unlocked(
- min(n, len(self._read_buf) - self._read_pos))
- def tell(self):
- return _BufferedIOMixin.tell(self) - len(self._read_buf) + self._read_pos
- def seek(self, pos, whence=0):
- if not (0 <= whence <= 2):
- raise ValueError("invalid whence value")
- with self._read_lock:
- if whence == 1:
- pos -= len(self._read_buf) - self._read_pos
- pos = _BufferedIOMixin.seek(self, pos, whence)
- self._reset_read_buf()
- return pos
- class BufferedWriter(_BufferedIOMixin):
- """A buffer for a writeable sequential RawIO object.
- The constructor creates a BufferedWriter for the given writeable raw
- stream. If the buffer_size is not given, it defaults to
- DEFAULT_BUFFER_SIZE.
- """
- _warning_stack_offset = 2
- def __init__(self, raw,
- buffer_size=DEFAULT_BUFFER_SIZE, max_buffer_size=None):
- if not raw.writable():
- raise IOError('"raw" argument must be writable.')
- _BufferedIOMixin.__init__(self, raw)
- if buffer_size <= 0:
- raise ValueError("invalid buffer size")
- if max_buffer_size is not None:
- warnings.warn("max_buffer_size is deprecated", DeprecationWarning,
- self._warning_stack_offset)
- self.buffer_size = buffer_size
- self._write_buf = bytearray()
- self._write_lock = Lock()
- def write(self, b):
- if self.closed:
- raise ValueError("write to closed file")
- if isinstance(b, unicode):
- raise TypeError("can't write unicode to binary stream")
- with self._write_lock:
- # XXX we can implement some more tricks to try and avoid
- # partial writes
- if len(self._write_buf) > self.buffer_size:
- # We're full, so let's pre-flush the buffer. (This may
- # raise BlockingIOError with characters_written == 0.)
- self._flush_unlocked()
- before = len(self._write_buf)
- self._write_buf.extend(b)
- written = len(self._write_buf) - before
- if len(self._write_buf) > self.buffer_size:
- try:
- self._flush_unlocked()
- except BlockingIOError as e:
- if len(self._write_buf) > self.buffer_size:
- # We've hit the buffer_size. We have to accept a partial
- # write and cut back our buffer.
- overage = len(self._write_buf) - self.buffer_size
- written -= overage
- self._write_buf = self._write_buf[:self.buffer_size]
- raise BlockingIOError(e.errno, e.strerror, written)
- return written
- def truncate(self, pos=None):
- with self._write_lock:
- self._flush_unlocked()
- if pos is None:
- pos = self.raw.tell()
- return self.raw.truncate(pos)
- def flush(self):
- with self._write_lock:
- self._flush_unlocked()
- def _flush_unlocked(self):
- if self.closed:
- raise ValueError("flush of closed file")
- while self._write_buf:
- try:
- n = self.raw.write(self._write_buf)
- except BlockingIOError:
- raise RuntimeError("self.raw should implement RawIOBase: it "
- "should not raise BlockingIOError")
- except IOError as e:
- if e.errno != EINTR:
- raise
- continue
- if n is None:
- raise BlockingIOError(
- errno.EAGAIN,
- "write could not complete without blocking", 0)
- if n > len(self._write_buf) or n < 0:
- raise IOError("write() returned incorrect number of bytes")
- del self._write_buf[:n]
- def tell(self):
- return _BufferedIOMixin.tell(self) + len(self._write_buf)
- def seek(self, pos, whence=0):
- if not (0 <= whence <= 2):
- raise ValueError("invalid whence")
- with self._write_lock:
- self._flush_unlocked()
- return _BufferedIOMixin.seek(self, pos, whence)
- class BufferedRWPair(BufferedIOBase):
- """A buffered reader and writer object together.
- A buffered reader object and buffered writer object put together to
- form a sequential IO object that can read and write. This is typically
- used with a socket or two-way pipe.
- reader and writer are RawIOBase objects that are readable and
- writeable respectively. If the buffer_size is omitted it defaults to
- DEFAULT_BUFFER_SIZE.
- """
- # XXX The usefulness of this (compared to having two separate IO
- # objects) is questionable.
- def __init__(self, reader, writer,
- buffer_size=DEFAULT_BUFFER_SIZE, max_buffer_size=None):
- """Constructor.
- The arguments are two RawIO instances.
- """
- if max_buffer_size is not None:
- warnings.warn("max_buffer_size is deprecated", DeprecationWarning, 2)
- if not reader.readable():
- raise IOError('"reader" argument must be readable.')
- if not writer.writable():
- raise IOError('"writer" argument must be writable.')
- self.reader = BufferedReader(reader, buffer_size)
- self.writer = BufferedWriter(writer, buffer_size)
- def read(self, n=None):
- if n is None:
- n = -1
- return self.reader.read(n)
- def readinto(self, b):
- return self.reader.readinto(b)
- def write(self, b):
- return self.writer.write(b)
- def peek(self, n=0):
- return self.reader.peek(n)
- def read1(self, n):
- return self.reader.read1(n)
- def readable(self):
- return self.reader.readable()
- def writable(self):
- return self.writer.writable()
- def flush(self):
- return self.writer.flush()
- def close(self):
- try:
- self.writer.close()
- finally:
- self.reader.close()
- def isatty(self):
- return self.reader.isatty() or self.writer.isatty()
- @property
- def closed(self):
- return self.writer.closed
- class BufferedRandom(BufferedWriter, BufferedReader):
- """A buffered interface to random access streams.
- The constructor creates a reader and writer for a seekable stream,
- raw, given in the first argument. If the buffer_size is omitted it
- defaults to DEFAULT_BUFFER_SIZE.
- """
- _warning_stack_offset = 3
- def __init__(self, raw,
- buffer_size=DEFAULT_BUFFER_SIZE, max_buffer_size=None):
- raw._checkSeekable()
- BufferedReader.__init__(self, raw, buffer_size)
- BufferedWriter.__init__(self, raw, buffer_size, max_buffer_size)
- def seek(self, pos, whence=0):
- if not (0 <= whence <= 2):
- raise ValueError("invalid whence")
- self.flush()
- if self._read_buf:
- # Undo read ahead.
- with self._read_lock:
- self.raw.seek(self._read_pos - len(self._read_buf), 1)
- # First do the raw seek, then empty the read buffer, so that
- # if the raw seek fails, we don't lose buffered data forever.
- pos = self.raw.seek(pos, whence)
- with self._read_lock:
- self._reset_read_buf()
- if pos < 0:
- raise IOError("seek() returned invalid position")
- return pos
- def tell(self):
- if self._write_buf:
- return BufferedWriter.tell(self)
- else:
- return BufferedReader.tell(self)
- def truncate(self, pos=None):
- if pos is None:
- pos = self.tell()
- # Use seek to flush the read buffer.
- return BufferedWriter.truncate(self, pos)
- def read(self, n=None):
- if n is None:
- n = -1
- self.flush()
- return BufferedReader.read(self, n)
- def readinto(self, b):
- self.flush()
- return BufferedReader.readinto(self, b)
- def peek(self, n=0):
- self.flush()
- return BufferedReader.peek(self, n)
- def read1(self, n):
- self.flush()
- return BufferedReader.read1(self, n)
- def write(self, b):
- if self._read_buf:
- # Undo readahead
- with self._read_lock:
- self.raw.seek(self._read_pos - len(self._read_buf), 1)
- self._reset_read_buf()
- return BufferedWriter.write(self, b)
- class TextIOBase(IOBase):
- """Base class for text I/O.
- This class provides a character and line based interface to stream
- I/O. There is no readinto method because Python's character strings
- are immutable. There is no public constructor.
- """
- def read(self, n=-1):
- """Read at most n characters from stream.
- Read from underlying buffer until we have n characters or we hit EOF.
- If n is negative or omitted, read until EOF.
- """
- self._unsupported("read")
- def write(self, s):
- """Write string s to stream."""
- self._unsupported("write")
- def truncate(self, pos=None):
- """Truncate size to pos."""
- self._unsupported("truncate")
- def readline(self):
- """Read until newline or EOF.
- Returns an empty string if EOF is hit immediately.
- """
- self._unsupported("readline")
- def detach(self):
- """
- Separate the underlying buffer from the TextIOBase and return it.
- After the underlying buffer has been detached, the TextIO is in an
- unusable state.
- """
- self._unsupported("detach")
- @property
- def encoding(self):
- """Subclasses should override."""
- return None
- @property
- def newlines(self):
- """Line endings translated so far.
- Only line endings translated during reading are considered.
- Subclasses should override.
- """
- return None
- @property
- def errors(self):
- """Error setting of the decoder or encoder.
- Subclasses should override."""
- return None
- io.TextIOBase.register(TextIOBase)
- class IncrementalNewlineDecoder(codecs.IncrementalDecoder):
- r"""Codec used when reading a file in universal newlines mode. It wraps
- another incremental decoder, translating \r\n and \r into \n. It also
- records the types of newlines encountered. When used with
- translate=False, it ensures that the newline sequence is returned in
- one piece.
- """
- def __init__(self, decoder, translate, errors='strict'):
- codecs.IncrementalDecoder.__init__(self, errors=errors)
- self.translate = translate
- self.decoder = decoder
- self.seennl = 0
- self.pendingcr = False
- def decode(self, input, final=False):
- # decode input (with the eventual \r from a previous pass)
- if self.decoder is None:
- output = input
- else:
- output = self.decoder.decode(input, final=final)
- if self.pendingcr and (output or final):
- output = "\r" + output
- self.pendingcr = False
- # retain last \r even when not translating data:
- # then readline() is sure to get \r\n in one pass
- if output.endswith("\r") and not final:
- output = output[:-1]
- self.pendingcr = True
- # Record which newlines are read
- crlf = output.count('\r\n')
- cr = output.count('\r') - crlf
- lf = output.count('\n') - crlf
- self.seennl |= (lf and self._LF) | (cr and self._CR) \
- | (crlf and self._CRLF)
- if self.translate:
- if crlf:
- output = output.replace("\r\n", "\n")
- if cr:
- output = output.replace("\r", "\n")
- return output
- def getstate(self):
- if self.decoder is None:
- buf = b""
- flag = 0
- else:
- buf, flag = self.decoder.getstate()
- flag <<= 1
- if self.pendingcr:
- flag |= 1
- return buf, flag
- def setstate(self, state):
- buf, flag = state
- self.pendingcr = bool(flag & 1)
- if self.decoder is not None:
- self.decoder.setstate((buf, flag >> 1))
- def reset(self):
- self.seennl = 0
- self.pendingcr = False
- if self.decoder is not None:
- self.decoder.reset()
- _LF = 1
- _CR = 2
- _CRLF = 4
- @property
- def newlines(self):
- return (None,
- "\n",
- "\r",
- ("\r", "\n"),
- "\r\n",
- ("\n", "\r\n"),
- ("\r", "\r\n"),
- ("\r", "\n", "\r\n")
- )[self.seennl]
- class TextIOWrapper(TextIOBase):
- r"""Character and line based layer over a BufferedIOBase object, buffer.
- encoding gives the name of the encoding that the stream will be
- decoded or encoded with. It defaults to locale.getpreferredencoding.
- errors determines the strictness of encoding and decoding (see the
- codecs.register) and defaults to "strict".
- newline can be None, '', '\n', '\r', or '\r\n'. It controls the
- handling of line endings. If it is None, universal newlines is
- enabled. With this enabled, on input, the lines endings '\n', '\r',
- or '\r\n' are translated to '\n' before being returned to the
- caller. Conversely, on output, '\n' is translated to the system
- default line separator, os.linesep. If newline is any other of its
- legal values, that newline becomes the newline when the file is read
- and it is returned untranslated. On output, '\n' is converted to the
- newline.
- If line_buffering is True, a call to flush is implied when a call to
- write contains a newline character.
- """
- _CHUNK_SIZE = 2048
- def __init__(self, buffer, encoding=None, errors=None, newline=None,
- line_buffering=False):
- if newline is not None and not isinstance(newline, basestring):
- raise TypeError("illegal newline type: %r" % (type(newline),))
- if newline not in (None, "", "\n", "\r", "\r\n"):
- raise ValueError("illegal newline value: %r" % (newline,))
- if encoding is None:
- try:
- import locale
- except ImportError:
- # Importing locale may fail if Python is being built
- encoding = "ascii"
- else:
- encoding = locale.getpreferredencoding()
- if not isinstance(encoding, basestring):
- raise ValueError("invalid encoding: %r" % encoding)
- if sys.py3kwarning and not codecs.lookup(encoding)._is_text_encoding:
- msg = ("%r is not a text encoding; "
- "use codecs.open() to handle ar…
Large files files are truncated, but you can click here to view the full file