/External.LCA_RESTRICTED/Languages/CPython/27/Lib/test/test_io.py

http://github.com/IronLanguages/main · Python · 2782 lines · 2269 code · 317 blank · 196 comment · 207 complexity · 8639be738ac6943e6cc6b971842110ff MD5 · raw file

  1. """Unit tests for the io module."""
  2. # Tests of io are scattered over the test suite:
  3. # * test_bufio - tests file buffering
  4. # * test_memoryio - tests BytesIO and StringIO
  5. # * test_fileio - tests FileIO
  6. # * test_file - tests the file interface
  7. # * test_io - tests everything else in the io module
  8. # * test_univnewlines - tests universal newline support
  9. # * test_largefile - tests operations on a file greater than 2**32 bytes
  10. # (only enabled with -ulargefile)
  11. ################################################################################
  12. # ATTENTION TEST WRITERS!!!
  13. ################################################################################
  14. # When writing tests for io, it's important to test both the C and Python
  15. # implementations. This is usually done by writing a base test that refers to
  16. # the type it is testing as a attribute. Then it provides custom subclasses to
  17. # test both implementations. This file has lots of examples.
  18. ################################################################################
  19. from __future__ import print_function
  20. from __future__ import unicode_literals
  21. import os
  22. import sys
  23. import time
  24. import array
  25. import random
  26. import unittest
  27. import weakref
  28. import abc
  29. import signal
  30. import errno
  31. from itertools import cycle, count
  32. from collections import deque
  33. from test import test_support as support
  34. import codecs
  35. import io # C implementation of io
  36. import _pyio as pyio # Python implementation of io
  37. try:
  38. import threading
  39. except ImportError:
  40. threading = None
  41. __metaclass__ = type
  42. bytes = support.py3k_bytes
  43. def _default_chunk_size():
  44. """Get the default TextIOWrapper chunk size"""
  45. with io.open(__file__, "r", encoding="latin1") as f:
  46. return f._CHUNK_SIZE
  47. class MockRawIOWithoutRead:
  48. """A RawIO implementation without read(), so as to exercise the default
  49. RawIO.read() which calls readinto()."""
  50. def __init__(self, read_stack=()):
  51. self._read_stack = list(read_stack)
  52. self._write_stack = []
  53. self._reads = 0
  54. self._extraneous_reads = 0
  55. def write(self, b):
  56. self._write_stack.append(bytes(b))
  57. return len(b)
  58. def writable(self):
  59. return True
  60. def fileno(self):
  61. return 42
  62. def readable(self):
  63. return True
  64. def seekable(self):
  65. return True
  66. def seek(self, pos, whence):
  67. return 0 # wrong but we gotta return something
  68. def tell(self):
  69. return 0 # same comment as above
  70. def readinto(self, buf):
  71. self._reads += 1
  72. max_len = len(buf)
  73. try:
  74. data = self._read_stack[0]
  75. except IndexError:
  76. self._extraneous_reads += 1
  77. return 0
  78. if data is None:
  79. del self._read_stack[0]
  80. return None
  81. n = len(data)
  82. if len(data) <= max_len:
  83. del self._read_stack[0]
  84. buf[:n] = data
  85. return n
  86. else:
  87. buf[:] = data[:max_len]
  88. self._read_stack[0] = data[max_len:]
  89. return max_len
  90. def truncate(self, pos=None):
  91. return pos
  92. class CMockRawIOWithoutRead(MockRawIOWithoutRead, io.RawIOBase):
  93. pass
  94. class PyMockRawIOWithoutRead(MockRawIOWithoutRead, pyio.RawIOBase):
  95. pass
  96. class MockRawIO(MockRawIOWithoutRead):
  97. def read(self, n=None):
  98. self._reads += 1
  99. try:
  100. return self._read_stack.pop(0)
  101. except:
  102. self._extraneous_reads += 1
  103. return b""
  104. class CMockRawIO(MockRawIO, io.RawIOBase):
  105. pass
  106. class PyMockRawIO(MockRawIO, pyio.RawIOBase):
  107. pass
  108. class MisbehavedRawIO(MockRawIO):
  109. def write(self, b):
  110. return MockRawIO.write(self, b) * 2
  111. def read(self, n=None):
  112. return MockRawIO.read(self, n) * 2
  113. def seek(self, pos, whence):
  114. return -123
  115. def tell(self):
  116. return -456
  117. def readinto(self, buf):
  118. MockRawIO.readinto(self, buf)
  119. return len(buf) * 5
  120. class CMisbehavedRawIO(MisbehavedRawIO, io.RawIOBase):
  121. pass
  122. class PyMisbehavedRawIO(MisbehavedRawIO, pyio.RawIOBase):
  123. pass
  124. class CloseFailureIO(MockRawIO):
  125. closed = 0
  126. def close(self):
  127. if not self.closed:
  128. self.closed = 1
  129. raise IOError
  130. class CCloseFailureIO(CloseFailureIO, io.RawIOBase):
  131. pass
  132. class PyCloseFailureIO(CloseFailureIO, pyio.RawIOBase):
  133. pass
  134. class MockFileIO:
  135. def __init__(self, data):
  136. self.read_history = []
  137. super(MockFileIO, self).__init__(data)
  138. def read(self, n=None):
  139. res = super(MockFileIO, self).read(n)
  140. self.read_history.append(None if res is None else len(res))
  141. return res
  142. def readinto(self, b):
  143. res = super(MockFileIO, self).readinto(b)
  144. self.read_history.append(res)
  145. return res
  146. class CMockFileIO(MockFileIO, io.BytesIO):
  147. pass
  148. class PyMockFileIO(MockFileIO, pyio.BytesIO):
  149. pass
  150. class MockNonBlockWriterIO:
  151. def __init__(self):
  152. self._write_stack = []
  153. self._blocker_char = None
  154. def pop_written(self):
  155. s = b"".join(self._write_stack)
  156. self._write_stack[:] = []
  157. return s
  158. def block_on(self, char):
  159. """Block when a given char is encountered."""
  160. self._blocker_char = char
  161. def readable(self):
  162. return True
  163. def seekable(self):
  164. return True
  165. def writable(self):
  166. return True
  167. def write(self, b):
  168. b = bytes(b)
  169. n = -1
  170. if self._blocker_char:
  171. try:
  172. n = b.index(self._blocker_char)
  173. except ValueError:
  174. pass
  175. else:
  176. self._blocker_char = None
  177. self._write_stack.append(b[:n])
  178. raise self.BlockingIOError(0, "test blocking", n)
  179. self._write_stack.append(b)
  180. return len(b)
  181. class CMockNonBlockWriterIO(MockNonBlockWriterIO, io.RawIOBase):
  182. BlockingIOError = io.BlockingIOError
  183. class PyMockNonBlockWriterIO(MockNonBlockWriterIO, pyio.RawIOBase):
  184. BlockingIOError = pyio.BlockingIOError
  185. class IOTest(unittest.TestCase):
  186. def setUp(self):
  187. support.unlink(support.TESTFN)
  188. def tearDown(self):
  189. support.unlink(support.TESTFN)
  190. def write_ops(self, f):
  191. self.assertEqual(f.write(b"blah."), 5)
  192. f.truncate(0)
  193. self.assertEqual(f.tell(), 5)
  194. f.seek(0)
  195. self.assertEqual(f.write(b"blah."), 5)
  196. self.assertEqual(f.seek(0), 0)
  197. self.assertEqual(f.write(b"Hello."), 6)
  198. self.assertEqual(f.tell(), 6)
  199. self.assertEqual(f.seek(-1, 1), 5)
  200. self.assertEqual(f.tell(), 5)
  201. self.assertEqual(f.write(bytearray(b" world\n\n\n")), 9)
  202. self.assertEqual(f.seek(0), 0)
  203. self.assertEqual(f.write(b"h"), 1)
  204. self.assertEqual(f.seek(-1, 2), 13)
  205. self.assertEqual(f.tell(), 13)
  206. self.assertEqual(f.truncate(12), 12)
  207. self.assertEqual(f.tell(), 13)
  208. self.assertRaises(TypeError, f.seek, 0.0)
  209. def read_ops(self, f, buffered=False):
  210. data = f.read(5)
  211. self.assertEqual(data, b"hello")
  212. data = bytearray(data)
  213. self.assertEqual(f.readinto(data), 5)
  214. self.assertEqual(data, b" worl")
  215. self.assertEqual(f.readinto(data), 2)
  216. self.assertEqual(len(data), 5)
  217. self.assertEqual(data[:2], b"d\n")
  218. self.assertEqual(f.seek(0), 0)
  219. self.assertEqual(f.read(20), b"hello world\n")
  220. self.assertEqual(f.read(1), b"")
  221. self.assertEqual(f.readinto(bytearray(b"x")), 0)
  222. self.assertEqual(f.seek(-6, 2), 6)
  223. self.assertEqual(f.read(5), b"world")
  224. self.assertEqual(f.read(0), b"")
  225. self.assertEqual(f.readinto(bytearray()), 0)
  226. self.assertEqual(f.seek(-6, 1), 5)
  227. self.assertEqual(f.read(5), b" worl")
  228. self.assertEqual(f.tell(), 10)
  229. self.assertRaises(TypeError, f.seek, 0.0)
  230. if buffered:
  231. f.seek(0)
  232. self.assertEqual(f.read(), b"hello world\n")
  233. f.seek(6)
  234. self.assertEqual(f.read(), b"world\n")
  235. self.assertEqual(f.read(), b"")
  236. LARGE = 2**31
  237. def large_file_ops(self, f):
  238. assert f.readable()
  239. assert f.writable()
  240. self.assertEqual(f.seek(self.LARGE), self.LARGE)
  241. self.assertEqual(f.tell(), self.LARGE)
  242. self.assertEqual(f.write(b"xxx"), 3)
  243. self.assertEqual(f.tell(), self.LARGE + 3)
  244. self.assertEqual(f.seek(-1, 1), self.LARGE + 2)
  245. self.assertEqual(f.truncate(), self.LARGE + 2)
  246. self.assertEqual(f.tell(), self.LARGE + 2)
  247. self.assertEqual(f.seek(0, 2), self.LARGE + 2)
  248. self.assertEqual(f.truncate(self.LARGE + 1), self.LARGE + 1)
  249. self.assertEqual(f.tell(), self.LARGE + 2)
  250. self.assertEqual(f.seek(0, 2), self.LARGE + 1)
  251. self.assertEqual(f.seek(-1, 2), self.LARGE)
  252. self.assertEqual(f.read(2), b"x")
  253. def test_invalid_operations(self):
  254. # Try writing on a file opened in read mode and vice-versa.
  255. for mode in ("w", "wb"):
  256. with self.open(support.TESTFN, mode) as fp:
  257. self.assertRaises(IOError, fp.read)
  258. self.assertRaises(IOError, fp.readline)
  259. with self.open(support.TESTFN, "rb") as fp:
  260. self.assertRaises(IOError, fp.write, b"blah")
  261. self.assertRaises(IOError, fp.writelines, [b"blah\n"])
  262. with self.open(support.TESTFN, "r") as fp:
  263. self.assertRaises(IOError, fp.write, "blah")
  264. self.assertRaises(IOError, fp.writelines, ["blah\n"])
  265. def test_raw_file_io(self):
  266. with self.open(support.TESTFN, "wb", buffering=0) as f:
  267. self.assertEqual(f.readable(), False)
  268. self.assertEqual(f.writable(), True)
  269. self.assertEqual(f.seekable(), True)
  270. self.write_ops(f)
  271. with self.open(support.TESTFN, "rb", buffering=0) as f:
  272. self.assertEqual(f.readable(), True)
  273. self.assertEqual(f.writable(), False)
  274. self.assertEqual(f.seekable(), True)
  275. self.read_ops(f)
  276. def test_buffered_file_io(self):
  277. with self.open(support.TESTFN, "wb") as f:
  278. self.assertEqual(f.readable(), False)
  279. self.assertEqual(f.writable(), True)
  280. self.assertEqual(f.seekable(), True)
  281. self.write_ops(f)
  282. with self.open(support.TESTFN, "rb") as f:
  283. self.assertEqual(f.readable(), True)
  284. self.assertEqual(f.writable(), False)
  285. self.assertEqual(f.seekable(), True)
  286. self.read_ops(f, True)
  287. def test_readline(self):
  288. with self.open(support.TESTFN, "wb") as f:
  289. f.write(b"abc\ndef\nxyzzy\nfoo\x00bar\nanother line")
  290. with self.open(support.TESTFN, "rb") as f:
  291. self.assertEqual(f.readline(), b"abc\n")
  292. self.assertEqual(f.readline(10), b"def\n")
  293. self.assertEqual(f.readline(2), b"xy")
  294. self.assertEqual(f.readline(4), b"zzy\n")
  295. self.assertEqual(f.readline(), b"foo\x00bar\n")
  296. self.assertEqual(f.readline(None), b"another line")
  297. self.assertRaises(TypeError, f.readline, 5.3)
  298. with self.open(support.TESTFN, "r") as f:
  299. self.assertRaises(TypeError, f.readline, 5.3)
  300. def test_raw_bytes_io(self):
  301. f = self.BytesIO()
  302. self.write_ops(f)
  303. data = f.getvalue()
  304. self.assertEqual(data, b"hello world\n")
  305. f = self.BytesIO(data)
  306. self.read_ops(f, True)
  307. def test_large_file_ops(self):
  308. # On Windows and Mac OSX this test comsumes large resources; It takes
  309. # a long time to build the >2GB file and takes >2GB of disk space
  310. # therefore the resource must be enabled to run this test.
  311. if sys.platform[:3] == 'win' or sys.platform == 'darwin':
  312. if not support.is_resource_enabled("largefile"):
  313. print("\nTesting large file ops skipped on %s." % sys.platform,
  314. file=sys.stderr)
  315. print("It requires %d bytes and a long time." % self.LARGE,
  316. file=sys.stderr)
  317. print("Use 'regrtest.py -u largefile test_io' to run it.",
  318. file=sys.stderr)
  319. return
  320. with self.open(support.TESTFN, "w+b", 0) as f:
  321. self.large_file_ops(f)
  322. with self.open(support.TESTFN, "w+b") as f:
  323. self.large_file_ops(f)
  324. def test_with_open(self):
  325. for bufsize in (0, 1, 100):
  326. f = None
  327. with self.open(support.TESTFN, "wb", bufsize) as f:
  328. f.write(b"xxx")
  329. self.assertEqual(f.closed, True)
  330. f = None
  331. try:
  332. with self.open(support.TESTFN, "wb", bufsize) as f:
  333. 1 // 0
  334. except ZeroDivisionError:
  335. self.assertEqual(f.closed, True)
  336. else:
  337. self.fail("1 // 0 didn't raise an exception")
  338. # issue 5008
  339. def test_append_mode_tell(self):
  340. with self.open(support.TESTFN, "wb") as f:
  341. f.write(b"xxx")
  342. with self.open(support.TESTFN, "ab", buffering=0) as f:
  343. self.assertEqual(f.tell(), 3)
  344. with self.open(support.TESTFN, "ab") as f:
  345. self.assertEqual(f.tell(), 3)
  346. with self.open(support.TESTFN, "a") as f:
  347. self.assertTrue(f.tell() > 0)
  348. def test_destructor(self):
  349. record = []
  350. class MyFileIO(self.FileIO):
  351. def __del__(self):
  352. record.append(1)
  353. try:
  354. f = super(MyFileIO, self).__del__
  355. except AttributeError:
  356. pass
  357. else:
  358. f()
  359. def close(self):
  360. record.append(2)
  361. super(MyFileIO, self).close()
  362. def flush(self):
  363. record.append(3)
  364. super(MyFileIO, self).flush()
  365. f = MyFileIO(support.TESTFN, "wb")
  366. f.write(b"xxx")
  367. del f
  368. support.gc_collect()
  369. self.assertEqual(record, [1, 2, 3])
  370. with self.open(support.TESTFN, "rb") as f:
  371. self.assertEqual(f.read(), b"xxx")
  372. def _check_base_destructor(self, base):
  373. record = []
  374. class MyIO(base):
  375. def __init__(self):
  376. # This exercises the availability of attributes on object
  377. # destruction.
  378. # (in the C version, close() is called by the tp_dealloc
  379. # function, not by __del__)
  380. self.on_del = 1
  381. self.on_close = 2
  382. self.on_flush = 3
  383. def __del__(self):
  384. record.append(self.on_del)
  385. try:
  386. f = super(MyIO, self).__del__
  387. except AttributeError:
  388. pass
  389. else:
  390. f()
  391. def close(self):
  392. record.append(self.on_close)
  393. super(MyIO, self).close()
  394. def flush(self):
  395. record.append(self.on_flush)
  396. super(MyIO, self).flush()
  397. f = MyIO()
  398. del f
  399. support.gc_collect()
  400. self.assertEqual(record, [1, 2, 3])
  401. def test_IOBase_destructor(self):
  402. self._check_base_destructor(self.IOBase)
  403. def test_RawIOBase_destructor(self):
  404. self._check_base_destructor(self.RawIOBase)
  405. def test_BufferedIOBase_destructor(self):
  406. self._check_base_destructor(self.BufferedIOBase)
  407. def test_TextIOBase_destructor(self):
  408. self._check_base_destructor(self.TextIOBase)
  409. def test_close_flushes(self):
  410. with self.open(support.TESTFN, "wb") as f:
  411. f.write(b"xxx")
  412. with self.open(support.TESTFN, "rb") as f:
  413. self.assertEqual(f.read(), b"xxx")
  414. def test_array_writes(self):
  415. a = array.array(b'i', range(10))
  416. n = len(a.tostring())
  417. with self.open(support.TESTFN, "wb", 0) as f:
  418. self.assertEqual(f.write(a), n)
  419. with self.open(support.TESTFN, "wb") as f:
  420. self.assertEqual(f.write(a), n)
  421. def test_closefd(self):
  422. self.assertRaises(ValueError, self.open, support.TESTFN, 'w',
  423. closefd=False)
  424. def test_read_closed(self):
  425. with self.open(support.TESTFN, "w") as f:
  426. f.write("egg\n")
  427. with self.open(support.TESTFN, "r") as f:
  428. file = self.open(f.fileno(), "r", closefd=False)
  429. self.assertEqual(file.read(), "egg\n")
  430. file.seek(0)
  431. file.close()
  432. self.assertRaises(ValueError, file.read)
  433. def test_no_closefd_with_filename(self):
  434. # can't use closefd in combination with a file name
  435. self.assertRaises(ValueError, self.open, support.TESTFN, "r", closefd=False)
  436. def test_closefd_attr(self):
  437. with self.open(support.TESTFN, "wb") as f:
  438. f.write(b"egg\n")
  439. with self.open(support.TESTFN, "r") as f:
  440. self.assertEqual(f.buffer.raw.closefd, True)
  441. file = self.open(f.fileno(), "r", closefd=False)
  442. self.assertEqual(file.buffer.raw.closefd, False)
  443. def test_garbage_collection(self):
  444. # FileIO objects are collected, and collecting them flushes
  445. # all data to disk.
  446. f = self.FileIO(support.TESTFN, "wb")
  447. f.write(b"abcxxx")
  448. f.f = f
  449. wr = weakref.ref(f)
  450. del f
  451. support.gc_collect()
  452. self.assertTrue(wr() is None, wr)
  453. with self.open(support.TESTFN, "rb") as f:
  454. self.assertEqual(f.read(), b"abcxxx")
  455. def test_unbounded_file(self):
  456. # Issue #1174606: reading from an unbounded stream such as /dev/zero.
  457. zero = "/dev/zero"
  458. if not os.path.exists(zero):
  459. self.skipTest("{0} does not exist".format(zero))
  460. if sys.maxsize > 0x7FFFFFFF:
  461. self.skipTest("test can only run in a 32-bit address space")
  462. if support.real_max_memuse < support._2G:
  463. self.skipTest("test requires at least 2GB of memory")
  464. with self.open(zero, "rb", buffering=0) as f:
  465. self.assertRaises(OverflowError, f.read)
  466. with self.open(zero, "rb") as f:
  467. self.assertRaises(OverflowError, f.read)
  468. with self.open(zero, "r") as f:
  469. self.assertRaises(OverflowError, f.read)
  470. def test_flush_error_on_close(self):
  471. f = self.open(support.TESTFN, "wb", buffering=0)
  472. def bad_flush():
  473. raise IOError()
  474. f.flush = bad_flush
  475. self.assertRaises(IOError, f.close) # exception not swallowed
  476. def test_multi_close(self):
  477. f = self.open(support.TESTFN, "wb", buffering=0)
  478. f.close()
  479. f.close()
  480. f.close()
  481. self.assertRaises(ValueError, f.flush)
  482. def test_RawIOBase_read(self):
  483. # Exercise the default RawIOBase.read() implementation (which calls
  484. # readinto() internally).
  485. rawio = self.MockRawIOWithoutRead((b"abc", b"d", None, b"efg", None))
  486. self.assertEqual(rawio.read(2), b"ab")
  487. self.assertEqual(rawio.read(2), b"c")
  488. self.assertEqual(rawio.read(2), b"d")
  489. self.assertEqual(rawio.read(2), None)
  490. self.assertEqual(rawio.read(2), b"ef")
  491. self.assertEqual(rawio.read(2), b"g")
  492. self.assertEqual(rawio.read(2), None)
  493. self.assertEqual(rawio.read(2), b"")
  494. class CIOTest(IOTest):
  495. pass
  496. class PyIOTest(IOTest):
  497. test_array_writes = unittest.skip(
  498. "len(array.array) returns number of elements rather than bytelength"
  499. )(IOTest.test_array_writes)
  500. class CommonBufferedTests:
  501. # Tests common to BufferedReader, BufferedWriter and BufferedRandom
  502. def test_detach(self):
  503. raw = self.MockRawIO()
  504. buf = self.tp(raw)
  505. self.assertIs(buf.detach(), raw)
  506. self.assertRaises(ValueError, buf.detach)
  507. def test_fileno(self):
  508. rawio = self.MockRawIO()
  509. bufio = self.tp(rawio)
  510. self.assertEqual(42, bufio.fileno())
  511. def test_no_fileno(self):
  512. # XXX will we always have fileno() function? If so, kill
  513. # this test. Else, write it.
  514. pass
  515. def test_invalid_args(self):
  516. rawio = self.MockRawIO()
  517. bufio = self.tp(rawio)
  518. # Invalid whence
  519. self.assertRaises(ValueError, bufio.seek, 0, -1)
  520. self.assertRaises(ValueError, bufio.seek, 0, 3)
  521. def test_override_destructor(self):
  522. tp = self.tp
  523. record = []
  524. class MyBufferedIO(tp):
  525. def __del__(self):
  526. record.append(1)
  527. try:
  528. f = super(MyBufferedIO, self).__del__
  529. except AttributeError:
  530. pass
  531. else:
  532. f()
  533. def close(self):
  534. record.append(2)
  535. super(MyBufferedIO, self).close()
  536. def flush(self):
  537. record.append(3)
  538. super(MyBufferedIO, self).flush()
  539. rawio = self.MockRawIO()
  540. bufio = MyBufferedIO(rawio)
  541. writable = bufio.writable()
  542. del bufio
  543. support.gc_collect()
  544. if writable:
  545. self.assertEqual(record, [1, 2, 3])
  546. else:
  547. self.assertEqual(record, [1, 2])
  548. def test_context_manager(self):
  549. # Test usability as a context manager
  550. rawio = self.MockRawIO()
  551. bufio = self.tp(rawio)
  552. def _with():
  553. with bufio:
  554. pass
  555. _with()
  556. # bufio should now be closed, and using it a second time should raise
  557. # a ValueError.
  558. self.assertRaises(ValueError, _with)
  559. def test_error_through_destructor(self):
  560. # Test that the exception state is not modified by a destructor,
  561. # even if close() fails.
  562. rawio = self.CloseFailureIO()
  563. def f():
  564. self.tp(rawio).xyzzy
  565. with support.captured_output("stderr") as s:
  566. self.assertRaises(AttributeError, f)
  567. s = s.getvalue().strip()
  568. if s:
  569. # The destructor *may* have printed an unraisable error, check it
  570. self.assertEqual(len(s.splitlines()), 1)
  571. self.assertTrue(s.startswith("Exception IOError: "), s)
  572. self.assertTrue(s.endswith(" ignored"), s)
  573. def test_repr(self):
  574. raw = self.MockRawIO()
  575. b = self.tp(raw)
  576. clsname = "%s.%s" % (self.tp.__module__, self.tp.__name__)
  577. self.assertEqual(repr(b), "<%s>" % clsname)
  578. raw.name = "dummy"
  579. self.assertEqual(repr(b), "<%s name=u'dummy'>" % clsname)
  580. raw.name = b"dummy"
  581. self.assertEqual(repr(b), "<%s name='dummy'>" % clsname)
  582. def test_flush_error_on_close(self):
  583. raw = self.MockRawIO()
  584. def bad_flush():
  585. raise IOError()
  586. raw.flush = bad_flush
  587. b = self.tp(raw)
  588. self.assertRaises(IOError, b.close) # exception not swallowed
  589. def test_multi_close(self):
  590. raw = self.MockRawIO()
  591. b = self.tp(raw)
  592. b.close()
  593. b.close()
  594. b.close()
  595. self.assertRaises(ValueError, b.flush)
  596. def test_readonly_attributes(self):
  597. raw = self.MockRawIO()
  598. buf = self.tp(raw)
  599. x = self.MockRawIO()
  600. with self.assertRaises((AttributeError, TypeError)):
  601. buf.raw = x
  602. class BufferedReaderTest(unittest.TestCase, CommonBufferedTests):
  603. read_mode = "rb"
  604. def test_constructor(self):
  605. rawio = self.MockRawIO([b"abc"])
  606. bufio = self.tp(rawio)
  607. bufio.__init__(rawio)
  608. bufio.__init__(rawio, buffer_size=1024)
  609. bufio.__init__(rawio, buffer_size=16)
  610. self.assertEqual(b"abc", bufio.read())
  611. self.assertRaises(ValueError, bufio.__init__, rawio, buffer_size=0)
  612. self.assertRaises(ValueError, bufio.__init__, rawio, buffer_size=-16)
  613. self.assertRaises(ValueError, bufio.__init__, rawio, buffer_size=-1)
  614. rawio = self.MockRawIO([b"abc"])
  615. bufio.__init__(rawio)
  616. self.assertEqual(b"abc", bufio.read())
  617. def test_read(self):
  618. for arg in (None, 7):
  619. rawio = self.MockRawIO((b"abc", b"d", b"efg"))
  620. bufio = self.tp(rawio)
  621. self.assertEqual(b"abcdefg", bufio.read(arg))
  622. # Invalid args
  623. self.assertRaises(ValueError, bufio.read, -2)
  624. def test_read1(self):
  625. rawio = self.MockRawIO((b"abc", b"d", b"efg"))
  626. bufio = self.tp(rawio)
  627. self.assertEqual(b"a", bufio.read(1))
  628. self.assertEqual(b"b", bufio.read1(1))
  629. self.assertEqual(rawio._reads, 1)
  630. self.assertEqual(b"c", bufio.read1(100))
  631. self.assertEqual(rawio._reads, 1)
  632. self.assertEqual(b"d", bufio.read1(100))
  633. self.assertEqual(rawio._reads, 2)
  634. self.assertEqual(b"efg", bufio.read1(100))
  635. self.assertEqual(rawio._reads, 3)
  636. self.assertEqual(b"", bufio.read1(100))
  637. self.assertEqual(rawio._reads, 4)
  638. # Invalid args
  639. self.assertRaises(ValueError, bufio.read1, -1)
  640. def test_readinto(self):
  641. rawio = self.MockRawIO((b"abc", b"d", b"efg"))
  642. bufio = self.tp(rawio)
  643. b = bytearray(2)
  644. self.assertEqual(bufio.readinto(b), 2)
  645. self.assertEqual(b, b"ab")
  646. self.assertEqual(bufio.readinto(b), 2)
  647. self.assertEqual(b, b"cd")
  648. self.assertEqual(bufio.readinto(b), 2)
  649. self.assertEqual(b, b"ef")
  650. self.assertEqual(bufio.readinto(b), 1)
  651. self.assertEqual(b, b"gf")
  652. self.assertEqual(bufio.readinto(b), 0)
  653. self.assertEqual(b, b"gf")
  654. def test_readlines(self):
  655. def bufio():
  656. rawio = self.MockRawIO((b"abc\n", b"d\n", b"ef"))
  657. return self.tp(rawio)
  658. self.assertEqual(bufio().readlines(), [b"abc\n", b"d\n", b"ef"])
  659. self.assertEqual(bufio().readlines(5), [b"abc\n", b"d\n"])
  660. self.assertEqual(bufio().readlines(None), [b"abc\n", b"d\n", b"ef"])
  661. def test_buffering(self):
  662. data = b"abcdefghi"
  663. dlen = len(data)
  664. tests = [
  665. [ 100, [ 3, 1, 4, 8 ], [ dlen, 0 ] ],
  666. [ 100, [ 3, 3, 3], [ dlen ] ],
  667. [ 4, [ 1, 2, 4, 2 ], [ 4, 4, 1 ] ],
  668. ]
  669. for bufsize, buf_read_sizes, raw_read_sizes in tests:
  670. rawio = self.MockFileIO(data)
  671. bufio = self.tp(rawio, buffer_size=bufsize)
  672. pos = 0
  673. for nbytes in buf_read_sizes:
  674. self.assertEqual(bufio.read(nbytes), data[pos:pos+nbytes])
  675. pos += nbytes
  676. # this is mildly implementation-dependent
  677. self.assertEqual(rawio.read_history, raw_read_sizes)
  678. def test_read_non_blocking(self):
  679. # Inject some None's in there to simulate EWOULDBLOCK
  680. rawio = self.MockRawIO((b"abc", b"d", None, b"efg", None, None, None))
  681. bufio = self.tp(rawio)
  682. self.assertEqual(b"abcd", bufio.read(6))
  683. self.assertEqual(b"e", bufio.read(1))
  684. self.assertEqual(b"fg", bufio.read())
  685. self.assertEqual(b"", bufio.peek(1))
  686. self.assertIsNone(bufio.read())
  687. self.assertEqual(b"", bufio.read())
  688. rawio = self.MockRawIO((b"a", None, None))
  689. self.assertEqual(b"a", rawio.readall())
  690. self.assertIsNone(rawio.readall())
  691. def test_read_past_eof(self):
  692. rawio = self.MockRawIO((b"abc", b"d", b"efg"))
  693. bufio = self.tp(rawio)
  694. self.assertEqual(b"abcdefg", bufio.read(9000))
  695. def test_read_all(self):
  696. rawio = self.MockRawIO((b"abc", b"d", b"efg"))
  697. bufio = self.tp(rawio)
  698. self.assertEqual(b"abcdefg", bufio.read())
  699. @unittest.skipUnless(threading, 'Threading required for this test.')
  700. @support.requires_resource('cpu')
  701. def test_threads(self):
  702. try:
  703. # Write out many bytes with exactly the same number of 0's,
  704. # 1's... 255's. This will help us check that concurrent reading
  705. # doesn't duplicate or forget contents.
  706. N = 1000
  707. l = list(range(256)) * N
  708. random.shuffle(l)
  709. s = bytes(bytearray(l))
  710. with self.open(support.TESTFN, "wb") as f:
  711. f.write(s)
  712. with self.open(support.TESTFN, self.read_mode, buffering=0) as raw:
  713. bufio = self.tp(raw, 8)
  714. errors = []
  715. results = []
  716. def f():
  717. try:
  718. # Intra-buffer read then buffer-flushing read
  719. for n in cycle([1, 19]):
  720. s = bufio.read(n)
  721. if not s:
  722. break
  723. # list.append() is atomic
  724. results.append(s)
  725. except Exception as e:
  726. errors.append(e)
  727. raise
  728. threads = [threading.Thread(target=f) for x in range(20)]
  729. for t in threads:
  730. t.start()
  731. time.sleep(0.02) # yield
  732. for t in threads:
  733. t.join()
  734. self.assertFalse(errors,
  735. "the following exceptions were caught: %r" % errors)
  736. s = b''.join(results)
  737. for i in range(256):
  738. c = bytes(bytearray([i]))
  739. self.assertEqual(s.count(c), N)
  740. finally:
  741. support.unlink(support.TESTFN)
  742. def test_misbehaved_io(self):
  743. rawio = self.MisbehavedRawIO((b"abc", b"d", b"efg"))
  744. bufio = self.tp(rawio)
  745. self.assertRaises(IOError, bufio.seek, 0)
  746. self.assertRaises(IOError, bufio.tell)
  747. def test_no_extraneous_read(self):
  748. # Issue #9550; when the raw IO object has satisfied the read request,
  749. # we should not issue any additional reads, otherwise it may block
  750. # (e.g. socket).
  751. bufsize = 16
  752. for n in (2, bufsize - 1, bufsize, bufsize + 1, bufsize * 2):
  753. rawio = self.MockRawIO([b"x" * n])
  754. bufio = self.tp(rawio, bufsize)
  755. self.assertEqual(bufio.read(n), b"x" * n)
  756. # Simple case: one raw read is enough to satisfy the request.
  757. self.assertEqual(rawio._extraneous_reads, 0,
  758. "failed for {}: {} != 0".format(n, rawio._extraneous_reads))
  759. # A more complex case where two raw reads are needed to satisfy
  760. # the request.
  761. rawio = self.MockRawIO([b"x" * (n - 1), b"x"])
  762. bufio = self.tp(rawio, bufsize)
  763. self.assertEqual(bufio.read(n), b"x" * n)
  764. self.assertEqual(rawio._extraneous_reads, 0,
  765. "failed for {}: {} != 0".format(n, rawio._extraneous_reads))
  766. class CBufferedReaderTest(BufferedReaderTest):
  767. tp = io.BufferedReader
  768. def test_constructor(self):
  769. BufferedReaderTest.test_constructor(self)
  770. # The allocation can succeed on 32-bit builds, e.g. with more
  771. # than 2GB RAM and a 64-bit kernel.
  772. if sys.maxsize > 0x7FFFFFFF:
  773. rawio = self.MockRawIO()
  774. bufio = self.tp(rawio)
  775. self.assertRaises((OverflowError, MemoryError, ValueError),
  776. bufio.__init__, rawio, sys.maxsize)
  777. def test_initialization(self):
  778. rawio = self.MockRawIO([b"abc"])
  779. bufio = self.tp(rawio)
  780. self.assertRaises(ValueError, bufio.__init__, rawio, buffer_size=0)
  781. self.assertRaises(ValueError, bufio.read)
  782. self.assertRaises(ValueError, bufio.__init__, rawio, buffer_size=-16)
  783. self.assertRaises(ValueError, bufio.read)
  784. self.assertRaises(ValueError, bufio.__init__, rawio, buffer_size=-1)
  785. self.assertRaises(ValueError, bufio.read)
  786. def test_misbehaved_io_read(self):
  787. rawio = self.MisbehavedRawIO((b"abc", b"d", b"efg"))
  788. bufio = self.tp(rawio)
  789. # _pyio.BufferedReader seems to implement reading different, so that
  790. # checking this is not so easy.
  791. self.assertRaises(IOError, bufio.read, 10)
  792. def test_garbage_collection(self):
  793. # C BufferedReader objects are collected.
  794. # The Python version has __del__, so it ends into gc.garbage instead
  795. rawio = self.FileIO(support.TESTFN, "w+b")
  796. f = self.tp(rawio)
  797. f.f = f
  798. wr = weakref.ref(f)
  799. del f
  800. support.gc_collect()
  801. self.assertTrue(wr() is None, wr)
  802. class PyBufferedReaderTest(BufferedReaderTest):
  803. tp = pyio.BufferedReader
  804. class BufferedWriterTest(unittest.TestCase, CommonBufferedTests):
  805. write_mode = "wb"
  806. def test_constructor(self):
  807. rawio = self.MockRawIO()
  808. bufio = self.tp(rawio)
  809. bufio.__init__(rawio)
  810. bufio.__init__(rawio, buffer_size=1024)
  811. bufio.__init__(rawio, buffer_size=16)
  812. self.assertEqual(3, bufio.write(b"abc"))
  813. bufio.flush()
  814. self.assertRaises(ValueError, bufio.__init__, rawio, buffer_size=0)
  815. self.assertRaises(ValueError, bufio.__init__, rawio, buffer_size=-16)
  816. self.assertRaises(ValueError, bufio.__init__, rawio, buffer_size=-1)
  817. bufio.__init__(rawio)
  818. self.assertEqual(3, bufio.write(b"ghi"))
  819. bufio.flush()
  820. self.assertEqual(b"".join(rawio._write_stack), b"abcghi")
  821. def test_detach_flush(self):
  822. raw = self.MockRawIO()
  823. buf = self.tp(raw)
  824. buf.write(b"howdy!")
  825. self.assertFalse(raw._write_stack)
  826. buf.detach()
  827. self.assertEqual(raw._write_stack, [b"howdy!"])
  828. def test_write(self):
  829. # Write to the buffered IO but don't overflow the buffer.
  830. writer = self.MockRawIO()
  831. bufio = self.tp(writer, 8)
  832. bufio.write(b"abc")
  833. self.assertFalse(writer._write_stack)
  834. def test_write_overflow(self):
  835. writer = self.MockRawIO()
  836. bufio = self.tp(writer, 8)
  837. contents = b"abcdefghijklmnop"
  838. for n in range(0, len(contents), 3):
  839. bufio.write(contents[n:n+3])
  840. flushed = b"".join(writer._write_stack)
  841. # At least (total - 8) bytes were implicitly flushed, perhaps more
  842. # depending on the implementation.
  843. self.assertTrue(flushed.startswith(contents[:-8]), flushed)
  844. def check_writes(self, intermediate_func):
  845. # Lots of writes, test the flushed output is as expected.
  846. contents = bytes(range(256)) * 1000
  847. n = 0
  848. writer = self.MockRawIO()
  849. bufio = self.tp(writer, 13)
  850. # Generator of write sizes: repeat each N 15 times then proceed to N+1
  851. def gen_sizes():
  852. for size in count(1):
  853. for i in range(15):
  854. yield size
  855. sizes = gen_sizes()
  856. while n < len(contents):
  857. size = min(next(sizes), len(contents) - n)
  858. self.assertEqual(bufio.write(contents[n:n+size]), size)
  859. intermediate_func(bufio)
  860. n += size
  861. bufio.flush()
  862. self.assertEqual(contents,
  863. b"".join(writer._write_stack))
  864. def test_writes(self):
  865. self.check_writes(lambda bufio: None)
  866. def test_writes_and_flushes(self):
  867. self.check_writes(lambda bufio: bufio.flush())
  868. def test_writes_and_seeks(self):
  869. def _seekabs(bufio):
  870. pos = bufio.tell()
  871. bufio.seek(pos + 1, 0)
  872. bufio.seek(pos - 1, 0)
  873. bufio.seek(pos, 0)
  874. self.check_writes(_seekabs)
  875. def _seekrel(bufio):
  876. pos = bufio.seek(0, 1)
  877. bufio.seek(+1, 1)
  878. bufio.seek(-1, 1)
  879. bufio.seek(pos, 0)
  880. self.check_writes(_seekrel)
  881. def test_writes_and_truncates(self):
  882. self.check_writes(lambda bufio: bufio.truncate(bufio.tell()))
  883. def test_write_non_blocking(self):
  884. raw = self.MockNonBlockWriterIO()
  885. bufio = self.tp(raw, 8)
  886. self.assertEqual(bufio.write(b"abcd"), 4)
  887. self.assertEqual(bufio.write(b"efghi"), 5)
  888. # 1 byte will be written, the rest will be buffered
  889. raw.block_on(b"k")
  890. self.assertEqual(bufio.write(b"jklmn"), 5)
  891. # 8 bytes will be written, 8 will be buffered and the rest will be lost
  892. raw.block_on(b"0")
  893. try:
  894. bufio.write(b"opqrwxyz0123456789")
  895. except self.BlockingIOError as e:
  896. written = e.characters_written
  897. else:
  898. self.fail("BlockingIOError should have been raised")
  899. self.assertEqual(written, 16)
  900. self.assertEqual(raw.pop_written(),
  901. b"abcdefghijklmnopqrwxyz")
  902. self.assertEqual(bufio.write(b"ABCDEFGHI"), 9)
  903. s = raw.pop_written()
  904. # Previously buffered bytes were flushed
  905. self.assertTrue(s.startswith(b"01234567A"), s)
  906. def test_write_and_rewind(self):
  907. raw = io.BytesIO()
  908. bufio = self.tp(raw, 4)
  909. self.assertEqual(bufio.write(b"abcdef"), 6)
  910. self.assertEqual(bufio.tell(), 6)
  911. bufio.seek(0, 0)
  912. self.assertEqual(bufio.write(b"XY"), 2)
  913. bufio.seek(6, 0)
  914. self.assertEqual(raw.getvalue(), b"XYcdef")
  915. self.assertEqual(bufio.write(b"123456"), 6)
  916. bufio.flush()
  917. self.assertEqual(raw.getvalue(), b"XYcdef123456")
  918. def test_flush(self):
  919. writer = self.MockRawIO()
  920. bufio = self.tp(writer, 8)
  921. bufio.write(b"abc")
  922. bufio.flush()
  923. self.assertEqual(b"abc", writer._write_stack[0])
  924. def test_destructor(self):
  925. writer = self.MockRawIO()
  926. bufio = self.tp(writer, 8)
  927. bufio.write(b"abc")
  928. del bufio
  929. support.gc_collect()
  930. self.assertEqual(b"abc", writer._write_stack[0])
  931. def test_truncate(self):
  932. # Truncate implicitly flushes the buffer.
  933. with self.open(support.TESTFN, self.write_mode, buffering=0) as raw:
  934. bufio = self.tp(raw, 8)
  935. bufio.write(b"abcdef")
  936. self.assertEqual(bufio.truncate(3), 3)
  937. self.assertEqual(bufio.tell(), 6)
  938. with self.open(support.TESTFN, "rb", buffering=0) as f:
  939. self.assertEqual(f.read(), b"abc")
  940. @unittest.skipUnless(threading, 'Threading required for this test.')
  941. @support.requires_resource('cpu')
  942. def test_threads(self):
  943. try:
  944. # Write out many bytes from many threads and test they were
  945. # all flushed.
  946. N = 1000
  947. contents = bytes(range(256)) * N
  948. sizes = cycle([1, 19])
  949. n = 0
  950. queue = deque()
  951. while n < len(contents):
  952. size = next(sizes)
  953. queue.append(contents[n:n+size])
  954. n += size
  955. del contents
  956. # We use a real file object because it allows us to
  957. # exercise situations where the GIL is released before
  958. # writing the buffer to the raw streams. This is in addition
  959. # to concurrency issues due to switching threads in the middle
  960. # of Python code.
  961. with self.open(support.TESTFN, self.write_mode, buffering=0) as raw:
  962. bufio = self.tp(raw, 8)
  963. errors = []
  964. def f():
  965. try:
  966. while True:
  967. try:
  968. s = queue.popleft()
  969. except IndexError:
  970. return
  971. bufio.write(s)
  972. except Exception as e:
  973. errors.append(e)
  974. raise
  975. threads = [threading.Thread(target=f) for x in range(20)]
  976. for t in threads:
  977. t.start()
  978. time.sleep(0.02) # yield
  979. for t in threads:
  980. t.join()
  981. self.assertFalse(errors,
  982. "the following exceptions were caught: %r" % errors)
  983. bufio.close()
  984. with self.open(support.TESTFN, "rb") as f:
  985. s = f.read()
  986. for i in range(256):
  987. self.assertEqual(s.count(bytes([i])), N)
  988. finally:
  989. support.unlink(support.TESTFN)
  990. def test_misbehaved_io(self):
  991. rawio = self.MisbehavedRawIO()
  992. bufio = self.tp(rawio, 5)
  993. self.assertRaises(IOError, bufio.seek, 0)
  994. self.assertRaises(IOError, bufio.tell)
  995. self.assertRaises(IOError, bufio.write, b"abcdef")
  996. def test_max_buffer_size_deprecation(self):
  997. with support.check_warnings(("max_buffer_size is deprecated",
  998. DeprecationWarning)):
  999. self.tp(self.MockRawIO(), 8, 12)
  1000. class CBufferedWriterTest(BufferedWriterTest):
  1001. tp = io.BufferedWriter
  1002. def test_constructor(self):
  1003. BufferedWriterTest.test_constructor(self)
  1004. # The allocation can succeed on 32-bit builds, e.g. with more
  1005. # than 2GB RAM and a 64-bit kernel.
  1006. if sys.maxsize > 0x7FFFFFFF:
  1007. rawio = self.MockRawIO()
  1008. bufio = self.tp(rawio)
  1009. self.assertRaises((OverflowError, MemoryError, ValueError),
  1010. bufio.__init__, rawio, sys.maxsize)
  1011. def test_initialization(self):
  1012. rawio = self.MockRawIO()
  1013. bufio = self.tp(rawio)
  1014. self.assertRaises(ValueError, bufio.__init__, rawio, buffer_size=0)
  1015. self.assertRaises(ValueError, bufio.write, b"def")
  1016. self.assertRaises(ValueError, bufio.__init__, rawio, buffer_size=-16)
  1017. self.assertRaises(ValueError, bufio.write, b"def")
  1018. self.assertRaises(ValueError, bufio.__init__, rawio, buffer_size=-1)
  1019. self.assertRaises(ValueError, bufio.write, b"def")
  1020. def test_garbage_collection(self):
  1021. # C BufferedWriter objects are collected, and collecting them flushes
  1022. # all data to disk.
  1023. # The Python version has __del__, so it ends into gc.garbage instead
  1024. rawio = self.FileIO(support.TESTFN, "w+b")
  1025. f = self.tp(rawio)
  1026. f.write(b"123xxx")
  1027. f.x = f
  1028. wr = weakref.ref(f)
  1029. del f
  1030. support.gc_collect()
  1031. self.assertTrue(wr() is None, wr)
  1032. with self.open(support.TESTFN, "rb") as f:
  1033. self.assertEqual(f.read(), b"123xxx")
  1034. class PyBufferedWriterTest(BufferedWriterTest):
  1035. tp = pyio.BufferedWriter
  1036. class BufferedRWPairTest(unittest.TestCase):
  1037. def test_constructor(self):
  1038. pair = self.tp(self.MockRawIO(), self.MockRawIO())
  1039. self.assertFalse(pair.closed)
  1040. def test_detach(self):
  1041. pair = self.tp(self.MockRawIO(), self.MockRawIO())
  1042. self.assertRaises(self.UnsupportedOperation, pair.detach)
  1043. def test_constructor_max_buffer_size_deprecation(self):
  1044. with support.check_warnings(("max_buffer_size is deprecated",
  1045. DeprecationWarning)):
  1046. self.tp(self.MockRawIO(), self.MockRawIO(), 8, 12)
  1047. def test_constructor_with_not_readable(self):
  1048. class NotReadable(MockRawIO):
  1049. def readable(self):
  1050. return False
  1051. self.assertRaises(IOError, self.tp, NotReadable(), self.MockRawIO())
  1052. def test_constructor_with_not_writeable(self):
  1053. class NotWriteable(MockRawIO):
  1054. def writable(self):
  1055. return False
  1056. self.assertRaises(IOError, self.tp, self.MockRawIO(), NotWriteable())
  1057. def test_read(self):
  1058. pair = self.tp(self.BytesIO(b"abcdef"), self.MockRawIO())
  1059. self.assertEqual(pair.read(3), b"abc")
  1060. self.assertEqual(pair.read(1), b"d")
  1061. self.assertEqual(pair.read(), b"ef")
  1062. pair = self.tp(self.BytesIO(b"abc"), self.MockRawIO())
  1063. self.assertEqual(pair.read(None), b"abc")
  1064. def test_readlines(self):
  1065. pair = lambda: self.tp(self.BytesIO(b"abc\ndef\nh"), self.MockRawIO())
  1066. self.assertEqual(pair().readlines(), [b"abc\n", b"def\n", b"h"])
  1067. self.assertEqual(pair().readlines(), [b"abc\n", b"def\n", b"h"])
  1068. self.assertEqual(pair().readlines(5), [b"abc\n", b"def\n"])
  1069. def test_read1(self):
  1070. # .read1() is delegated to the underlying reader object, so this test
  1071. # can be shallow.
  1072. pair = self.tp(self.BytesIO(b"abcdef"), self.MockRawIO())
  1073. self.assertEqual(pair.read1(3), b"abc")
  1074. def test_readinto(self):
  1075. pair = self.tp(self.BytesIO(b"abcdef"), self.MockRawIO())
  1076. data = bytearray(5)
  1077. self.assertEqual(pair.readinto(data), 5)
  1078. self.assertEqual(data, b"abcde")
  1079. def test_write(self):
  1080. w = self.MockRawIO()
  1081. pair = self.tp(self.MockRawIO(), w)
  1082. pair.write(b"abc")
  1083. pair.flush()
  1084. pair.write(b"def")
  1085. pair.flush()
  1086. self.assertEqual(w._write_stack, [b"abc", b"def"])
  1087. def test_peek(self):
  1088. pair = self.tp(self.BytesIO(b"abcdef"), self.MockRawIO())
  1089. self.assertTrue(pair.peek(3).startswith(b"abc"))
  1090. self.assertEqual(pair.read(3), b"abc")
  1091. def test_readable(self):
  1092. pair = self.tp(self.MockRawIO(), self.MockRawIO())
  1093. self.assertTrue(pair.readable())
  1094. def test_writeable(self):
  1095. pair = self.tp(self.MockRawIO(), self.MockRawIO())
  1096. self.assertTrue(pair.writable())
  1097. def test_seekable(self):
  1098. # BufferedRWPairs are never seekable, even if their readers and writers
  1099. # are.
  1100. pair = self.tp(self.MockRawIO(), self.MockRawIO())
  1101. self.assertFalse(pair.seekable())
  1102. # .flush() is delegated to the underlying writer object and has been
  1103. # tested in the test_write method.
  1104. def test_close_and_closed(self):
  1105. pair = self.tp(self.MockRawIO(), self.MockRawIO())
  1106. self.assertFalse(pair.closed)
  1107. pair.close()
  1108. self.assertTrue(pair.closed)
  1109. def test_isatty(self):
  1110. class SelectableIsAtty(MockRawIO):
  1111. def __init__(self, isatty):
  1112. MockRawIO.__init__(self)
  1113. self._isatty = isatty
  1114. def isatty(self):
  1115. return self._isatty
  1116. pair = self.tp(SelectableIsAtty(False), SelectableIsAtty(False))
  1117. self.assertFalse(pair.isatty())
  1118. pair = self.tp(SelectableIsAtty(True), SelectableIsAtty(False))
  1119. self.assertTrue(pair.isatty())
  1120. pair = self.tp(SelectableIsAtty(False), SelectableIsAtty(True))
  1121. self.assertTrue(pair.isatty())
  1122. pair = self.tp(SelectableIsAtty(True), SelectableIsAtty(True))
  1123. self.assertTrue(pair.isatty())
  1124. class CBufferedRWPairTest(BufferedRWPairTest):
  1125. tp = io.BufferedRWPair
  1126. class PyBufferedRWPairTest(BufferedRWPairTest):
  1127. tp = pyio.BufferedRWPair
  1128. class BufferedRandomTest(BufferedReaderTest, BufferedWriterTest):
  1129. read_mode = "rb+"
  1130. write_mode = "wb+"
  1131. def test_constructor(self):
  1132. BufferedReaderTest.test_constructor(self)
  1133. BufferedWriterTest.test_constructor(self)
  1134. def test_read_and_write(self):
  1135. raw = self.MockRawIO((b"asdf", b"ghjk"))
  1136. rw = self.tp(raw, 8)
  1137. self.assertEqual(b"as", rw.read(2))
  1138. rw.write(b"ddd")
  1139. rw.write(b"eee")
  1140. self.assertFalse(raw._write_stack) # Buffer writes
  1141. self.assertEqual(b"ghjk", rw.read())
  1142. self.assertEqual(b"dddeee", raw._write_stack[0])
  1143. def test_seek_and_tell(self):
  1144. raw = self.BytesIO(b"asdfghjkl")
  1145. rw = self.tp(raw)
  1146. self.assertEqual(b"as", rw.read(2))
  1147. self.assertEqual(2, rw.tell())
  1148. rw.seek(0, 0)
  1149. self.assertEqual(b"asdf", rw.read(4))
  1150. rw.write(b"asdf")
  1151. rw.seek(0, 0)
  1152. self.assertEqual(b"asdfasdfl", rw.read())
  1153. self.assertEqual(9, rw.tell())
  1154. rw.seek(-4, 2)
  1155. self.assertEqual(5, rw.tell())
  1156. rw.seek(2, 1)
  1157. self.assertEqual(7, rw.tell())
  1158. self.assertEqual(b"fl", rw.read(11))
  1159. self.assertRaises(TypeError, rw.seek, 0.0)
  1160. def check_flush_and_read(self, read_func):
  1161. raw = self.BytesIO(b"abcdefghi")
  1162. bufio = self.tp(raw)
  1163. self.assertEqual(b"ab", read_func(bufio, 2))
  1164. bufio.write(b"12")
  1165. self.assertEqual(b"ef", read_func(bufio, 2))
  1166. self.assertEqual(6, bufio.tell())
  1167. bufio.flush()
  1168. self.assertEqual(6, bufio.tell())
  1169. self.assertEqual(b"ghi", read_func(bufio))
  1170. raw.seek(0, 0)
  1171. raw.write(b"XYZ")
  1172. # flush() resets the read buffer
  1173. bufio.flush()
  1174. bufio.seek(0, 0)
  1175. self.assertEqual(b"XYZ", read_func(bufio, 3))
  1176. def test_flush_and_read(self):
  1177. self.check_flush_and_read(lambda bufio, *args: bufio.read(*args))
  1178. def test_flush_and_readinto(self):
  1179. def _readinto(bufio, n=-1):
  1180. b = bytearray(n if n >= 0 else 9999)
  1181. n = bufio.readinto(b)
  1182. return bytes(b[:n])
  1183. self.check_flush_and_read(_readinto)
  1184. def test_flush_and_peek(self):
  1185. def _peek(bufio, n=-1):
  1186. # This relies on the fact that the buffer can contain the whole
  1187. # raw stream, otherwise peek() can return less.
  1188. b = bufio.peek(n)
  1189. if n != -1:
  1190. b = b[:n]
  1191. bufio.seek(len(b), 1)
  1192. return b
  1193. self.check_flush_and_read(_peek)
  1194. def test_flush_and_write(self):
  1195. raw = self.BytesIO(b"abcdefghi")
  1196. bufio = self.tp(raw)
  1197. bufio.write(b"123")
  1198. bufio.flush()
  1199. bufio.write(b"45")
  1200. bufio.flush()
  1201. bufio.seek(0, 0)
  1202. self.assertEqual(b"12345fghi", raw.getvalue())
  1203. self.assertEqual(b"12345fghi", bufio.read())
  1204. def test_threads(self):
  1205. BufferedReaderTest.test_threads(self)
  1206. BufferedWriterTest.test_threads(self)
  1207. def test_writes_and_peek(self):
  1208. def _peek(bufio):
  1209. bufio.peek(1)
  1210. self.check_writes(_peek)
  1211. def _peek(bufio):
  1212. pos = bufio.tell()
  1213. bufio.seek(-1, 1)
  1214. bufio.peek(1)
  1215. bufio.seek(pos, 0)
  1216. self.check_writes(_peek)
  1217. def test_writes_and_reads(self):
  1218. def _read(bufio):
  1219. bufio.seek(-1, 1)
  1220. bufio.read(1)
  1221. self.check_writes(_read)
  1222. def test_writes_and_read1s(self):
  1223. def _read1(bufio):
  1224. bufio.seek(-1, 1)
  1225. bufio.read1(1)
  1226. self.check_writes(_read1)
  1227. def test_writes_and_readintos(self):
  1228. def _read(bufio):
  1229. bufio.seek(-1, 1)
  1230. bufio.readinto(bytearray(1))
  1231. self.check_writes(_read)
  1232. def test_write_after_readahead(self):
  1233. # Issue #6629: writing after the buffer was filled by readahead should
  1234. # first rewind the raw stream.
  1235. for overwrite_size in [1, 5]:
  1236. raw = self.BytesIO(b"A" * 10)
  1237. bufio = self.tp(raw, 4)
  1238. # Trigger readahead
  1239. self.assertEqual(bufio.read(1), b"A")
  1240. self.assertEqual(bufio.tell(), 1)
  1241. # Overwriting should rewind the raw stream if it needs so
  1242. bufio.write(b"B" * overwrite_size)
  1243. self.assertEqual(bufio.tell(), overwrite_size + 1)
  1244. # If the write size was smaller than the buffer size, flush() and
  1245. # check that rewind happens.
  1246. bufio.flush()
  1247. self.assertEqual(bufio.tell(), overwrite_size + 1)
  1248. s = raw.getvalue()
  1249. self.assertEqual(s,
  1250. b"A" + b"B" * overwrite_size + b"A" * (9 - overwrite_size))
  1251. def test_write_rewind_write(self):
  1252. # Various combinations of reading / writing / seeking backwards / writing again
  1253. def mutate(bufio, pos1, pos2):
  1254. assert pos2 >= pos1
  1255. # Fill the buffer
  1256. bufio.seek(pos1)
  1257. bufio.read(pos2 - pos1)
  1258. bufio.write(b'\x02')
  1259. # This writes earlier than the previous write, but still inside
  1260. # the buffer.
  1261. bufio.seek(pos1)
  1262. bufio.write(b'\x01')
  1263. b = b"\x80\x81\x82\x83\x84"
  1264. for i in range(0, len(b)):
  1265. for j in range(i, len(b)):
  1266. raw = self.BytesIO(b)
  1267. bufio = self.tp(raw, 100)
  1268. mutate(bufio, i, j)
  1269. bufio.flush()
  1270. expected = bytearray(b)
  1271. expected[j] = 2
  1272. expected[i] = 1
  1273. self.assertEqual(raw.getvalue(), expected,
  1274. "failed result for i=%d, j=%d" % (i, j))
  1275. def test_truncate_after_read_or_write(self):
  1276. raw = self.BytesIO(b"A" * 10)
  1277. bufio = self.tp(raw, 100)
  1278. self.assertEqual(bufio.read(2), b"AA") # the read buffer gets filled
  1279. self.assertEqual(bufio.truncate(), 2)
  1280. self.assertEqual(bufio.write(b"BB"), 2) # the write buffer increases
  1281. self.assertEqual(bufio.truncate(), 4)
  1282. def test_misbehaved_io(self):
  1283. BufferedReaderTest.test_misbehaved_io(self)
  1284. BufferedWriterTest.test_misbehaved_io(self)
  1285. class CBufferedRandomTest(CBufferedReaderTest, CBufferedWriterTest, BufferedRandomTest):
  1286. tp = io.BufferedRandom
  1287. def test_constructor(self):
  1288. BufferedRandomTest.test_constructor(self)
  1289. # The allocation can succeed on 32-bit builds, e.g. with more
  1290. # than 2GB RAM and a 64-bit kernel.
  1291. if sys.maxsize > 0x7FFFFFFF:
  1292. rawio = self.MockRawIO()
  1293. bufio = self.tp(rawio)
  1294. self.assertRaises((OverflowError, MemoryError, ValueError),
  1295. bufio.__init__, rawio, sys.maxsize)
  1296. def test_garbage_collection(self):
  1297. CBufferedReaderTest.test_garbage_collection(self)
  1298. CBufferedWriterTest.test_garbage_collection(self)
  1299. class PyBufferedRandomTest(BufferedRandomTest):
  1300. tp = pyio.BufferedRandom
  1301. # To fully exercise seek/tell, the StatefulIncrementalDecoder has these
  1302. # properties:
  1303. # - A single output character can correspond to many bytes of input.
  1304. # - The number of input bytes to complete the character can be
  1305. # undetermined until the last input byte is received.
  1306. # - The number of input bytes can vary depending on previous input.
  1307. # - A single input byte can correspond to many characters of output.
  1308. # - The number of output characters can be undetermined until the
  1309. # last input byte is received.
  1310. # - The number of output characters can vary depending on previous input.
  1311. class StatefulIncrementalDecoder(codecs.IncrementalDecoder):
  1312. """
  1313. For testing seek/tell behavior with a stateful, buffering decoder.
  1314. Input is a sequence of words. Words may be fixed-length (length set
  1315. by input) or variable-length (period-terminated). In variable-length
  1316. mode, extra periods are ignored. Possible words are:
  1317. - 'i' followed by a number sets the input length, I (maximum 99).
  1318. When I is set to 0, words are space-terminated.
  1319. - 'o' followed by a number sets the output length, O (maximum 99).
  1320. - Any other word is converted into a word followed by a period on
  1321. the output. The output word consists of the input word truncated
  1322. or padded out with hyphens to make its length equal to O. If O
  1323. is 0, the word is output verbatim without truncating or padding.
  1324. I and O are initially set to 1. When I changes, any buffered input is
  1325. re-scanned according to the new I. EOF also terminates the last word.
  1326. """
  1327. def __init__(self, errors='strict'):
  1328. codecs.IncrementalDecoder.__init__(self, errors)
  1329. self.reset()
  1330. def __repr__(self):
  1331. return '<SID %x>' % id(self)
  1332. def reset(self):
  1333. self.i = 1
  1334. self.o = 1
  1335. self.buffer = bytearray()
  1336. def getstate(self):
  1337. i, o = self.i ^ 1, self.o ^ 1 # so that flags = 0 after reset()
  1338. return bytes(self.buffer), i*100 + o
  1339. def setstate(self, state):
  1340. buffer, io = state
  1341. self.buffer = bytearray(buffer)
  1342. i, o = divmod(io, 100)
  1343. self.i, self.o = i ^ 1, o ^ 1
  1344. def decode(self, input, final=False):
  1345. output = ''
  1346. for b in input:
  1347. if self.i == 0: # variable-length, terminated with period
  1348. if b == '.':
  1349. if self.buffer:
  1350. output += self.process_word()
  1351. else:
  1352. self.buffer.append(b)
  1353. else: # fixed-length, terminate after self.i bytes
  1354. self.buffer.append(b)
  1355. if len(self.buffer) == self.i:
  1356. output += self.process_word()
  1357. if final and self.buffer: # EOF terminates the last word
  1358. output += self.process_word()
  1359. return output
  1360. def process_word(self):
  1361. output = ''
  1362. if self.buffer[0] == ord('i'):
  1363. self.i = min(99, int(self.buffer[1:] or 0)) # set input length
  1364. elif self.buffer[0] == ord('o'):
  1365. self.o = min(99, int(self.buffer[1:] or 0)) # set output length
  1366. else:
  1367. output = self.buffer.decode('ascii')
  1368. if len(output) < self.o:
  1369. output += '-'*self.o # pad out with hyphens
  1370. if self.o:
  1371. output = output[:self.o] # truncate to output length
  1372. output += '.'
  1373. self.buffer = bytearray()
  1374. return output
  1375. codecEnabled = False
  1376. @classmethod
  1377. def lookupTestDecoder(cls, name):
  1378. if cls.codecEnabled and name == 'test_decoder':
  1379. latin1 = codecs.lookup('latin-1')
  1380. return codecs.CodecInfo(
  1381. name='test_decoder', encode=latin1.encode, decode=None,
  1382. incrementalencoder=None,
  1383. streamreader=None, streamwriter=None,
  1384. incrementaldecoder=cls)
  1385. # Register the previous decoder for testing.
  1386. # Disabled by default, tests will enable it.
  1387. codecs.register(StatefulIncrementalDecoder.lookupTestDecoder)
  1388. class StatefulIncrementalDecoderTest(unittest.TestCase):
  1389. """
  1390. Make sure the StatefulIncrementalDecoder actually works.
  1391. """
  1392. test_cases = [
  1393. # I=1, O=1 (fixed-length input == fixed-length output)
  1394. (b'abcd', False, 'a.b.c.d.'),
  1395. # I=0, O=0 (variable-length input, variable-length output)
  1396. (b'oiabcd', True, 'abcd.'),
  1397. # I=0, O=0 (should ignore extra periods)
  1398. (b'oi...abcd...', True, 'abcd.'),
  1399. # I=0, O=6 (variable-length input, fixed-length output)
  1400. (b'i.o6.x.xyz.toolongtofit.', False, 'x-----.xyz---.toolon.'),
  1401. # I=2, O=6 (fixed-length input < fixed-length output)
  1402. (b'i.i2.o6xyz', True, 'xy----.z-----.'),
  1403. # I=6, O=3 (fixed-length input > fixed-length output)
  1404. (b'i.o3.i6.abcdefghijklmnop', True, 'abc.ghi.mno.'),
  1405. # I=0, then 3; O=29, then 15 (with longer output)
  1406. (b'i.o29.a.b.cde.o15.abcdefghijabcdefghij.i3.a.b.c.d.ei00k.l.m', True,
  1407. 'a----------------------------.' +
  1408. 'b----------------------------.' +
  1409. 'cde--------------------------.' +
  1410. 'abcdefghijabcde.' +
  1411. 'a.b------------.' +
  1412. '.c.------------.' +
  1413. 'd.e------------.' +
  1414. 'k--------------.' +
  1415. 'l--------------.' +
  1416. 'm--------------.')
  1417. ]
  1418. def test_decoder(self):
  1419. # Try a few one-shot test cases.
  1420. for input, eof, output in self.test_cases:
  1421. d = StatefulIncrementalDecoder()
  1422. self.assertEqual(d.decode(input, eof), output)
  1423. # Also test an unfinished decode, followed by forcing EOF.
  1424. d = StatefulIncrementalDecoder()
  1425. self.assertEqual(d.decode(b'oiabcd'), '')
  1426. self.assertEqual(d.decode(b'', 1), 'abcd.')
  1427. class TextIOWrapperTest(unittest.TestCase):
  1428. def setUp(self):
  1429. self.testdata = b"AAA\r\nBBB\rCCC\r\nDDD\nEEE\r\n"
  1430. self.normalized = b"AAA\nBBB\nCCC\nDDD\nEEE\n".decode("ascii")
  1431. support.unlink(support.TESTFN)
  1432. def tearDown(self):
  1433. support.unlink(support.TESTFN)
  1434. def test_constructor(self):
  1435. r = self.BytesIO(b"\xc3\xa9\n\n")
  1436. b = self.BufferedReader(r, 1000)
  1437. t = self.TextIOWrapper(b)
  1438. t.__init__(b, encoding="latin1", newline="\r\n")
  1439. self.assertEqual(t.encoding, "latin1")
  1440. self.assertEqual(t.line_buffering, False)
  1441. t.__init__(b, encoding="utf8", line_buffering=True)
  1442. self.assertEqual(t.encoding, "utf8")
  1443. self.assertEqual(t.line_buffering, True)
  1444. self.assertEqual("\xe9\n", t.readline())
  1445. self.assertRaises(TypeError, t.__init__, b, newline=42)
  1446. self.assertRaises(ValueError, t.__init__, b, newline='xyzzy')
  1447. def test_detach(self):
  1448. r = self.BytesIO()
  1449. b = self.BufferedWriter(r)
  1450. t = self.TextIOWrapper(b)
  1451. self.assertIs(t.detach(), b)
  1452. t = self.TextIOWrapper(b, encoding="ascii")
  1453. t.write("howdy")
  1454. self.assertFalse(r.getvalue())
  1455. t.detach()
  1456. self.assertEqual(r.getvalue(), b"howdy")
  1457. self.assertRaises(ValueError, t.detach)
  1458. def test_repr(self):
  1459. raw = self.BytesIO("hello".encode("utf-8"))
  1460. b = self.BufferedReader(raw)
  1461. t = self.TextIOWrapper(b, encoding="utf-8")
  1462. modname = self.TextIOWrapper.__module__
  1463. self.assertEqual(repr(t),
  1464. "<%s.TextIOWrapper encoding='utf-8'>" % modname)
  1465. raw.name = "dummy"
  1466. self.assertEqual(repr(t),
  1467. "<%s.TextIOWrapper name=u'dummy' encoding='utf-8'>" % modname)
  1468. raw.name = b"dummy"
  1469. self.assertEqual(repr(t),
  1470. "<%s.TextIOWrapper name='dummy' encoding='utf-8'>" % modname)
  1471. def test_line_buffering(self):
  1472. r = self.BytesIO()
  1473. b = self.BufferedWriter(r, 1000)
  1474. t = self.TextIOWrapper(b, newline="\n", line_buffering=True)
  1475. t.write("X")
  1476. self.assertEqual(r.getvalue(), b"") # No flush happened
  1477. t.write("Y\nZ")
  1478. self.assertEqual(r.getvalue(), b"XY\nZ") # All got flushed
  1479. t.write("A\rB")
  1480. self.assertEqual(r.getvalue(), b"XY\nZA\rB")
  1481. def test_encoding(self):
  1482. # Check the encoding attribute is always set, and valid
  1483. b = self.BytesIO()
  1484. t = self.TextIOWrapper(b, encoding="utf8")
  1485. self.assertEqual(t.encoding, "utf8")
  1486. t = self.TextIOWrapper(b)
  1487. self.assertTrue(t.encoding is not None)
  1488. codecs.lookup(t.encoding)
  1489. def test_encoding_errors_reading(self):
  1490. # (1) default
  1491. b = self.BytesIO(b"abc\n\xff\n")
  1492. t = self.TextIOWrapper(b, encoding="ascii")
  1493. self.assertRaises(UnicodeError, t.read)
  1494. # (2) explicit strict
  1495. b = self.BytesIO(b"abc\n\xff\n")
  1496. t = self.TextIOWrapper(b, encoding="ascii", errors="strict")
  1497. self.assertRaises(UnicodeError, t.read)
  1498. # (3) ignore
  1499. b = self.BytesIO(b"abc\n\xff\n")
  1500. t = self.TextIOWrapper(b, encoding="ascii", errors="ignore")
  1501. self.assertEqual(t.read(), "abc\n\n")
  1502. # (4) replace
  1503. b = self.BytesIO(b"abc\n\xff\n")
  1504. t = self.TextIOWrapper(b, encoding="ascii", errors="replace")
  1505. self.assertEqual(t.read(), "abc\n\ufffd\n")
  1506. def test_encoding_errors_writing(self):
  1507. # (1) default
  1508. b = self.BytesIO()
  1509. t = self.TextIOWrapper(b, encoding="ascii")
  1510. self.assertRaises(UnicodeError, t.write, "\xff")
  1511. # (2) explicit strict
  1512. b = self.BytesIO()
  1513. t = self.TextIOWrapper(b, encoding="ascii", errors="strict")
  1514. self.assertRaises(UnicodeError, t.write, "\xff")
  1515. # (3) ignore
  1516. b = self.BytesIO()
  1517. t = self.TextIOWrapper(b, encoding="ascii", errors="ignore",
  1518. newline="\n")
  1519. t.write("abc\xffdef\n")
  1520. t.flush()
  1521. self.assertEqual(b.getvalue(), b"abcdef\n")
  1522. # (4) replace
  1523. b = self.BytesIO()
  1524. t = self.TextIOWrapper(b, encoding="ascii", errors="replace",
  1525. newline="\n")
  1526. t.write("abc\xffdef\n")
  1527. t.flush()
  1528. self.assertEqual(b.getvalue(), b"abc?def\n")
  1529. def test_newlines(self):
  1530. input_lines = [ "unix\n", "windows\r\n", "os9\r", "last\n", "nonl" ]
  1531. tests = [
  1532. [ None, [ 'unix\n', 'windows\n', 'os9\n', 'last\n', 'nonl' ] ],
  1533. [ '', input_lines ],
  1534. [ '\n', [ "unix\n", "windows\r\n", "os9\rlast\n", "nonl" ] ],
  1535. [ '\r\n', [ "unix\nwindows\r\n", "os9\rlast\nnonl" ] ],
  1536. [ '\r', [ "unix\nwindows\r", "\nos9\r", "last\nnonl" ] ],
  1537. ]
  1538. encodings = (
  1539. 'utf-8', 'latin-1',
  1540. 'utf-16', 'utf-16-le', 'utf-16-be',
  1541. 'utf-32', 'utf-32-le', 'utf-32-be',
  1542. )
  1543. # Try a range of buffer sizes to test the case where \r is the last
  1544. # character in TextIOWrapper._pending_line.
  1545. for encoding in encodings:
  1546. # XXX: str.encode() should return bytes
  1547. data = bytes(''.join(input_lines).encode(encoding))
  1548. for do_reads in (False, True):
  1549. for bufsize in range(1, 10):
  1550. for newline, exp_lines in tests:
  1551. bufio = self.BufferedReader(self.BytesIO(data), bufsize)
  1552. textio = self.TextIOWrapper(bufio, newline=newline,
  1553. encoding=encoding)
  1554. if do_reads:
  1555. got_lines = []
  1556. while True:
  1557. c2 = textio.read(2)
  1558. if c2 == '':
  1559. break
  1560. self.assertEqual(len(c2), 2)
  1561. got_lines.append(c2 + textio.readline())
  1562. else:
  1563. got_lines = list(textio)
  1564. for got_line, exp_line in zip(got_lines, exp_lines):
  1565. self.assertEqual(got_line, exp_line)
  1566. self.assertEqual(len(got_lines), len(exp_lines))
  1567. def test_newlines_input(self):
  1568. testdata = b"AAA\nBB\x00B\nCCC\rDDD\rEEE\r\nFFF\r\nGGG"
  1569. normalized = testdata.replace(b"\r\n", b"\n").replace(b"\r", b"\n")
  1570. for newline, expected in [
  1571. (None, normalized.decode("ascii").splitlines(True)),
  1572. ("", testdata.decode("ascii").splitlines(True)),
  1573. ("\n", ["AAA\n", "BB\x00B\n", "CCC\rDDD\rEEE\r\n", "FFF\r\n", "GGG"]),
  1574. ("\r\n", ["AAA\nBB\x00B\nCCC\rDDD\rEEE\r\n", "FFF\r\n", "GGG"]),
  1575. ("\r", ["AAA\nBB\x00B\nCCC\r", "DDD\r", "EEE\r", "\nFFF\r", "\nGGG"]),
  1576. ]:
  1577. buf = self.BytesIO(testdata)
  1578. txt = self.TextIOWrapper(buf, encoding="ascii", newline=newline)
  1579. self.assertEqual(txt.readlines(), expected)
  1580. txt.seek(0)
  1581. self.assertEqual(txt.read(), "".join(expected))
  1582. def test_newlines_output(self):
  1583. testdict = {
  1584. "": b"AAA\nBBB\nCCC\nX\rY\r\nZ",
  1585. "\n": b"AAA\nBBB\nCCC\nX\rY\r\nZ",
  1586. "\r": b"AAA\rBBB\rCCC\rX\rY\r\rZ",
  1587. "\r\n": b"AAA\r\nBBB\r\nCCC\r\nX\rY\r\r\nZ",
  1588. }
  1589. tests = [(None, testdict[os.linesep])] + sorted(testdict.items())
  1590. for newline, expected in tests:
  1591. buf = self.BytesIO()
  1592. txt = self.TextIOWrapper(buf, encoding="ascii", newline=newline)
  1593. txt.write("AAA\nB")
  1594. txt.write("BB\nCCC\n")
  1595. txt.write("X\rY\r\nZ")
  1596. txt.flush()
  1597. self.assertEqual(buf.closed, False)
  1598. self.assertEqual(buf.getvalue(), expected)
  1599. def test_destructor(self):
  1600. l = []
  1601. base = self.BytesIO
  1602. class MyBytesIO(base):
  1603. def close(self):
  1604. l.append(self.getvalue())
  1605. base.close(self)
  1606. b = MyBytesIO()
  1607. t = self.TextIOWrapper(b, encoding="ascii")
  1608. t.write("abc")
  1609. del t
  1610. support.gc_collect()
  1611. self.assertEqual([b"abc"], l)
  1612. def test_override_destructor(self):
  1613. record = []
  1614. class MyTextIO(self.TextIOWrapper):
  1615. def __del__(self):
  1616. record.append(1)
  1617. try:
  1618. f = super(MyTextIO, self).__del__
  1619. except AttributeError:
  1620. pass
  1621. else:
  1622. f()
  1623. def close(self):
  1624. record.append(2)
  1625. super(MyTextIO, self).close()
  1626. def flush(self):
  1627. record.append(3)
  1628. super(MyTextIO, self).flush()
  1629. b = self.BytesIO()
  1630. t = MyTextIO(b, encoding="ascii")
  1631. del t
  1632. support.gc_collect()
  1633. self.assertEqual(record, [1, 2, 3])
  1634. def test_error_through_destructor(self):
  1635. # Test that the exception state is not modified by a destructor,
  1636. # even if close() fails.
  1637. rawio = self.CloseFailureIO()
  1638. def f():
  1639. self.TextIOWrapper(rawio).xyzzy
  1640. with support.captured_output("stderr") as s:
  1641. self.assertRaises(AttributeError, f)
  1642. s = s.getvalue().strip()
  1643. if s:
  1644. # The destructor *may* have printed an unraisable error, check it
  1645. self.assertEqual(len(s.splitlines()), 1)
  1646. self.assertTrue(s.startswith("Exception IOError: "), s)
  1647. self.assertTrue(s.endswith(" ignored"), s)
  1648. # Systematic tests of the text I/O API
  1649. def test_basic_io(self):
  1650. for chunksize in (1, 2, 3, 4, 5, 15, 16, 17, 31, 32, 33, 63, 64, 65):
  1651. for enc in "ascii", "latin1", "utf8" :# , "utf-16-be", "utf-16-le":
  1652. f = self.open(support.TESTFN, "w+", encoding=enc)
  1653. f._CHUNK_SIZE = chunksize
  1654. self.assertEqual(f.write("abc"), 3)
  1655. f.close()
  1656. f = self.open(support.TESTFN, "r+", encoding=enc)
  1657. f._CHUNK_SIZE = chunksize
  1658. self.assertEqual(f.tell(), 0)
  1659. self.assertEqual(f.read(), "abc")
  1660. cookie = f.tell()
  1661. self.assertEqual(f.seek(0), 0)
  1662. self.assertEqual(f.read(None), "abc")
  1663. f.seek(0)
  1664. self.assertEqual(f.read(2), "ab")
  1665. self.assertEqual(f.read(1), "c")
  1666. self.assertEqual(f.read(1), "")
  1667. self.assertEqual(f.read(), "")
  1668. self.assertEqual(f.tell(), cookie)
  1669. self.assertEqual(f.seek(0), 0)
  1670. self.assertEqual(f.seek(0, 2), cookie)
  1671. self.assertEqual(f.write("def"), 3)
  1672. self.assertEqual(f.seek(cookie), cookie)
  1673. self.assertEqual(f.read(), "def")
  1674. if enc.startswith("utf"):
  1675. self.multi_line_test(f, enc)
  1676. f.close()
  1677. def multi_line_test(self, f, enc):
  1678. f.seek(0)
  1679. f.truncate()
  1680. sample = "s\xff\u0fff\uffff"
  1681. wlines = []
  1682. for size in (0, 1, 2, 3, 4, 5, 30, 31, 32, 33, 62, 63, 64, 65, 1000):
  1683. chars = []
  1684. for i in range(size):
  1685. chars.append(sample[i % len(sample)])
  1686. line = "".join(chars) + "\n"
  1687. wlines.append((f.tell(), line))
  1688. f.write(line)
  1689. f.seek(0)
  1690. rlines = []
  1691. while True:
  1692. pos = f.tell()
  1693. line = f.readline()
  1694. if not line:
  1695. break
  1696. rlines.append((pos, line))
  1697. self.assertEqual(rlines, wlines)
  1698. def test_telling(self):
  1699. f = self.open(support.TESTFN, "w+", encoding="utf8")
  1700. p0 = f.tell()
  1701. f.write("\xff\n")
  1702. p1 = f.tell()
  1703. f.write("\xff\n")
  1704. p2 = f.tell()
  1705. f.seek(0)
  1706. self.assertEqual(f.tell(), p0)
  1707. self.assertEqual(f.readline(), "\xff\n")
  1708. self.assertEqual(f.tell(), p1)
  1709. self.assertEqual(f.readline(), "\xff\n")
  1710. self.assertEqual(f.tell(), p2)
  1711. f.seek(0)
  1712. for line in f:
  1713. self.assertEqual(line, "\xff\n")
  1714. self.assertRaises(IOError, f.tell)
  1715. self.assertEqual(f.tell(), p2)
  1716. f.close()
  1717. def test_seeking(self):
  1718. chunk_size = _default_chunk_size()
  1719. prefix_size = chunk_size - 2
  1720. u_prefix = "a" * prefix_size
  1721. prefix = bytes(u_prefix.encode("utf-8"))
  1722. self.assertEqual(len(u_prefix), len(prefix))
  1723. u_suffix = "\u8888\n"
  1724. suffix = bytes(u_suffix.encode("utf-8"))
  1725. line = prefix + suffix
  1726. f = self.open(support.TESTFN, "wb")
  1727. f.write(line*2)
  1728. f.close()
  1729. f = self.open(support.TESTFN, "r", encoding="utf-8")
  1730. s = f.read(prefix_size)
  1731. self.assertEqual(s, prefix.decode("ascii"))
  1732. self.assertEqual(f.tell(), prefix_size)
  1733. self.assertEqual(f.readline(), u_suffix)
  1734. def test_seeking_too(self):
  1735. # Regression test for a specific bug
  1736. data = b'\xe0\xbf\xbf\n'
  1737. f = self.open(support.TESTFN, "wb")
  1738. f.write(data)
  1739. f.close()
  1740. f = self.open(support.TESTFN, "r", encoding="utf-8")
  1741. f._CHUNK_SIZE # Just test that it exists
  1742. f._CHUNK_SIZE = 2
  1743. f.readline()
  1744. f.tell()
  1745. def test_seek_and_tell(self):
  1746. #Test seek/tell using the StatefulIncrementalDecoder.
  1747. # Make test faster by doing smaller seeks
  1748. CHUNK_SIZE = 128
  1749. def test_seek_and_tell_with_data(data, min_pos=0):
  1750. """Tell/seek to various points within a data stream and ensure
  1751. that the decoded data returned by read() is consistent."""
  1752. f = self.open(support.TESTFN, 'wb')
  1753. f.write(data)
  1754. f.close()
  1755. f = self.open(support.TESTFN, encoding='test_decoder')
  1756. f._CHUNK_SIZE = CHUNK_SIZE
  1757. decoded = f.read()
  1758. f.close()
  1759. for i in range(min_pos, len(decoded) + 1): # seek positions
  1760. for j in [1, 5, len(decoded) - i]: # read lengths
  1761. f = self.open(support.TESTFN, encoding='test_decoder')
  1762. self.assertEqual(f.read(i), decoded[:i])
  1763. cookie = f.tell()
  1764. self.assertEqual(f.read(j), decoded[i:i + j])
  1765. f.seek(cookie)
  1766. self.assertEqual(f.read(), decoded[i:])
  1767. f.close()
  1768. # Enable the test decoder.
  1769. StatefulIncrementalDecoder.codecEnabled = 1
  1770. # Run the tests.
  1771. try:
  1772. # Try each test case.
  1773. for input, _, _ in StatefulIncrementalDecoderTest.test_cases:
  1774. test_seek_and_tell_with_data(input)
  1775. # Position each test case so that it crosses a chunk boundary.
  1776. for input, _, _ in StatefulIncrementalDecoderTest.test_cases:
  1777. offset = CHUNK_SIZE - len(input)//2
  1778. prefix = b'.'*offset
  1779. # Don't bother seeking into the prefix (takes too long).
  1780. min_pos = offset*2
  1781. test_seek_and_tell_with_data(prefix + input, min_pos)
  1782. # Ensure our test decoder won't interfere with subsequent tests.
  1783. finally:
  1784. StatefulIncrementalDecoder.codecEnabled = 0
  1785. def test_encoded_writes(self):
  1786. data = "1234567890"
  1787. tests = ("utf-16",
  1788. "utf-16-le",
  1789. "utf-16-be",
  1790. "utf-32",
  1791. "utf-32-le",
  1792. "utf-32-be")
  1793. for encoding in tests:
  1794. buf = self.BytesIO()
  1795. f = self.TextIOWrapper(buf, encoding=encoding)
  1796. # Check if the BOM is written only once (see issue1753).
  1797. f.write(data)
  1798. f.write(data)
  1799. f.seek(0)
  1800. self.assertEqual(f.read(), data * 2)
  1801. f.seek(0)
  1802. self.assertEqual(f.read(), data * 2)
  1803. self.assertEqual(buf.getvalue(), (data * 2).encode(encoding))
  1804. def test_unreadable(self):
  1805. class UnReadable(self.BytesIO):
  1806. def readable(self):
  1807. return False
  1808. txt = self.TextIOWrapper(UnReadable())
  1809. self.assertRaises(IOError, txt.read)
  1810. def test_read_one_by_one(self):
  1811. txt = self.TextIOWrapper(self.BytesIO(b"AA\r\nBB"))
  1812. reads = ""
  1813. while True:
  1814. c = txt.read(1)
  1815. if not c:
  1816. break
  1817. reads += c
  1818. self.assertEqual(reads, "AA\nBB")
  1819. def test_readlines(self):
  1820. txt = self.TextIOWrapper(self.BytesIO(b"AA\nBB\nCC"))
  1821. self.assertEqual(txt.readlines(), ["AA\n", "BB\n", "CC"])
  1822. txt.seek(0)
  1823. self.assertEqual(txt.readlines(None), ["AA\n", "BB\n", "CC"])
  1824. txt.seek(0)
  1825. self.assertEqual(txt.readlines(5), ["AA\n", "BB\n"])
  1826. # read in amounts equal to TextIOWrapper._CHUNK_SIZE which is 128.
  1827. def test_read_by_chunk(self):
  1828. # make sure "\r\n" straddles 128 char boundary.
  1829. txt = self.TextIOWrapper(self.BytesIO(b"A" * 127 + b"\r\nB"))
  1830. reads = ""
  1831. while True:
  1832. c = txt.read(128)
  1833. if not c:
  1834. break
  1835. reads += c
  1836. self.assertEqual(reads, "A"*127+"\nB")
  1837. def test_issue1395_1(self):
  1838. txt = self.TextIOWrapper(self.BytesIO(self.testdata), encoding="ascii")
  1839. # read one char at a time
  1840. reads = ""
  1841. while True:
  1842. c = txt.read(1)
  1843. if not c:
  1844. break
  1845. reads += c
  1846. self.assertEqual(reads, self.normalized)
  1847. def test_issue1395_2(self):
  1848. txt = self.TextIOWrapper(self.BytesIO(self.testdata), encoding="ascii")
  1849. txt._CHUNK_SIZE = 4
  1850. reads = ""
  1851. while True:
  1852. c = txt.read(4)
  1853. if not c:
  1854. break
  1855. reads += c
  1856. self.assertEqual(reads, self.normalized)
  1857. def test_issue1395_3(self):
  1858. txt = self.TextIOWrapper(self.BytesIO(self.testdata), encoding="ascii")
  1859. txt._CHUNK_SIZE = 4
  1860. reads = txt.read(4)
  1861. reads += txt.read(4)
  1862. reads += txt.readline()
  1863. reads += txt.readline()
  1864. reads += txt.readline()
  1865. self.assertEqual(reads, self.normalized)
  1866. def test_issue1395_4(self):
  1867. txt = self.TextIOWrapper(self.BytesIO(self.testdata), encoding="ascii")
  1868. txt._CHUNK_SIZE = 4
  1869. reads = txt.read(4)
  1870. reads += txt.read()
  1871. self.assertEqual(reads, self.normalized)
  1872. def test_issue1395_5(self):
  1873. txt = self.TextIOWrapper(self.BytesIO(self.testdata), encoding="ascii")
  1874. txt._CHUNK_SIZE = 4
  1875. reads = txt.read(4)
  1876. pos = txt.tell()
  1877. txt.seek(0)
  1878. txt.seek(pos)
  1879. self.assertEqual(txt.read(4), "BBB\n")
  1880. def test_issue2282(self):
  1881. buffer = self.BytesIO(self.testdata)
  1882. txt = self.TextIOWrapper(buffer, encoding="ascii")
  1883. self.assertEqual(buffer.seekable(), txt.seekable())
  1884. def test_append_bom(self):
  1885. # The BOM is not written again when appending to a non-empty file
  1886. filename = support.TESTFN
  1887. for charset in ('utf-8-sig', 'utf-16', 'utf-32'):
  1888. with self.open(filename, 'w', encoding=charset) as f:
  1889. f.write('aaa')
  1890. pos = f.tell()
  1891. with self.open(filename, 'rb') as f:
  1892. self.assertEqual(f.read(), 'aaa'.encode(charset))
  1893. with self.open(filename, 'a', encoding=charset) as f:
  1894. f.write('xxx')
  1895. with self.open(filename, 'rb') as f:
  1896. self.assertEqual(f.read(), 'aaaxxx'.encode(charset))
  1897. def test_seek_bom(self):
  1898. # Same test, but when seeking manually
  1899. filename = support.TESTFN
  1900. for charset in ('utf-8-sig', 'utf-16', 'utf-32'):
  1901. with self.open(filename, 'w', encoding=charset) as f:
  1902. f.write('aaa')
  1903. pos = f.tell()
  1904. with self.open(filename, 'r+', encoding=charset) as f:
  1905. f.seek(pos)
  1906. f.write('zzz')
  1907. f.seek(0)
  1908. f.write('bbb')
  1909. with self.open(filename, 'rb') as f:
  1910. self.assertEqual(f.read(), 'bbbzzz'.encode(charset))
  1911. def test_errors_property(self):
  1912. with self.open(support.TESTFN, "w") as f:
  1913. self.assertEqual(f.errors, "strict")
  1914. with self.open(support.TESTFN, "w", errors="replace") as f:
  1915. self.assertEqual(f.errors, "replace")
  1916. @unittest.skipUnless(threading, 'Threading required for this test.')
  1917. def test_threads_write(self):
  1918. # Issue6750: concurrent writes could duplicate data
  1919. event = threading.Event()
  1920. with self.open(support.TESTFN, "w", buffering=1) as f:
  1921. def run(n):
  1922. text = "Thread%03d\n" % n
  1923. event.wait()
  1924. f.write(text)
  1925. threads = [threading.Thread(target=lambda n=x: run(n))
  1926. for x in range(20)]
  1927. for t in threads:
  1928. t.start()
  1929. time.sleep(0.02)
  1930. event.set()
  1931. for t in threads:
  1932. t.join()
  1933. with self.open(support.TESTFN) as f:
  1934. content = f.read()
  1935. for n in range(20):
  1936. self.assertEqual(content.count("Thread%03d\n" % n), 1)
  1937. def test_flush_error_on_close(self):
  1938. txt = self.TextIOWrapper(self.BytesIO(self.testdata), encoding="ascii")
  1939. def bad_flush():
  1940. raise IOError()
  1941. txt.flush = bad_flush
  1942. self.assertRaises(IOError, txt.close) # exception not swallowed
  1943. def test_multi_close(self):
  1944. txt = self.TextIOWrapper(self.BytesIO(self.testdata), encoding="ascii")
  1945. txt.close()
  1946. txt.close()
  1947. txt.close()
  1948. self.assertRaises(ValueError, txt.flush)
  1949. def test_readonly_attributes(self):
  1950. txt = self.TextIOWrapper(self.BytesIO(self.testdata), encoding="ascii")
  1951. buf = self.BytesIO(self.testdata)
  1952. with self.assertRaises((AttributeError, TypeError)):
  1953. txt.buffer = buf
  1954. class CTextIOWrapperTest(TextIOWrapperTest):
  1955. def test_initialization(self):
  1956. r = self.BytesIO(b"\xc3\xa9\n\n")
  1957. b = self.BufferedReader(r, 1000)
  1958. t = self.TextIOWrapper(b)
  1959. self.assertRaises(TypeError, t.__init__, b, newline=42)
  1960. self.assertRaises(ValueError, t.read)
  1961. self.assertRaises(ValueError, t.__init__, b, newline='xyzzy')
  1962. self.assertRaises(ValueError, t.read)
  1963. def test_garbage_collection(self):
  1964. # C TextIOWrapper objects are collected, and collecting them flushes
  1965. # all data to disk.
  1966. # The Python version has __del__, so it ends in gc.garbage instead.
  1967. rawio = io.FileIO(support.TESTFN, "wb")
  1968. b = self.BufferedWriter(rawio)
  1969. t = self.TextIOWrapper(b, encoding="ascii")
  1970. t.write("456def")
  1971. t.x = t
  1972. wr = weakref.ref(t)
  1973. del t
  1974. support.gc_collect()
  1975. self.assertTrue(wr() is None, wr)
  1976. with self.open(support.TESTFN, "rb") as f:
  1977. self.assertEqual(f.read(), b"456def")
  1978. class PyTextIOWrapperTest(TextIOWrapperTest):
  1979. pass
  1980. class IncrementalNewlineDecoderTest(unittest.TestCase):
  1981. def check_newline_decoding_utf8(self, decoder):
  1982. # UTF-8 specific tests for a newline decoder
  1983. def _check_decode(b, s, **kwargs):
  1984. # We exercise getstate() / setstate() as well as decode()
  1985. state = decoder.getstate()
  1986. self.assertEqual(decoder.decode(b, **kwargs), s)
  1987. decoder.setstate(state)
  1988. self.assertEqual(decoder.decode(b, **kwargs), s)
  1989. _check_decode(b'\xe8\xa2\x88', "\u8888")
  1990. _check_decode(b'\xe8', "")
  1991. _check_decode(b'\xa2', "")
  1992. _check_decode(b'\x88', "\u8888")
  1993. _check_decode(b'\xe8', "")
  1994. _check_decode(b'\xa2', "")
  1995. _check_decode(b'\x88', "\u8888")
  1996. _check_decode(b'\xe8', "")
  1997. self.assertRaises(UnicodeDecodeError, decoder.decode, b'', final=True)
  1998. decoder.reset()
  1999. _check_decode(b'\n', "\n")
  2000. _check_decode(b'\r', "")
  2001. _check_decode(b'', "\n", final=True)
  2002. _check_decode(b'\r', "\n", final=True)
  2003. _check_decode(b'\r', "")
  2004. _check_decode(b'a', "\na")
  2005. _check_decode(b'\r\r\n', "\n\n")
  2006. _check_decode(b'\r', "")
  2007. _check_decode(b'\r', "\n")
  2008. _check_decode(b'\na', "\na")
  2009. _check_decode(b'\xe8\xa2\x88\r\n', "\u8888\n")
  2010. _check_decode(b'\xe8\xa2\x88', "\u8888")
  2011. _check_decode(b'\n', "\n")
  2012. _check_decode(b'\xe8\xa2\x88\r', "\u8888")
  2013. _check_decode(b'\n', "\n")
  2014. def check_newline_decoding(self, decoder, encoding):
  2015. result = []
  2016. if encoding is not None:
  2017. encoder = codecs.getincrementalencoder(encoding)()
  2018. def _decode_bytewise(s):
  2019. # Decode one byte at a time
  2020. for b in encoder.encode(s):
  2021. result.append(decoder.decode(b))
  2022. else:
  2023. encoder = None
  2024. def _decode_bytewise(s):
  2025. # Decode one char at a time
  2026. for c in s:
  2027. result.append(decoder.decode(c))
  2028. self.assertEqual(decoder.newlines, None)
  2029. _decode_bytewise("abc\n\r")
  2030. self.assertEqual(decoder.newlines, '\n')
  2031. _decode_bytewise("\nabc")
  2032. self.assertEqual(decoder.newlines, ('\n', '\r\n'))
  2033. _decode_bytewise("abc\r")
  2034. self.assertEqual(decoder.newlines, ('\n', '\r\n'))
  2035. _decode_bytewise("abc")
  2036. self.assertEqual(decoder.newlines, ('\r', '\n', '\r\n'))
  2037. _decode_bytewise("abc\r")
  2038. self.assertEqual("".join(result), "abc\n\nabcabc\nabcabc")
  2039. decoder.reset()
  2040. input = "abc"
  2041. if encoder is not None:
  2042. encoder.reset()
  2043. input = encoder.encode(input)
  2044. self.assertEqual(decoder.decode(input), "abc")
  2045. self.assertEqual(decoder.newlines, None)
  2046. def test_newline_decoder(self):
  2047. encodings = (
  2048. # None meaning the IncrementalNewlineDecoder takes unicode input
  2049. # rather than bytes input
  2050. None, 'utf-8', 'latin-1',
  2051. 'utf-16', 'utf-16-le', 'utf-16-be',
  2052. 'utf-32', 'utf-32-le', 'utf-32-be',
  2053. )
  2054. for enc in encodings:
  2055. decoder = enc and codecs.getincrementaldecoder(enc)()
  2056. decoder = self.IncrementalNewlineDecoder(decoder, translate=True)
  2057. self.check_newline_decoding(decoder, enc)
  2058. decoder = codecs.getincrementaldecoder("utf-8")()
  2059. decoder = self.IncrementalNewlineDecoder(decoder, translate=True)
  2060. self.check_newline_decoding_utf8(decoder)
  2061. def test_newline_bytes(self):
  2062. # Issue 5433: Excessive optimization in IncrementalNewlineDecoder
  2063. def _check(dec):
  2064. self.assertEqual(dec.newlines, None)
  2065. self.assertEqual(dec.decode("\u0D00"), "\u0D00")
  2066. self.assertEqual(dec.newlines, None)
  2067. self.assertEqual(dec.decode("\u0A00"), "\u0A00")
  2068. self.assertEqual(dec.newlines, None)
  2069. dec = self.IncrementalNewlineDecoder(None, translate=False)
  2070. _check(dec)
  2071. dec = self.IncrementalNewlineDecoder(None, translate=True)
  2072. _check(dec)
  2073. class CIncrementalNewlineDecoderTest(IncrementalNewlineDecoderTest):
  2074. pass
  2075. class PyIncrementalNewlineDecoderTest(IncrementalNewlineDecoderTest):
  2076. pass
  2077. # XXX Tests for open()
  2078. class MiscIOTest(unittest.TestCase):
  2079. def tearDown(self):
  2080. support.unlink(support.TESTFN)
  2081. def test___all__(self):
  2082. for name in self.io.__all__:
  2083. obj = getattr(self.io, name, None)
  2084. self.assertTrue(obj is not None, name)
  2085. if name == "open":
  2086. continue
  2087. elif "error" in name.lower() or name == "UnsupportedOperation":
  2088. self.assertTrue(issubclass(obj, Exception), name)
  2089. elif not name.startswith("SEEK_"):
  2090. self.assertTrue(issubclass(obj, self.IOBase))
  2091. def test_attributes(self):
  2092. f = self.open(support.TESTFN, "wb", buffering=0)
  2093. self.assertEqual(f.mode, "wb")
  2094. f.close()
  2095. f = self.open(support.TESTFN, "U")
  2096. self.assertEqual(f.name, support.TESTFN)
  2097. self.assertEqual(f.buffer.name, support.TESTFN)
  2098. self.assertEqual(f.buffer.raw.name, support.TESTFN)
  2099. self.assertEqual(f.mode, "U")
  2100. self.assertEqual(f.buffer.mode, "rb")
  2101. self.assertEqual(f.buffer.raw.mode, "rb")
  2102. f.close()
  2103. f = self.open(support.TESTFN, "w+")
  2104. self.assertEqual(f.mode, "w+")
  2105. self.assertEqual(f.buffer.mode, "rb+") # Does it really matter?
  2106. self.assertEqual(f.buffer.raw.mode, "rb+")
  2107. g = self.open(f.fileno(), "wb", closefd=False)
  2108. self.assertEqual(g.mode, "wb")
  2109. self.assertEqual(g.raw.mode, "wb")
  2110. self.assertEqual(g.name, f.fileno())
  2111. self.assertEqual(g.raw.name, f.fileno())
  2112. f.close()
  2113. g.close()
  2114. def test_io_after_close(self):
  2115. for kwargs in [
  2116. {"mode": "w"},
  2117. {"mode": "wb"},
  2118. {"mode": "w", "buffering": 1},
  2119. {"mode": "w", "buffering": 2},
  2120. {"mode": "wb", "buffering": 0},
  2121. {"mode": "r"},
  2122. {"mode": "rb"},
  2123. {"mode": "r", "buffering": 1},
  2124. {"mode": "r", "buffering": 2},
  2125. {"mode": "rb", "buffering": 0},
  2126. {"mode": "w+"},
  2127. {"mode": "w+b"},
  2128. {"mode": "w+", "buffering": 1},
  2129. {"mode": "w+", "buffering": 2},
  2130. {"mode": "w+b", "buffering": 0},
  2131. ]:
  2132. f = self.open(support.TESTFN, **kwargs)
  2133. f.close()
  2134. self.assertRaises(ValueError, f.flush)
  2135. self.assertRaises(ValueError, f.fileno)
  2136. self.assertRaises(ValueError, f.isatty)
  2137. self.assertRaises(ValueError, f.__iter__)
  2138. if hasattr(f, "peek"):
  2139. self.assertRaises(ValueError, f.peek, 1)
  2140. self.assertRaises(ValueError, f.read)
  2141. if hasattr(f, "read1"):
  2142. self.assertRaises(ValueError, f.read1, 1024)
  2143. if hasattr(f, "readall"):
  2144. self.assertRaises(ValueError, f.readall)
  2145. if hasattr(f, "readinto"):
  2146. self.assertRaises(ValueError, f.readinto, bytearray(1024))
  2147. self.assertRaises(ValueError, f.readline)
  2148. self.assertRaises(ValueError, f.readlines)
  2149. self.assertRaises(ValueError, f.seek, 0)
  2150. self.assertRaises(ValueError, f.tell)
  2151. self.assertRaises(ValueError, f.truncate)
  2152. self.assertRaises(ValueError, f.write,
  2153. b"" if "b" in kwargs['mode'] else "")
  2154. self.assertRaises(ValueError, f.writelines, [])
  2155. self.assertRaises(ValueError, next, f)
  2156. def test_blockingioerror(self):
  2157. # Various BlockingIOError issues
  2158. self.assertRaises(TypeError, self.BlockingIOError)
  2159. self.assertRaises(TypeError, self.BlockingIOError, 1)
  2160. self.assertRaises(TypeError, self.BlockingIOError, 1, 2, 3, 4)
  2161. self.assertRaises(TypeError, self.BlockingIOError, 1, "", None)
  2162. b = self.BlockingIOError(1, "")
  2163. self.assertEqual(b.characters_written, 0)
  2164. class C(unicode):
  2165. pass
  2166. c = C("")
  2167. b = self.BlockingIOError(1, c)
  2168. c.b = b
  2169. b.c = c
  2170. wr = weakref.ref(c)
  2171. del c, b
  2172. support.gc_collect()
  2173. self.assertTrue(wr() is None, wr)
  2174. def test_abcs(self):
  2175. # Test the visible base classes are ABCs.
  2176. self.assertIsInstance(self.IOBase, abc.ABCMeta)
  2177. self.assertIsInstance(self.RawIOBase, abc.ABCMeta)
  2178. self.assertIsInstance(self.BufferedIOBase, abc.ABCMeta)
  2179. self.assertIsInstance(self.TextIOBase, abc.ABCMeta)
  2180. def _check_abc_inheritance(self, abcmodule):
  2181. with self.open(support.TESTFN, "wb", buffering=0) as f:
  2182. self.assertIsInstance(f, abcmodule.IOBase)
  2183. self.assertIsInstance(f, abcmodule.RawIOBase)
  2184. self.assertNotIsInstance(f, abcmodule.BufferedIOBase)
  2185. self.assertNotIsInstance(f, abcmodule.TextIOBase)
  2186. with self.open(support.TESTFN, "wb") as f:
  2187. self.assertIsInstance(f, abcmodule.IOBase)
  2188. self.assertNotIsInstance(f, abcmodule.RawIOBase)
  2189. self.assertIsInstance(f, abcmodule.BufferedIOBase)
  2190. self.assertNotIsInstance(f, abcmodule.TextIOBase)
  2191. with self.open(support.TESTFN, "w") as f:
  2192. self.assertIsInstance(f, abcmodule.IOBase)
  2193. self.assertNotIsInstance(f, abcmodule.RawIOBase)
  2194. self.assertNotIsInstance(f, abcmodule.BufferedIOBase)
  2195. self.assertIsInstance(f, abcmodule.TextIOBase)
  2196. def test_abc_inheritance(self):
  2197. # Test implementations inherit from their respective ABCs
  2198. self._check_abc_inheritance(self)
  2199. def test_abc_inheritance_official(self):
  2200. # Test implementations inherit from the official ABCs of the
  2201. # baseline "io" module.
  2202. self._check_abc_inheritance(io)
  2203. class CMiscIOTest(MiscIOTest):
  2204. io = io
  2205. class PyMiscIOTest(MiscIOTest):
  2206. io = pyio
  2207. @unittest.skipIf(os.name == 'nt', 'POSIX signals required for this test.')
  2208. class SignalsTest(unittest.TestCase):
  2209. def setUp(self):
  2210. self.oldalrm = signal.signal(signal.SIGALRM, self.alarm_interrupt)
  2211. def tearDown(self):
  2212. signal.signal(signal.SIGALRM, self.oldalrm)
  2213. def alarm_interrupt(self, sig, frame):
  2214. 1 // 0
  2215. @unittest.skipUnless(threading, 'Threading required for this test.')
  2216. def check_interrupted_write(self, item, bytes, **fdopen_kwargs):
  2217. """Check that a partial write, when it gets interrupted, properly
  2218. invokes the signal handler, and bubbles up the exception raised
  2219. in the latter."""
  2220. read_results = []
  2221. def _read():
  2222. s = os.read(r, 1)
  2223. read_results.append(s)
  2224. t = threading.Thread(target=_read)
  2225. t.daemon = True
  2226. r, w = os.pipe()
  2227. try:
  2228. wio = self.io.open(w, **fdopen_kwargs)
  2229. t.start()
  2230. signal.alarm(1)
  2231. # Fill the pipe enough that the write will be blocking.
  2232. # It will be interrupted by the timer armed above. Since the
  2233. # other thread has read one byte, the low-level write will
  2234. # return with a successful (partial) result rather than an EINTR.
  2235. # The buffered IO layer must check for pending signal
  2236. # handlers, which in this case will invoke alarm_interrupt().
  2237. self.assertRaises(ZeroDivisionError,
  2238. wio.write, item * (1024 * 1024))
  2239. t.join()
  2240. # We got one byte, get another one and check that it isn't a
  2241. # repeat of the first one.
  2242. read_results.append(os.read(r, 1))
  2243. self.assertEqual(read_results, [bytes[0:1], bytes[1:2]])
  2244. finally:
  2245. os.close(w)
  2246. os.close(r)
  2247. # This is deliberate. If we didn't close the file descriptor
  2248. # before closing wio, wio would try to flush its internal
  2249. # buffer, and block again.
  2250. try:
  2251. wio.close()
  2252. except IOError as e:
  2253. if e.errno != errno.EBADF:
  2254. raise
  2255. def test_interrupted_write_unbuffered(self):
  2256. self.check_interrupted_write(b"xy", b"xy", mode="wb", buffering=0)
  2257. def test_interrupted_write_buffered(self):
  2258. self.check_interrupted_write(b"xy", b"xy", mode="wb")
  2259. def test_interrupted_write_text(self):
  2260. self.check_interrupted_write("xy", b"xy", mode="w", encoding="ascii")
  2261. def check_reentrant_write(self, data, **fdopen_kwargs):
  2262. def on_alarm(*args):
  2263. # Will be called reentrantly from the same thread
  2264. wio.write(data)
  2265. 1/0
  2266. signal.signal(signal.SIGALRM, on_alarm)
  2267. r, w = os.pipe()
  2268. wio = self.io.open(w, **fdopen_kwargs)
  2269. try:
  2270. signal.alarm(1)
  2271. # Either the reentrant call to wio.write() fails with RuntimeError,
  2272. # or the signal handler raises ZeroDivisionError.
  2273. with self.assertRaises((ZeroDivisionError, RuntimeError)) as cm:
  2274. while 1:
  2275. for i in range(100):
  2276. wio.write(data)
  2277. wio.flush()
  2278. # Make sure the buffer doesn't fill up and block further writes
  2279. os.read(r, len(data) * 100)
  2280. exc = cm.exception
  2281. if isinstance(exc, RuntimeError):
  2282. self.assertTrue(str(exc).startswith("reentrant call"), str(exc))
  2283. finally:
  2284. wio.close()
  2285. os.close(r)
  2286. def test_reentrant_write_buffered(self):
  2287. self.check_reentrant_write(b"xy", mode="wb")
  2288. def test_reentrant_write_text(self):
  2289. self.check_reentrant_write("xy", mode="w", encoding="ascii")
  2290. def check_interrupted_read_retry(self, decode, **fdopen_kwargs):
  2291. """Check that a buffered read, when it gets interrupted (either
  2292. returning a partial result or EINTR), properly invokes the signal
  2293. handler and retries if the latter returned successfully."""
  2294. r, w = os.pipe()
  2295. fdopen_kwargs["closefd"] = False
  2296. def alarm_handler(sig, frame):
  2297. os.write(w, b"bar")
  2298. signal.signal(signal.SIGALRM, alarm_handler)
  2299. try:
  2300. rio = self.io.open(r, **fdopen_kwargs)
  2301. os.write(w, b"foo")
  2302. signal.alarm(1)
  2303. # Expected behaviour:
  2304. # - first raw read() returns partial b"foo"
  2305. # - second raw read() returns EINTR
  2306. # - third raw read() returns b"bar"
  2307. self.assertEqual(decode(rio.read(6)), "foobar")
  2308. finally:
  2309. rio.close()
  2310. os.close(w)
  2311. os.close(r)
  2312. def test_interrupterd_read_retry_buffered(self):
  2313. self.check_interrupted_read_retry(lambda x: x.decode('latin1'),
  2314. mode="rb")
  2315. def test_interrupterd_read_retry_text(self):
  2316. self.check_interrupted_read_retry(lambda x: x,
  2317. mode="r")
  2318. @unittest.skipUnless(threading, 'Threading required for this test.')
  2319. def check_interrupted_write_retry(self, item, **fdopen_kwargs):
  2320. """Check that a buffered write, when it gets interrupted (either
  2321. returning a partial result or EINTR), properly invokes the signal
  2322. handler and retries if the latter returned successfully."""
  2323. select = support.import_module("select")
  2324. # A quantity that exceeds the buffer size of an anonymous pipe's
  2325. # write end.
  2326. N = 1024 * 1024
  2327. r, w = os.pipe()
  2328. fdopen_kwargs["closefd"] = False
  2329. # We need a separate thread to read from the pipe and allow the
  2330. # write() to finish. This thread is started after the SIGALRM is
  2331. # received (forcing a first EINTR in write()).
  2332. read_results = []
  2333. write_finished = False
  2334. def _read():
  2335. while not write_finished:
  2336. while r in select.select([r], [], [], 1.0)[0]:
  2337. s = os.read(r, 1024)
  2338. read_results.append(s)
  2339. t = threading.Thread(target=_read)
  2340. t.daemon = True
  2341. def alarm1(sig, frame):
  2342. signal.signal(signal.SIGALRM, alarm2)
  2343. signal.alarm(1)
  2344. def alarm2(sig, frame):
  2345. t.start()
  2346. signal.signal(signal.SIGALRM, alarm1)
  2347. try:
  2348. wio = self.io.open(w, **fdopen_kwargs)
  2349. signal.alarm(1)
  2350. # Expected behaviour:
  2351. # - first raw write() is partial (because of the limited pipe buffer
  2352. # and the first alarm)
  2353. # - second raw write() returns EINTR (because of the second alarm)
  2354. # - subsequent write()s are successful (either partial or complete)
  2355. self.assertEqual(N, wio.write(item * N))
  2356. wio.flush()
  2357. write_finished = True
  2358. t.join()
  2359. self.assertEqual(N, sum(len(x) for x in read_results))
  2360. finally:
  2361. write_finished = True
  2362. os.close(w)
  2363. os.close(r)
  2364. # This is deliberate. If we didn't close the file descriptor
  2365. # before closing wio, wio would try to flush its internal
  2366. # buffer, and could block (in case of failure).
  2367. try:
  2368. wio.close()
  2369. except IOError as e:
  2370. if e.errno != errno.EBADF:
  2371. raise
  2372. def test_interrupterd_write_retry_buffered(self):
  2373. self.check_interrupted_write_retry(b"x", mode="wb")
  2374. def test_interrupterd_write_retry_text(self):
  2375. self.check_interrupted_write_retry("x", mode="w", encoding="latin1")
  2376. class CSignalsTest(SignalsTest):
  2377. io = io
  2378. class PySignalsTest(SignalsTest):
  2379. io = pyio
  2380. # Handling reentrancy issues would slow down _pyio even more, so the
  2381. # tests are disabled.
  2382. test_reentrant_write_buffered = None
  2383. test_reentrant_write_text = None
  2384. def test_main():
  2385. tests = (CIOTest, PyIOTest,
  2386. CBufferedReaderTest, PyBufferedReaderTest,
  2387. CBufferedWriterTest, PyBufferedWriterTest,
  2388. CBufferedRWPairTest, PyBufferedRWPairTest,
  2389. CBufferedRandomTest, PyBufferedRandomTest,
  2390. StatefulIncrementalDecoderTest,
  2391. CIncrementalNewlineDecoderTest, PyIncrementalNewlineDecoderTest,
  2392. CTextIOWrapperTest, PyTextIOWrapperTest,
  2393. CMiscIOTest, PyMiscIOTest,
  2394. CSignalsTest, PySignalsTest,
  2395. )
  2396. # Put the namespaces of the IO module we are testing and some useful mock
  2397. # classes in the __dict__ of each test.
  2398. mocks = (MockRawIO, MisbehavedRawIO, MockFileIO, CloseFailureIO,
  2399. MockNonBlockWriterIO, MockRawIOWithoutRead)
  2400. all_members = io.__all__ + ["IncrementalNewlineDecoder"]
  2401. c_io_ns = dict((name, getattr(io, name)) for name in all_members)
  2402. py_io_ns = dict((name, getattr(pyio, name)) for name in all_members)
  2403. globs = globals()
  2404. c_io_ns.update((x.__name__, globs["C" + x.__name__]) for x in mocks)
  2405. py_io_ns.update((x.__name__, globs["Py" + x.__name__]) for x in mocks)
  2406. # Avoid turning open into a bound method.
  2407. py_io_ns["open"] = pyio.OpenWrapper
  2408. for test in tests:
  2409. if test.__name__.startswith("C"):
  2410. for name, obj in c_io_ns.items():
  2411. setattr(test, name, obj)
  2412. elif test.__name__.startswith("Py"):
  2413. for name, obj in py_io_ns.items():
  2414. setattr(test, name, obj)
  2415. support.run_unittest(*tests)
  2416. if __name__ == "__main__":
  2417. test_main()