/Lib/test/test_gc.py

http://unladen-swallow.googlecode.com/ · Python · 617 lines · 390 code · 85 blank · 142 comment · 31 complexity · 3b4cc69c294c0f93c1628f7c88e4e953 MD5 · raw file

  1. import unittest
  2. from test.test_support import verbose, run_unittest
  3. import sys
  4. import gc
  5. import weakref
  6. try:
  7. import _llvm
  8. except ImportError:
  9. _llvm = None
  10. ### Support code
  11. ###############################################################################
  12. # Bug 1055820 has several tests of longstanding bugs involving weakrefs and
  13. # cyclic gc.
  14. # An instance of C1055820 has a self-loop, so becomes cyclic trash when
  15. # unreachable.
  16. class C1055820(object):
  17. def __init__(self, i):
  18. self.i = i
  19. self.loop = self
  20. class GC_Detector(object):
  21. # Create an instance I. Then gc hasn't happened again so long as
  22. # I.gc_happened is false.
  23. def __init__(self):
  24. self.gc_happened = False
  25. def it_happened(ignored):
  26. self.gc_happened = True
  27. # Create a piece of cyclic trash that triggers it_happened when
  28. # gc collects it.
  29. self.wr = weakref.ref(C1055820(666), it_happened)
  30. ### Tests
  31. ###############################################################################
  32. class GCTests(unittest.TestCase):
  33. def test_list(self):
  34. l = []
  35. l.append(l)
  36. gc.collect()
  37. del l
  38. self.assertEqual(gc.collect(), 1)
  39. def test_dict(self):
  40. d = {}
  41. d[1] = d
  42. gc.collect()
  43. del d
  44. self.assertEqual(gc.collect(), 1)
  45. def test_tuple(self):
  46. # since tuples are immutable we close the loop with a list
  47. l = []
  48. t = (l,)
  49. l.append(t)
  50. gc.collect()
  51. del t
  52. del l
  53. self.assertEqual(gc.collect(), 2)
  54. def test_class(self):
  55. class A:
  56. pass
  57. A.a = A
  58. gc.collect()
  59. del A
  60. self.assertNotEqual(gc.collect(), 0)
  61. def test_newstyleclass(self):
  62. class A(object):
  63. pass
  64. gc.collect()
  65. del A
  66. self.assertNotEqual(gc.collect(), 0)
  67. def test_instance(self):
  68. class A:
  69. pass
  70. a = A()
  71. a.a = a
  72. gc.collect()
  73. del a
  74. self.assertNotEqual(gc.collect(), 0)
  75. def test_newinstance(self):
  76. class A(object):
  77. pass
  78. a = A()
  79. a.a = a
  80. gc.collect()
  81. del a
  82. self.assertNotEqual(gc.collect(), 0)
  83. class B(list):
  84. pass
  85. class C(B, A):
  86. pass
  87. a = C()
  88. a.a = a
  89. gc.collect()
  90. del a
  91. self.assertNotEqual(gc.collect(), 0)
  92. del B, C
  93. # If we don't clear the feedback array, the FDO system will hold
  94. # references to B and C, which prevent gc.collect() from collecting
  95. # them.
  96. if _llvm:
  97. _llvm.clear_feedback(self.test_newinstance)
  98. self.assertNotEqual(gc.collect(), 0)
  99. A.a = A()
  100. del A
  101. self.assertNotEqual(gc.collect(), 0)
  102. self.assertEqual(gc.collect(), 0)
  103. def test_method(self):
  104. # Tricky: self.__init__ is a bound method, it references the instance.
  105. class A:
  106. def __init__(self):
  107. self.init = self.__init__
  108. a = A()
  109. gc.collect()
  110. del a
  111. self.assertNotEqual(gc.collect(), 0)
  112. def test_finalizer(self):
  113. # A() is uncollectable if it is part of a cycle, make sure it shows up
  114. # in gc.garbage.
  115. class A:
  116. def __del__(self): pass
  117. class B:
  118. pass
  119. a = A()
  120. a.a = a
  121. id_a = id(a)
  122. b = B()
  123. b.b = b
  124. gc.collect()
  125. del a
  126. del b
  127. self.assertNotEqual(gc.collect(), 0)
  128. for obj in gc.garbage:
  129. if id(obj) == id_a:
  130. del obj.a
  131. break
  132. else:
  133. self.fail("didn't find obj in garbage (finalizer)")
  134. gc.garbage.remove(obj)
  135. def test_finalizer_newclass(self):
  136. # A() is uncollectable if it is part of a cycle, make sure it shows up
  137. # in gc.garbage.
  138. class A(object):
  139. def __del__(self): pass
  140. class B(object):
  141. pass
  142. a = A()
  143. a.a = a
  144. id_a = id(a)
  145. b = B()
  146. b.b = b
  147. gc.collect()
  148. del a
  149. del b
  150. self.assertNotEqual(gc.collect(), 0)
  151. for obj in gc.garbage:
  152. if id(obj) == id_a:
  153. del obj.a
  154. break
  155. else:
  156. self.fail("didn't find obj in garbage (finalizer)")
  157. gc.garbage.remove(obj)
  158. def test_function(self):
  159. # Tricky: f -> d -> f, code should call d.clear() after the exec to
  160. # break the cycle.
  161. d = {}
  162. exec("def f(): pass\n") in d
  163. gc.collect()
  164. del d
  165. self.assertEqual(gc.collect(), 2)
  166. def test_frame(self):
  167. def f():
  168. frame = sys._getframe()
  169. gc.collect()
  170. f()
  171. self.assertEqual(gc.collect(), 1)
  172. def test_saveall(self):
  173. # Verify that cyclic garbage like lists show up in gc.garbage if the
  174. # SAVEALL option is enabled.
  175. # First make sure we don't save away other stuff that just happens to
  176. # be waiting for collection.
  177. gc.collect()
  178. # if this fails, someone else created immortal trash
  179. self.assertEqual(gc.garbage, [])
  180. L = []
  181. L.append(L)
  182. id_L = id(L)
  183. debug = gc.get_debug()
  184. gc.set_debug(debug | gc.DEBUG_SAVEALL)
  185. del L
  186. gc.collect()
  187. gc.set_debug(debug)
  188. self.assertEqual(len(gc.garbage), 1)
  189. obj = gc.garbage.pop()
  190. self.assertEqual(id(obj), id_L)
  191. def test_del(self):
  192. # __del__ methods can trigger collection, make this to happen
  193. thresholds = gc.get_threshold()
  194. gc.enable()
  195. gc.set_threshold(1)
  196. class A:
  197. def __del__(self):
  198. dir(self)
  199. a = A()
  200. del a
  201. gc.disable()
  202. gc.set_threshold(*thresholds)
  203. def test_del_newclass(self):
  204. # __del__ methods can trigger collection, make this to happen
  205. thresholds = gc.get_threshold()
  206. gc.enable()
  207. gc.set_threshold(1)
  208. class A(object):
  209. def __del__(self):
  210. dir(self)
  211. a = A()
  212. del a
  213. gc.disable()
  214. gc.set_threshold(*thresholds)
  215. # The following two tests are fragile:
  216. # They precisely count the number of allocations,
  217. # which is highly implementation-dependent.
  218. # For example:
  219. # - disposed tuples are not freed, but reused
  220. # - the call to assertEqual somehow avoids building its args tuple
  221. def test_get_count(self):
  222. # Avoid future allocation of method object
  223. assertEqual = self.assertEqual
  224. gc.collect()
  225. assertEqual(gc.get_count(), (0, 0, 0))
  226. a = dict()
  227. # since gc.collect(), we created two objects:
  228. # the dict, and the tuple returned by get_count()
  229. assertEqual(gc.get_count(), (2, 0, 0))
  230. def test_collect_generations(self):
  231. # Avoid future allocation of method object
  232. assertEqual = self.assertEqual
  233. gc.collect()
  234. a = dict()
  235. gc.collect(0)
  236. assertEqual(gc.get_count(), (0, 1, 0))
  237. gc.collect(1)
  238. assertEqual(gc.get_count(), (0, 0, 1))
  239. gc.collect(2)
  240. assertEqual(gc.get_count(), (0, 0, 0))
  241. def test_trashcan(self):
  242. class Ouch:
  243. n = 0
  244. def __del__(self):
  245. Ouch.n = Ouch.n + 1
  246. if Ouch.n % 17 == 0:
  247. gc.collect()
  248. # "trashcan" is a hack to prevent stack overflow when deallocating
  249. # very deeply nested tuples etc. It works in part by abusing the
  250. # type pointer and refcount fields, and that can yield horrible
  251. # problems when gc tries to traverse the structures.
  252. # If this test fails (as it does in 2.0, 2.1 and 2.2), it will
  253. # most likely die via segfault.
  254. # Note: In 2.3 the possibility for compiling without cyclic gc was
  255. # removed, and that in turn allows the trashcan mechanism to work
  256. # via much simpler means (e.g., it never abuses the type pointer or
  257. # refcount fields anymore). Since it's much less likely to cause a
  258. # problem now, the various constants in this expensive (we force a lot
  259. # of full collections) test are cut back from the 2.2 version.
  260. gc.enable()
  261. N = 150
  262. for count in range(2):
  263. t = []
  264. for i in range(N):
  265. t = [t, Ouch()]
  266. u = []
  267. for i in range(N):
  268. u = [u, Ouch()]
  269. v = {}
  270. for i in range(N):
  271. v = {1: v, 2: Ouch()}
  272. gc.disable()
  273. def test_boom(self):
  274. class Boom:
  275. def __getattr__(self, someattribute):
  276. del self.attr
  277. raise AttributeError
  278. a = Boom()
  279. b = Boom()
  280. a.attr = b
  281. b.attr = a
  282. gc.collect()
  283. garbagelen = len(gc.garbage)
  284. del a, b
  285. # a<->b are in a trash cycle now. Collection will invoke
  286. # Boom.__getattr__ (to see whether a and b have __del__ methods), and
  287. # __getattr__ deletes the internal "attr" attributes as a side effect.
  288. # That causes the trash cycle to get reclaimed via refcounts falling to
  289. # 0, thus mutating the trash graph as a side effect of merely asking
  290. # whether __del__ exists. This used to (before 2.3b1) crash Python.
  291. # Now __getattr__ isn't called.
  292. self.assertEqual(gc.collect(), 4)
  293. self.assertEqual(len(gc.garbage), garbagelen)
  294. def test_boom2(self):
  295. class Boom2:
  296. def __init__(self):
  297. self.x = 0
  298. def __getattr__(self, someattribute):
  299. self.x += 1
  300. if self.x > 1:
  301. del self.attr
  302. raise AttributeError
  303. a = Boom2()
  304. b = Boom2()
  305. a.attr = b
  306. b.attr = a
  307. gc.collect()
  308. garbagelen = len(gc.garbage)
  309. del a, b
  310. # Much like test_boom(), except that __getattr__ doesn't break the
  311. # cycle until the second time gc checks for __del__. As of 2.3b1,
  312. # there isn't a second time, so this simply cleans up the trash cycle.
  313. # We expect a, b, a.__dict__ and b.__dict__ (4 objects) to get
  314. # reclaimed this way.
  315. self.assertEqual(gc.collect(), 4)
  316. self.assertEqual(len(gc.garbage), garbagelen)
  317. def test_boom_new(self):
  318. # boom__new and boom2_new are exactly like boom and boom2, except use
  319. # new-style classes.
  320. class Boom_New(object):
  321. def __getattr__(self, someattribute):
  322. del self.attr
  323. raise AttributeError
  324. a = Boom_New()
  325. b = Boom_New()
  326. a.attr = b
  327. b.attr = a
  328. gc.collect()
  329. garbagelen = len(gc.garbage)
  330. del a, b
  331. self.assertEqual(gc.collect(), 4)
  332. self.assertEqual(len(gc.garbage), garbagelen)
  333. def test_boom2_new(self):
  334. class Boom2_New(object):
  335. def __init__(self):
  336. self.x = 0
  337. def __getattr__(self, someattribute):
  338. self.x += 1
  339. if self.x > 1:
  340. del self.attr
  341. raise AttributeError
  342. a = Boom2_New()
  343. b = Boom2_New()
  344. a.attr = b
  345. b.attr = a
  346. gc.collect()
  347. garbagelen = len(gc.garbage)
  348. del a, b
  349. self.assertEqual(gc.collect(), 4)
  350. self.assertEqual(len(gc.garbage), garbagelen)
  351. def test_get_referents(self):
  352. alist = [1, 3, 5]
  353. got = gc.get_referents(alist)
  354. got.sort()
  355. self.assertEqual(got, alist)
  356. atuple = tuple(alist)
  357. got = gc.get_referents(atuple)
  358. got.sort()
  359. self.assertEqual(got, alist)
  360. adict = {1: 3, 5: 7}
  361. expected = [1, 3, 5, 7]
  362. got = gc.get_referents(adict)
  363. got.sort()
  364. self.assertEqual(got, expected)
  365. got = gc.get_referents([1, 2], {3: 4}, (0, 0, 0))
  366. got.sort()
  367. self.assertEqual(got, [0, 0] + range(5))
  368. self.assertEqual(gc.get_referents(1, 'a', 4j), [])
  369. def test_bug1055820b(self):
  370. # Corresponds to temp2b.py in the bug report.
  371. ouch = []
  372. def callback(ignored):
  373. ouch[:] = [wr() for wr in WRs]
  374. Cs = [C1055820(i) for i in range(2)]
  375. WRs = [weakref.ref(c, callback) for c in Cs]
  376. c = None
  377. gc.collect()
  378. self.assertEqual(len(ouch), 0)
  379. # Make the two instances trash, and collect again. The bug was that
  380. # the callback materialized a strong reference to an instance, but gc
  381. # cleared the instance's dict anyway.
  382. Cs = None
  383. gc.collect()
  384. self.assertEqual(len(ouch), 2) # else the callbacks didn't run
  385. for x in ouch:
  386. # If the callback resurrected one of these guys, the instance
  387. # would be damaged, with an empty __dict__.
  388. self.assertEqual(x, None)
  389. class GCTogglingTests(unittest.TestCase):
  390. def setUp(self):
  391. gc.enable()
  392. def tearDown(self):
  393. gc.disable()
  394. def test_bug1055820c(self):
  395. # Corresponds to temp2c.py in the bug report. This is pretty
  396. # elaborate.
  397. c0 = C1055820(0)
  398. # Move c0 into generation 2.
  399. gc.collect()
  400. c1 = C1055820(1)
  401. c1.keep_c0_alive = c0
  402. del c0.loop # now only c1 keeps c0 alive
  403. c2 = C1055820(2)
  404. c2wr = weakref.ref(c2) # no callback!
  405. ouch = []
  406. def callback(ignored):
  407. ouch[:] = [c2wr()]
  408. # The callback gets associated with a wr on an object in generation 2.
  409. c0wr = weakref.ref(c0, callback)
  410. c0 = c1 = c2 = None
  411. # What we've set up: c0, c1, and c2 are all trash now. c0 is in
  412. # generation 2. The only thing keeping it alive is that c1 points to
  413. # it. c1 and c2 are in generation 0, and are in self-loops. There's a
  414. # global weakref to c2 (c2wr), but that weakref has no callback.
  415. # There's also a global weakref to c0 (c0wr), and that does have a
  416. # callback, and that callback references c2 via c2wr().
  417. #
  418. # c0 has a wr with callback, which references c2wr
  419. # ^
  420. # |
  421. # | Generation 2 above dots
  422. #. . . . . . . .|. . . . . . . . . . . . . . . . . . . . . . . .
  423. # | Generation 0 below dots
  424. # |
  425. # |
  426. # ^->c1 ^->c2 has a wr but no callback
  427. # | | | |
  428. # <--v <--v
  429. #
  430. # So this is the nightmare: when generation 0 gets collected, we see
  431. # that c2 has a callback-free weakref, and c1 doesn't even have a
  432. # weakref. Collecting generation 0 doesn't see c0 at all, and c0 is
  433. # the only object that has a weakref with a callback. gc clears c1
  434. # and c2. Clearing c1 has the side effect of dropping the refcount on
  435. # c0 to 0, so c0 goes away (despite that it's in an older generation)
  436. # and c0's wr callback triggers. That in turn materializes a reference
  437. # to c2 via c2wr(), but c2 gets cleared anyway by gc.
  438. # We want to let gc happen "naturally", to preserve the distinction
  439. # between generations.
  440. junk = []
  441. i = 0
  442. detector = GC_Detector()
  443. while not detector.gc_happened:
  444. i += 1
  445. if i > 10000:
  446. self.fail("gc didn't happen after 10000 iterations")
  447. self.assertEqual(len(ouch), 0)
  448. junk.append([]) # this will eventually trigger gc
  449. self.assertEqual(len(ouch), 1) # else the callback wasn't invoked
  450. for x in ouch:
  451. # If the callback resurrected c2, the instance would be damaged,
  452. # with an empty __dict__.
  453. self.assertEqual(x, None)
  454. def test_bug1055820d(self):
  455. # Corresponds to temp2d.py in the bug report. This is very much like
  456. # test_bug1055820c, but uses a __del__ method instead of a weakref
  457. # callback to sneak in a resurrection of cyclic trash.
  458. ouch = []
  459. class D(C1055820):
  460. def __del__(self):
  461. ouch[:] = [c2wr()]
  462. d0 = D(0)
  463. # Move all the above into generation 2.
  464. gc.collect()
  465. c1 = C1055820(1)
  466. c1.keep_d0_alive = d0
  467. del d0.loop # now only c1 keeps d0 alive
  468. c2 = C1055820(2)
  469. c2wr = weakref.ref(c2) # no callback!
  470. d0 = c1 = c2 = None
  471. # What we've set up: d0, c1, and c2 are all trash now. d0 is in
  472. # generation 2. The only thing keeping it alive is that c1 points to
  473. # it. c1 and c2 are in generation 0, and are in self-loops. There's
  474. # a global weakref to c2 (c2wr), but that weakref has no callback.
  475. # There are no other weakrefs.
  476. #
  477. # d0 has a __del__ method that references c2wr
  478. # ^
  479. # |
  480. # | Generation 2 above dots
  481. #. . . . . . . .|. . . . . . . . . . . . . . . . . . . . . . . .
  482. # | Generation 0 below dots
  483. # |
  484. # |
  485. # ^->c1 ^->c2 has a wr but no callback
  486. # | | | |
  487. # <--v <--v
  488. #
  489. # So this is the nightmare: when generation 0 gets collected, we see
  490. # that c2 has a callback-free weakref, and c1 doesn't even have a
  491. # weakref. Collecting generation 0 doesn't see d0 at all. gc clears
  492. # c1 and c2. Clearing c1 has the side effect of dropping the refcount
  493. # on d0 to 0, so d0 goes away (despite that it's in an older
  494. # generation) and d0's __del__ triggers. That in turn materializes
  495. # a reference to c2 via c2wr(), but c2 gets cleared anyway by gc.
  496. # We want to let gc happen "naturally", to preserve the distinction
  497. # between generations.
  498. detector = GC_Detector()
  499. junk = []
  500. i = 0
  501. while not detector.gc_happened:
  502. i += 1
  503. if i > 10000:
  504. self.fail("gc didn't happen after 10000 iterations")
  505. self.assertEqual(len(ouch), 0)
  506. junk.append([]) # this will eventually trigger gc
  507. self.assertEqual(len(ouch), 1) # else __del__ wasn't invoked
  508. for x in ouch:
  509. # If __del__ resurrected c2, the instance would be damaged, with an
  510. # empty __dict__.
  511. self.assertEqual(x, None)
  512. def test_main():
  513. enabled = gc.isenabled()
  514. gc.disable()
  515. assert not gc.isenabled()
  516. debug = gc.get_debug()
  517. gc.set_debug(debug & ~gc.DEBUG_LEAK) # this test is supposed to leak
  518. try:
  519. gc.collect() # Delete 2nd generation garbage
  520. run_unittest(GCTests, GCTogglingTests)
  521. finally:
  522. gc.set_debug(debug)
  523. # test gc.enable() even if GC is disabled by default
  524. if verbose:
  525. print "restoring automatic collection"
  526. # make sure to always test gc.enable()
  527. gc.enable()
  528. assert gc.isenabled()
  529. if not enabled:
  530. gc.disable()
  531. if __name__ == "__main__":
  532. test_main()