/libs/headers/gc/private/gc_pmark.h

http://github.com/nddrylliog/ooc · C++ Header · 493 lines · 297 code · 48 blank · 148 comment · 66 complexity · 303e7a6a870d20e29211477a33f5bc5b MD5 · raw file

  1. /*
  2. * Copyright (c) 1991-1994 by Xerox Corporation. All rights reserved.
  3. * Copyright (c) 2001 by Hewlett-Packard Company. All rights reserved.
  4. *
  5. * THIS MATERIAL IS PROVIDED AS IS, WITH ABSOLUTELY NO WARRANTY EXPRESSED
  6. * OR IMPLIED. ANY USE IS AT YOUR OWN RISK.
  7. *
  8. * Permission is hereby granted to use or copy this program
  9. * for any purpose, provided the above notices are retained on all copies.
  10. * Permission to modify the code and to distribute modified code is granted,
  11. * provided the above notices are retained, and a notice that the code was
  12. * modified is included with the above copyright notice.
  13. *
  14. */
  15. /* Private declarations of GC marker data structures and macros */
  16. /*
  17. * Declarations of mark stack. Needed by marker and client supplied mark
  18. * routines. Transitively include gc_priv.h.
  19. * (Note that gc_priv.h should not be included before this, since this
  20. * includes dbg_mlc.h, which wants to include gc_priv.h AFTER defining
  21. * I_HIDE_POINTERS.)
  22. */
  23. #ifndef GC_PMARK_H
  24. # define GC_PMARK_H
  25. # if defined(KEEP_BACK_PTRS) || defined(PRINT_BLACK_LIST)
  26. # include "dbg_mlc.h"
  27. # endif
  28. # ifndef GC_MARK_H
  29. # include "../gc_mark.h"
  30. # endif
  31. # ifndef GC_PRIVATE_H
  32. # include "gc_priv.h"
  33. # endif
  34. /* The real declarations of the following is in gc_priv.h, so that */
  35. /* we can avoid scanning the following table. */
  36. /*
  37. extern mark_proc GC_mark_procs[MAX_MARK_PROCS];
  38. */
  39. /*
  40. * Mark descriptor stuff that should remain private for now, mostly
  41. * because it's hard to export WORDSZ without including gcconfig.h.
  42. */
  43. # define BITMAP_BITS (WORDSZ - GC_DS_TAG_BITS)
  44. # define PROC(descr) \
  45. (GC_mark_procs[((descr) >> GC_DS_TAG_BITS) & (GC_MAX_MARK_PROCS-1)])
  46. # define ENV(descr) \
  47. ((descr) >> (GC_DS_TAG_BITS + GC_LOG_MAX_MARK_PROCS))
  48. # define MAX_ENV \
  49. (((word)1 << (WORDSZ - GC_DS_TAG_BITS - GC_LOG_MAX_MARK_PROCS)) - 1)
  50. extern unsigned GC_n_mark_procs;
  51. /* Number of mark stack entries to discard on overflow. */
  52. #define GC_MARK_STACK_DISCARDS (INITIAL_MARK_STACK_SIZE/8)
  53. typedef struct GC_ms_entry {
  54. ptr_t mse_start; /* First word of object, word aligned */
  55. GC_word mse_descr; /* Descriptor; low order two bits are tags, */
  56. /* as described in gc_mark.h. */
  57. } mse;
  58. extern size_t GC_mark_stack_size;
  59. extern mse * GC_mark_stack_limit;
  60. #ifdef PARALLEL_MARK
  61. extern mse * volatile GC_mark_stack_top;
  62. #else
  63. extern mse * GC_mark_stack_top;
  64. #endif
  65. extern mse * GC_mark_stack;
  66. #ifdef PARALLEL_MARK
  67. /*
  68. * Allow multiple threads to participate in the marking process.
  69. * This works roughly as follows:
  70. * The main mark stack never shrinks, but it can grow.
  71. *
  72. * The initiating threads holds the GC lock, and sets GC_help_wanted.
  73. *
  74. * Other threads:
  75. * 1) update helper_count (while holding mark_lock.)
  76. * 2) allocate a local mark stack
  77. * repeatedly:
  78. * 3) Steal a global mark stack entry by atomically replacing
  79. * its descriptor with 0.
  80. * 4) Copy it to the local stack.
  81. * 5) Mark on the local stack until it is empty, or
  82. * it may be profitable to copy it back.
  83. * 6) If necessary, copy local stack to global one,
  84. * holding mark lock.
  85. * 7) Stop when the global mark stack is empty.
  86. * 8) decrement helper_count (holding mark_lock).
  87. *
  88. * This is an experiment to see if we can do something along the lines
  89. * of the University of Tokyo SGC in a less intrusive, though probably
  90. * also less performant, way.
  91. */
  92. void GC_do_parallel_mark();
  93. /* inititate parallel marking. */
  94. extern GC_bool GC_help_wanted; /* Protected by mark lock */
  95. extern unsigned GC_helper_count; /* Number of running helpers. */
  96. /* Protected by mark lock */
  97. extern unsigned GC_active_count; /* Number of active helpers. */
  98. /* Protected by mark lock */
  99. /* May increase and decrease */
  100. /* within each mark cycle. But */
  101. /* once it returns to 0, it */
  102. /* stays zero for the cycle. */
  103. /* GC_mark_stack_top is also protected by mark lock. */
  104. /*
  105. * GC_notify_all_marker() is used when GC_help_wanted is first set,
  106. * when the last helper becomes inactive,
  107. * when something is added to the global mark stack, and just after
  108. * GC_mark_no is incremented.
  109. * This could be split into multiple CVs (and probably should be to
  110. * scale to really large numbers of processors.)
  111. */
  112. #endif /* PARALLEL_MARK */
  113. /* Return a pointer to within 1st page of object. */
  114. /* Set *new_hdr_p to corr. hdr. */
  115. ptr_t GC_find_start(ptr_t current, hdr *hhdr, hdr **new_hdr_p);
  116. mse * GC_signal_mark_stack_overflow(mse *msp);
  117. /* Push the object obj with corresponding heap block header hhdr onto */
  118. /* the mark stack. */
  119. # define PUSH_OBJ(obj, hhdr, mark_stack_top, mark_stack_limit) \
  120. { \
  121. register word _descr = (hhdr) -> hb_descr; \
  122. \
  123. if (_descr != 0) { \
  124. mark_stack_top++; \
  125. if (mark_stack_top >= mark_stack_limit) { \
  126. mark_stack_top = GC_signal_mark_stack_overflow(mark_stack_top); \
  127. } \
  128. mark_stack_top -> mse_start = (obj); \
  129. mark_stack_top -> mse_descr = _descr; \
  130. } \
  131. }
  132. /* Push the contents of current onto the mark stack if it is a valid */
  133. /* ptr to a currently unmarked object. Mark it. */
  134. /* If we assumed a standard-conforming compiler, we could probably */
  135. /* generate the exit_label transparently. */
  136. # define PUSH_CONTENTS(current, mark_stack_top, mark_stack_limit, \
  137. source, exit_label) \
  138. { \
  139. hdr * my_hhdr; \
  140. \
  141. HC_GET_HDR(current, my_hhdr, source, exit_label); \
  142. PUSH_CONTENTS_HDR(current, mark_stack_top, mark_stack_limit, \
  143. source, exit_label, my_hhdr, TRUE); \
  144. exit_label: ; \
  145. }
  146. /* Set mark bit, exit if it was already set. */
  147. # ifdef USE_MARK_BITS
  148. # ifdef PARALLEL_MARK
  149. /* The following may fail to exit even if the bit was already set. */
  150. /* For our uses, that's benign: */
  151. # define OR_WORD_EXIT_IF_SET(addr, bits, exit_label) \
  152. { \
  153. if (!(*(addr) & (mask))) { \
  154. AO_or((AO_t *)(addr), (mask); \
  155. } else { \
  156. goto label; \
  157. } \
  158. }
  159. # else
  160. # define OR_WORD_EXIT_IF_SET(addr, bits, exit_label) \
  161. { \
  162. word old = *(addr); \
  163. word my_bits = (bits); \
  164. if (old & my_bits) goto exit_label; \
  165. *(addr) = (old | my_bits); \
  166. }
  167. # endif /* !PARALLEL_MARK */
  168. # define SET_MARK_BIT_EXIT_IF_SET(hhdr,bit_no,exit_label) \
  169. { \
  170. word * mark_word_addr = hhdr -> hb_marks + divWORDSZ(bit_no); \
  171. \
  172. OR_WORD_EXIT_IF_SET(mark_word_addr, (word)1 << modWORDSZ(bit_no), \
  173. exit_label); \
  174. }
  175. # endif
  176. #ifdef USE_MARK_BYTES
  177. # if defined(I386) && defined(__GNUC__)
  178. # define LONG_MULT(hprod, lprod, x, y) { \
  179. asm("mull %2" : "=a"(lprod), "=d"(hprod) : "g"(y), "0"(x)); \
  180. }
  181. # else /* No in-line X86 assembly code */
  182. # define LONG_MULT(hprod, lprod, x, y) { \
  183. unsigned long long prod = (unsigned long long)x \
  184. * (unsigned long long)y; \
  185. hprod = prod >> 32; \
  186. lprod = (unsigned32)prod; \
  187. }
  188. # endif
  189. /* There is a race here, and we may set */
  190. /* the bit twice in the concurrent case. This can result in the */
  191. /* object being pushed twice. But that's only a performance issue. */
  192. # define SET_MARK_BIT_EXIT_IF_SET(hhdr,bit_no,exit_label) \
  193. { \
  194. char * mark_byte_addr = (char *)hhdr -> hb_marks + (bit_no); \
  195. char mark_byte = *mark_byte_addr; \
  196. \
  197. if (mark_byte) goto exit_label; \
  198. *mark_byte_addr = 1; \
  199. }
  200. #endif /* USE_MARK_BYTES */
  201. #ifdef PARALLEL_MARK
  202. # define INCR_MARKS(hhdr) \
  203. AO_store(&(hhdr -> hb_n_marks), AO_load(&(hhdr -> hb_n_marks))+1);
  204. #else
  205. # define INCR_MARKS(hhdr) ++(hhdr -> hb_n_marks)
  206. #endif
  207. #ifdef ENABLE_TRACE
  208. # define TRACE(source, cmd) \
  209. if (GC_trace_addr != 0 && (ptr_t)(source) == GC_trace_addr) cmd
  210. # define TRACE_TARGET(target, cmd) \
  211. if (GC_trace_addr != 0 && (target) == *(ptr_t *)GC_trace_addr) cmd
  212. #else
  213. # define TRACE(source, cmd)
  214. # define TRACE_TARGET(source, cmd)
  215. #endif
  216. /* If the mark bit corresponding to current is not set, set it, and */
  217. /* push the contents of the object on the mark stack. Current points */
  218. /* to the bginning of the object. We rely on the fact that the */
  219. /* preceding header calculation will succeed for a pointer past the */
  220. /* first page of an object, only if it is in fact a valid pointer */
  221. /* to the object. Thus we can omit the otherwise necessary tests */
  222. /* here. Note in particular that the "displ" value is the displacement */
  223. /* from the beginning of the heap block, which may itself be in the */
  224. /* interior of a large object. */
  225. #ifdef MARK_BIT_PER_GRANULE
  226. # define PUSH_CONTENTS_HDR(current, mark_stack_top, mark_stack_limit, \
  227. source, exit_label, hhdr, do_offset_check) \
  228. { \
  229. size_t displ = HBLKDISPL(current); /* Displacement in block; in bytes. */\
  230. /* displ is always within range. If current doesn't point to */ \
  231. /* first block, then we are in the all_interior_pointers case, and */ \
  232. /* it is safe to use any displacement value. */ \
  233. size_t gran_displ = BYTES_TO_GRANULES(displ); \
  234. size_t gran_offset = hhdr -> hb_map[gran_displ]; \
  235. size_t byte_offset = displ & (GRANULE_BYTES - 1); \
  236. ptr_t base = current; \
  237. /* The following always fails for large block references. */ \
  238. if (EXPECT((gran_offset | byte_offset) != 0, FALSE)) { \
  239. if (hhdr -> hb_large_block) { \
  240. /* gran_offset is bogus. */ \
  241. size_t obj_displ; \
  242. base = (ptr_t)(hhdr -> hb_block); \
  243. obj_displ = (ptr_t)(current) - base; \
  244. if (obj_displ != displ) { \
  245. GC_ASSERT(obj_displ < hhdr -> hb_sz); \
  246. /* Must be in all_interior_pointer case, not first block */ \
  247. /* already did validity check on cache miss. */ \
  248. ; \
  249. } else { \
  250. if (do_offset_check && !GC_valid_offsets[obj_displ]) { \
  251. GC_ADD_TO_BLACK_LIST_NORMAL(current, source); \
  252. goto exit_label; \
  253. } \
  254. } \
  255. gran_displ = 0; \
  256. GC_ASSERT(hhdr -> hb_sz > HBLKSIZE || \
  257. hhdr -> hb_block == HBLKPTR(current)); \
  258. GC_ASSERT((ptr_t)(hhdr -> hb_block) <= (ptr_t) current); \
  259. } else { \
  260. size_t obj_displ = GRANULES_TO_BYTES(gran_offset) \
  261. + byte_offset; \
  262. if (do_offset_check && !GC_valid_offsets[obj_displ]) { \
  263. GC_ADD_TO_BLACK_LIST_NORMAL(current, source); \
  264. goto exit_label; \
  265. } \
  266. gran_displ -= gran_offset; \
  267. base -= obj_displ; \
  268. } \
  269. } \
  270. GC_ASSERT(hhdr == GC_find_header(base)); \
  271. GC_ASSERT(gran_displ % BYTES_TO_GRANULES(hhdr -> hb_sz) == 0); \
  272. TRACE(source, GC_log_printf("GC:%d: passed validity tests\n",GC_gc_no)); \
  273. SET_MARK_BIT_EXIT_IF_SET(hhdr, gran_displ, exit_label); \
  274. TRACE(source, GC_log_printf("GC:%d: previously unmarked\n",GC_gc_no)); \
  275. TRACE_TARGET(base, \
  276. GC_log_printf("GC:%d: marking %p from %p instead\n", GC_gc_no, \
  277. base, source)); \
  278. INCR_MARKS(hhdr); \
  279. GC_STORE_BACK_PTR((ptr_t)source, base); \
  280. PUSH_OBJ(base, hhdr, mark_stack_top, mark_stack_limit); \
  281. }
  282. #endif /* MARK_BIT_PER_GRANULE */
  283. #ifdef MARK_BIT_PER_OBJ
  284. # define PUSH_CONTENTS_HDR(current, mark_stack_top, mark_stack_limit, \
  285. source, exit_label, hhdr, do_offset_check) \
  286. { \
  287. size_t displ = HBLKDISPL(current); /* Displacement in block; in bytes. */\
  288. unsigned32 low_prod, high_prod, offset_fraction; \
  289. unsigned32 inv_sz = hhdr -> hb_inv_sz; \
  290. ptr_t base = current; \
  291. LONG_MULT(high_prod, low_prod, displ, inv_sz); \
  292. /* product is > and within sz_in_bytes of displ * sz_in_bytes * 2**32 */ \
  293. if (EXPECT(low_prod >> 16 != 0, FALSE)) { \
  294. FIXME: fails if offset is a multiple of HBLKSIZE which becomes 0 \
  295. if (inv_sz == LARGE_INV_SZ) { \
  296. size_t obj_displ; \
  297. base = (ptr_t)(hhdr -> hb_block); \
  298. obj_displ = (ptr_t)(current) - base; \
  299. if (obj_displ != displ) { \
  300. GC_ASSERT(obj_displ < hhdr -> hb_sz); \
  301. /* Must be in all_interior_pointer case, not first block */ \
  302. /* already did validity check on cache miss. */ \
  303. ; \
  304. } else { \
  305. if (do_offset_check && !GC_valid_offsets[obj_displ]) { \
  306. GC_ADD_TO_BLACK_LIST_NORMAL(current, source); \
  307. goto exit_label; \
  308. } \
  309. } \
  310. GC_ASSERT(hhdr -> hb_sz > HBLKSIZE || \
  311. hhdr -> hb_block == HBLKPTR(current)); \
  312. GC_ASSERT((ptr_t)(hhdr -> hb_block) < (ptr_t) current); \
  313. } else { \
  314. /* Accurate enough if HBLKSIZE <= 2**15. */ \
  315. GC_ASSERT(HBLKSIZE <= (1 << 15)); \
  316. size_t obj_displ = (((low_prod >> 16) + 1) * (hhdr -> hb_sz)) >> 16; \
  317. if (do_offset_check && !GC_valid_offsets[obj_displ]) { \
  318. GC_ADD_TO_BLACK_LIST_NORMAL(current, source); \
  319. goto exit_label; \
  320. } \
  321. base -= obj_displ; \
  322. } \
  323. } \
  324. /* May get here for pointer to start of block not at */ \
  325. /* beginning of object. If so, it's valid, and we're fine. */ \
  326. GC_ASSERT(high_prod >= 0 && high_prod <= HBLK_OBJS(hhdr -> hb_sz)); \
  327. TRACE(source, GC_log_printf("GC:%d: passed validity tests\n",GC_gc_no)); \
  328. SET_MARK_BIT_EXIT_IF_SET(hhdr, high_prod, exit_label); \
  329. TRACE(source, GC_log_printf("GC:%d: previously unmarked\n",GC_gc_no)); \
  330. TRACE_TARGET(base, \
  331. GC_log_printf("GC:%d: marking %p from %p instead\n", GC_gc_no, \
  332. base, source)); \
  333. INCR_MARKS(hhdr); \
  334. GC_STORE_BACK_PTR((ptr_t)source, base); \
  335. PUSH_OBJ(base, hhdr, mark_stack_top, mark_stack_limit); \
  336. }
  337. #endif /* MARK_BIT_PER_OBJ */
  338. #if defined(PRINT_BLACK_LIST) || defined(KEEP_BACK_PTRS)
  339. # define PUSH_ONE_CHECKED_STACK(p, source) \
  340. GC_mark_and_push_stack(p, (ptr_t)(source))
  341. #else
  342. # define PUSH_ONE_CHECKED_STACK(p, source) \
  343. GC_mark_and_push_stack(p)
  344. #endif
  345. /*
  346. * Push a single value onto mark stack. Mark from the object pointed to by p.
  347. * Invoke FIXUP_POINTER(p) before any further processing.
  348. * P is considered valid even if it is an interior pointer.
  349. * Previously marked objects are not pushed. Hence we make progress even
  350. * if the mark stack overflows.
  351. */
  352. # if NEED_FIXUP_POINTER
  353. /* Try both the raw version and the fixed up one. */
  354. # define GC_PUSH_ONE_STACK(p, source) \
  355. if ((p) >= (ptr_t)GC_least_plausible_heap_addr \
  356. && (p) < (ptr_t)GC_greatest_plausible_heap_addr) { \
  357. PUSH_ONE_CHECKED_STACK(p, source); \
  358. } \
  359. FIXUP_POINTER(p); \
  360. if ((p) >= (ptr_t)GC_least_plausible_heap_addr \
  361. && (p) < (ptr_t)GC_greatest_plausible_heap_addr) { \
  362. PUSH_ONE_CHECKED_STACK(p, source); \
  363. }
  364. # else /* !NEED_FIXUP_POINTER */
  365. # define GC_PUSH_ONE_STACK(p, source) \
  366. if ((ptr_t)(p) >= (ptr_t)GC_least_plausible_heap_addr \
  367. && (ptr_t)(p) < (ptr_t)GC_greatest_plausible_heap_addr) { \
  368. PUSH_ONE_CHECKED_STACK(p, source); \
  369. }
  370. # endif
  371. /*
  372. * As above, but interior pointer recognition as for
  373. * normal heap pointers.
  374. */
  375. # define GC_PUSH_ONE_HEAP(p,source) \
  376. FIXUP_POINTER(p); \
  377. if ((p) >= (ptr_t)GC_least_plausible_heap_addr \
  378. && (p) < (ptr_t)GC_greatest_plausible_heap_addr) { \
  379. GC_mark_stack_top = GC_mark_and_push( \
  380. (void *)(p), GC_mark_stack_top, \
  381. GC_mark_stack_limit, (void * *)(source)); \
  382. }
  383. /* Mark starting at mark stack entry top (incl.) down to */
  384. /* mark stack entry bottom (incl.). Stop after performing */
  385. /* about one page worth of work. Return the new mark stack */
  386. /* top entry. */
  387. mse * GC_mark_from(mse * top, mse * bottom, mse *limit);
  388. #define MARK_FROM_MARK_STACK() \
  389. GC_mark_stack_top = GC_mark_from(GC_mark_stack_top, \
  390. GC_mark_stack, \
  391. GC_mark_stack + GC_mark_stack_size);
  392. /*
  393. * Mark from one finalizable object using the specified
  394. * mark proc. May not mark the object pointed to by
  395. * real_ptr. That is the job of the caller, if appropriate.
  396. * Note that this is called with the mutator running, but
  397. * with us holding the allocation lock. This is safe only if the
  398. * mutator needs tha allocation lock to reveal hidden pointers.
  399. * FIXME: Why do we need the GC_mark_state test below?
  400. */
  401. # define GC_MARK_FO(real_ptr, mark_proc) \
  402. { \
  403. (*(mark_proc))(real_ptr); \
  404. while (!GC_mark_stack_empty()) MARK_FROM_MARK_STACK(); \
  405. if (GC_mark_state != MS_NONE) { \
  406. GC_set_mark_bit(real_ptr); \
  407. while (!GC_mark_some((ptr_t)0)) {} \
  408. } \
  409. }
  410. extern GC_bool GC_mark_stack_too_small;
  411. /* We need a larger mark stack. May be */
  412. /* set by client supplied mark routines.*/
  413. typedef int mark_state_t; /* Current state of marking, as follows:*/
  414. /* Used to remember where we are during */
  415. /* concurrent marking. */
  416. /* We say something is dirty if it was */
  417. /* written since the last time we */
  418. /* retrieved dirty bits. We say it's */
  419. /* grungy if it was marked dirty in the */
  420. /* last set of bits we retrieved. */
  421. /* Invariant I: all roots and marked */
  422. /* objects p are either dirty, or point */
  423. /* to objects q that are either marked */
  424. /* or a pointer to q appears in a range */
  425. /* on the mark stack. */
  426. # define MS_NONE 0 /* No marking in progress. I holds. */
  427. /* Mark stack is empty. */
  428. # define MS_PUSH_RESCUERS 1 /* Rescuing objects are currently */
  429. /* being pushed. I holds, except */
  430. /* that grungy roots may point to */
  431. /* unmarked objects, as may marked */
  432. /* grungy objects above scan_ptr. */
  433. # define MS_PUSH_UNCOLLECTABLE 2
  434. /* I holds, except that marked */
  435. /* uncollectable objects above scan_ptr */
  436. /* may point to unmarked objects. */
  437. /* Roots may point to unmarked objects */
  438. # define MS_ROOTS_PUSHED 3 /* I holds, mark stack may be nonempty */
  439. # define MS_PARTIALLY_INVALID 4 /* I may not hold, e.g. because of M.S. */
  440. /* overflow. However marked heap */
  441. /* objects below scan_ptr point to */
  442. /* marked or stacked objects. */
  443. # define MS_INVALID 5 /* I may not hold. */
  444. extern mark_state_t GC_mark_state;
  445. #endif /* GC_PMARK_H */