PageRenderTime 43ms CodeModel.GetById 12ms RepoModel.GetById 1ms app.codeStats 0ms

/trunk/src/lockfree/gc.c

#
C | 672 lines | 443 code | 112 blank | 117 comment | 83 complexity | 3f541404ec8cb2ed64567b97e73bf388 MD5 | raw file
Possible License(s): BSD-3-Clause
  1. /******************************************************************************
  2. * gc.c
  3. *
  4. * A fully recycling epoch-based garbage collector. Works by counting
  5. * threads in and out of critical regions, to work out when
  6. * garbage queues can be fully deleted.
  7. *
  8. * Copyright (c) 2001-2003, K A Fraser
  9. *
  10. * All rights reserved.
  11. *
  12. * Redistribution and use in source and binary forms, with or without
  13. * modification, are permitted provided that the following conditions are met:
  14. *
  15. * * Redistributions of source code must retain the above copyright
  16. * notice, this list of conditions and the following disclaimer.
  17. *
  18. * * Redistributions in binary form must reproduce the above copyright
  19. * notice, this list of conditions and the following disclaimer in the
  20. * documentation and/or other materials provided with the distribution.
  21. *
  22. * * The name of the author may not be used to endorse or promote products
  23. * derived from this software without specific prior written permission.
  24. *
  25. * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
  26. * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
  27. * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
  28. * DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT,
  29. * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
  30. * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
  31. * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  32. * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
  33. * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
  34. * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
  35. * POSSIBILITY OF SUCH DAMAGE.
  36. */
  37. #include <assert.h>
  38. #include <stdio.h>
  39. #include <stdlib.h>
  40. #include <string.h>
  41. #include <sys/mman.h>
  42. #include <unistd.h>
  43. #include "portable_defns.h"
  44. #include "gc.h"
  45. /*#define MINIMAL_GC*/
  46. /*#define YIELD_TO_HELP_PROGRESS*/
  47. #define PROFILE_GC
  48. /* Recycled nodes are filled with this value if WEAK_MEM_ORDER. */
  49. #define INVALID_BYTE 0
  50. #define INITIALISE_NODES(_p,_c) memset((_p), INVALID_BYTE, (_c));
  51. /* Number of unique block sizes we can deal with. */
  52. #define MAX_SIZES 20
  53. #define MAX_HOOKS 4
  54. /*
  55. * The initial number of allocation chunks for each per-blocksize list.
  56. * Popular allocation lists will steadily increase the allocation unit
  57. * in line with demand.
  58. */
  59. #define ALLOC_CHUNKS_PER_LIST 10
  60. /*
  61. * How many times should a thread call gc_enter(), seeing the same epoch
  62. * each time, before it makes a reclaim attempt?
  63. */
  64. #define ENTRIES_PER_RECLAIM_ATTEMPT 100
  65. /*
  66. * 0: current epoch -- threads are moving to this;
  67. * -1: some threads may still throw garbage into this epoch;
  68. * -2: no threads can see this epoch => we can zero garbage lists;
  69. * -3: all threads see zeros in these garbage lists => move to alloc lists.
  70. */
  71. #ifdef WEAK_MEM_ORDER
  72. #define NR_EPOCHS 4
  73. #else
  74. #define NR_EPOCHS 3
  75. #endif
  76. /*
  77. * A chunk amortises the cost of allocation from shared lists. It also
  78. * helps when zeroing nodes, as it increases per-cacheline pointer density
  79. * and means that node locations don't need to be brought into the cache
  80. * (most architectures have a non-temporal store instruction).
  81. */
  82. #define BLKS_PER_CHUNK 100
  83. typedef struct chunk_st chunk_t;
  84. struct chunk_st
  85. {
  86. chunk_t *next; /* chunk chaining */
  87. unsigned int i; /* the next entry in blk[] to use */
  88. void *blk[BLKS_PER_CHUNK];
  89. };
  90. static struct gc_global_st
  91. {
  92. CACHE_PAD(0);
  93. /* The current epoch. */
  94. VOLATILE unsigned int current;
  95. CACHE_PAD(1);
  96. /* Exclusive access to gc_reclaim(). */
  97. VOLATILE unsigned int inreclaim;
  98. CACHE_PAD(2);
  99. /*
  100. * RUN-TIME CONSTANTS (to first approximation)
  101. */
  102. /* Memory page size, in bytes. */
  103. unsigned int page_size;
  104. /* Node sizes (run-time constants). */
  105. int nr_sizes;
  106. int blk_sizes[MAX_SIZES];
  107. /* Registered epoch hooks. */
  108. int nr_hooks;
  109. hook_fn_t hook_fns[MAX_HOOKS];
  110. CACHE_PAD(3);
  111. /*
  112. * DATA WE MAY HIT HARD
  113. */
  114. /* Chain of free, empty chunks. */
  115. chunk_t * VOLATILE free_chunks;
  116. /* Main allocation lists. */
  117. chunk_t * VOLATILE alloc[MAX_SIZES];
  118. VOLATILE unsigned int alloc_size[MAX_SIZES];
  119. #ifdef PROFILE_GC
  120. VOLATILE unsigned int total_size;
  121. VOLATILE unsigned int allocations;
  122. #endif
  123. } gc_global;
  124. /* Per-thread state. */
  125. struct gc_st
  126. {
  127. /* Epoch that this thread sees. */
  128. unsigned int epoch;
  129. /* Number of calls to gc_entry() since last gc_reclaim() attempt. */
  130. unsigned int entries_since_reclaim;
  131. #ifdef YIELD_TO_HELP_PROGRESS
  132. /* Number of calls to gc_reclaim() since we last yielded. */
  133. unsigned int reclaim_attempts_since_yield;
  134. #endif
  135. /* Used by gc_async_barrier(). */
  136. void *async_page;
  137. int async_page_state;
  138. /* Garbage lists. */
  139. chunk_t *garbage[NR_EPOCHS][MAX_SIZES];
  140. chunk_t *garbage_tail[NR_EPOCHS][MAX_SIZES];
  141. chunk_t *chunk_cache;
  142. /* Local allocation lists. */
  143. chunk_t *alloc[MAX_SIZES];
  144. unsigned int alloc_chunks[MAX_SIZES];
  145. /* Hook pointer lists. */
  146. chunk_t *hook[NR_EPOCHS][MAX_HOOKS];
  147. };
  148. #define MEM_FAIL(_s) \
  149. do { \
  150. fprintf(stderr, "OUT OF MEMORY: %d bytes at line %d\n", (_s), __LINE__); \
  151. exit(1); \
  152. } while ( 0 )
  153. /* Allocate more empty chunks from the heap. */
  154. #define CHUNKS_PER_ALLOC 1000
  155. static chunk_t *alloc_more_chunks(void)
  156. {
  157. int i;
  158. chunk_t *h, *p;
  159. h = p = ALIGNED_ALLOC(CHUNKS_PER_ALLOC * sizeof(*h));
  160. if ( h == NULL ) MEM_FAIL( (int)( CHUNKS_PER_ALLOC * sizeof(*h) ) );
  161. for ( i = 1; i < CHUNKS_PER_ALLOC; i++ )
  162. {
  163. p->next = p + 1;
  164. p++;
  165. }
  166. p->next = h;
  167. return(h);
  168. }
  169. /* Put a chain of chunks onto a list. */
  170. static void add_chunks_to_list(chunk_t *ch, chunk_t *head)
  171. {
  172. chunk_t *h_next, *new_h_next, *ch_next;
  173. ch_next = ch->next;
  174. new_h_next = head->next;
  175. do { ch->next = h_next = new_h_next; WMB_NEAR_CAS(); }
  176. while ( (new_h_next = CASPO(&head->next, h_next, ch_next)) != h_next );
  177. }
  178. /* Allocate a chain of @n empty chunks. Pointers may be garbage. */
  179. static chunk_t *get_empty_chunks(int n)
  180. {
  181. int i;
  182. chunk_t *new_rh, *rh, *rt, *head;
  183. retry:
  184. head = gc_global.free_chunks;
  185. new_rh = head->next;
  186. do {
  187. rh = new_rh;
  188. rt = head;
  189. WEAK_DEP_ORDER_RMB();
  190. for ( i = 0; i < n; i++ )
  191. {
  192. if ( (rt = rt->next) == head )
  193. {
  194. /* Allocate some more chunks. */
  195. add_chunks_to_list(alloc_more_chunks(), head);
  196. goto retry;
  197. }
  198. }
  199. }
  200. while ( (new_rh = CASPO(&head->next, rh, rt->next)) != rh );
  201. rt->next = rh;
  202. return(rh);
  203. }
  204. /* Get @n filled chunks, pointing at blocks of @sz bytes each. */
  205. static chunk_t *get_filled_chunks(int n, int sz)
  206. {
  207. chunk_t *h, *p;
  208. char *node;
  209. int i;
  210. #ifdef PROFILE_GC
  211. ADD_TO(gc_global.total_size, n * BLKS_PER_CHUNK * sz);
  212. ADD_TO(gc_global.allocations, 1);
  213. #endif
  214. node = ALIGNED_ALLOC(n * BLKS_PER_CHUNK * sz);
  215. if ( node == NULL ) MEM_FAIL(n * BLKS_PER_CHUNK * sz);
  216. #ifdef WEAK_MEM_ORDER
  217. INITIALISE_NODES(node, n * BLKS_PER_CHUNK * sz);
  218. #endif
  219. h = p = get_empty_chunks(n);
  220. do {
  221. p->i = BLKS_PER_CHUNK;
  222. for ( i = 0; i < BLKS_PER_CHUNK; i++ )
  223. {
  224. p->blk[i] = node;
  225. node += sz;
  226. }
  227. }
  228. while ( (p = p->next) != h );
  229. return(h);
  230. }
  231. /*
  232. * gc_async_barrier: Cause an asynchronous barrier in all other threads. We do
  233. * this by causing a TLB shootdown to be propagated to all other processors.
  234. * Each time such an action is required, this function calls:
  235. * mprotect(async_page, <page size>, <new flags>)
  236. * Each thread's state contains a memory page dedicated for this purpose.
  237. */
  238. #ifdef WEAK_MEM_ORDER
  239. static void gc_async_barrier(gc_t *gc)
  240. {
  241. mprotect(gc->async_page, gc_global.page_size,
  242. gc->async_page_state ? PROT_READ : PROT_NONE);
  243. gc->async_page_state = !gc->async_page_state;
  244. }
  245. #else
  246. #define gc_async_barrier(_g) ((void)0)
  247. #endif
  248. /* Grab a level @i allocation chunk from main chain. */
  249. static chunk_t *get_alloc_chunk(gc_t *gc, int i)
  250. {
  251. chunk_t *alloc, *p, *new_p, *nh;
  252. unsigned int sz;
  253. alloc = gc_global.alloc[i];
  254. new_p = alloc->next;
  255. do {
  256. p = new_p;
  257. while ( p == alloc )
  258. {
  259. sz = gc_global.alloc_size[i];
  260. nh = get_filled_chunks(sz, gc_global.blk_sizes[i]);
  261. ADD_TO(gc_global.alloc_size[i], sz >> 3);
  262. gc_async_barrier(gc);
  263. add_chunks_to_list(nh, alloc);
  264. p = alloc->next;
  265. }
  266. WEAK_DEP_ORDER_RMB();
  267. }
  268. while ( (new_p = CASPO(&alloc->next, p, p->next)) != p );
  269. p->next = p;
  270. assert(p->i == BLKS_PER_CHUNK);
  271. return(p);
  272. }
  273. #ifndef MINIMAL_GC
  274. /*
  275. * gc_reclaim: Scans the list of struct gc_perthread looking for the lowest
  276. * maximum epoch number seen by a thread that's in the list code. If it's the
  277. * current epoch, the "nearly-free" lists from the previous epoch are
  278. * reclaimed, and the epoch is incremented.
  279. */
  280. static void gc_reclaim(void)
  281. {
  282. ptst_t *ptst, *first_ptst, *our_ptst = NULL;
  283. gc_t *gc = NULL;
  284. unsigned long curr_epoch;
  285. chunk_t *ch, *t;
  286. int two_ago, three_ago, i, j;
  287. /* Barrier to entering the reclaim critical section. */
  288. if ( gc_global.inreclaim || CASIO(&gc_global.inreclaim, 0, 1) ) return;
  289. /*
  290. * Grab first ptst structure *before* barrier -- prevent bugs
  291. * on weak-ordered architectures.
  292. */
  293. first_ptst = ptst_first();
  294. MB();
  295. curr_epoch = gc_global.current;
  296. /* Have all threads seen the current epoch, or not in mutator code? */
  297. for ( ptst = first_ptst; ptst != NULL; ptst = ptst_next(ptst) )
  298. {
  299. if ( (ptst->count > 1) && (ptst->gc->epoch != curr_epoch) ) goto out;
  300. }
  301. /*
  302. * Three-epoch-old garbage lists move to allocation lists.
  303. * Two-epoch-old garbage lists are cleaned out.
  304. */
  305. two_ago = (curr_epoch+2) % NR_EPOCHS;
  306. three_ago = (curr_epoch+1) % NR_EPOCHS;
  307. if ( gc_global.nr_hooks != 0 )
  308. our_ptst = (ptst_t *)pthread_getspecific(ptst_key);
  309. for ( ptst = first_ptst; ptst != NULL; ptst = ptst_next(ptst) )
  310. {
  311. gc = ptst->gc;
  312. for ( i = 0; i < gc_global.nr_sizes; i++ )
  313. {
  314. #ifdef WEAK_MEM_ORDER
  315. int sz = gc_global.blk_sizes[i];
  316. if ( gc->garbage[two_ago][i] != NULL )
  317. {
  318. chunk_t *head = gc->garbage[two_ago][i];
  319. ch = head;
  320. do {
  321. int j;
  322. for ( j = 0; j < ch->i; j++ )
  323. INITIALISE_NODES(ch->blk[j], sz);
  324. }
  325. while ( (ch = ch->next) != head );
  326. }
  327. #endif
  328. /* NB. Leave one chunk behind, as it is probably not yet full. */
  329. t = gc->garbage[three_ago][i];
  330. if ( (t == NULL) || ((ch = t->next) == t) ) continue;
  331. gc->garbage_tail[three_ago][i]->next = ch;
  332. gc->garbage_tail[three_ago][i] = t;
  333. t->next = t;
  334. add_chunks_to_list(ch, gc_global.alloc[i]);
  335. }
  336. for ( i = 0; i < gc_global.nr_hooks; i++ )
  337. {
  338. hook_fn_t fn = gc_global.hook_fns[i];
  339. ch = gc->hook[three_ago][i];
  340. if ( ch == NULL ) continue;
  341. gc->hook[three_ago][i] = NULL;
  342. t = ch;
  343. do { for ( j = 0; j < t->i; j++ ) fn(our_ptst, t->blk[j]); }
  344. while ( (t = t->next) != ch );
  345. add_chunks_to_list(ch, gc_global.free_chunks);
  346. }
  347. }
  348. /* Update current epoch. */
  349. WMB();
  350. gc_global.current = (curr_epoch+1) % NR_EPOCHS;
  351. out:
  352. gc_global.inreclaim = 0;
  353. }
  354. #endif /* MINIMAL_GC */
  355. void *gc_alloc(ptst_t *ptst, int alloc_id)
  356. {
  357. gc_t *gc = ptst->gc;
  358. chunk_t *ch;
  359. ch = gc->alloc[alloc_id];
  360. if ( ch->i == 0 )
  361. {
  362. if ( gc->alloc_chunks[alloc_id]++ == 100 )
  363. {
  364. gc->alloc_chunks[alloc_id] = 0;
  365. add_chunks_to_list(ch, gc_global.free_chunks);
  366. gc->alloc[alloc_id] = ch = get_alloc_chunk(gc, alloc_id);
  367. }
  368. else
  369. {
  370. chunk_t *och = ch;
  371. ch = get_alloc_chunk(gc, alloc_id);
  372. ch->next = och->next;
  373. och->next = ch;
  374. gc->alloc[alloc_id] = ch;
  375. }
  376. }
  377. return ch->blk[--ch->i];
  378. }
  379. static chunk_t *chunk_from_cache(gc_t *gc)
  380. {
  381. chunk_t *ch = gc->chunk_cache, *p = ch->next;
  382. if ( ch == p )
  383. {
  384. gc->chunk_cache = get_empty_chunks(100);
  385. }
  386. else
  387. {
  388. ch->next = p->next;
  389. p->next = p;
  390. }
  391. p->i = 0;
  392. return(p);
  393. }
  394. void gc_free(ptst_t *ptst, void *p, int alloc_id)
  395. {
  396. #ifndef MINIMAL_GC
  397. gc_t *gc = ptst->gc;
  398. chunk_t *prev, *new, *ch = gc->garbage[gc->epoch][alloc_id];
  399. if ( ch == NULL )
  400. {
  401. gc->garbage[gc->epoch][alloc_id] = ch = chunk_from_cache(gc);
  402. gc->garbage_tail[gc->epoch][alloc_id] = ch;
  403. }
  404. else if ( ch->i == BLKS_PER_CHUNK )
  405. {
  406. prev = gc->garbage_tail[gc->epoch][alloc_id];
  407. new = chunk_from_cache(gc);
  408. gc->garbage[gc->epoch][alloc_id] = new;
  409. new->next = ch;
  410. prev->next = new;
  411. ch = new;
  412. }
  413. ch->blk[ch->i++] = p;
  414. #endif
  415. }
  416. void gc_add_ptr_to_hook_list(ptst_t *ptst, void *ptr, int hook_id)
  417. {
  418. gc_t *gc = ptst->gc;
  419. chunk_t *och, *ch = gc->hook[gc->epoch][hook_id];
  420. if ( ch == NULL )
  421. {
  422. gc->hook[gc->epoch][hook_id] = ch = chunk_from_cache(gc);
  423. }
  424. else
  425. {
  426. ch = ch->next;
  427. if ( ch->i == BLKS_PER_CHUNK )
  428. {
  429. och = gc->hook[gc->epoch][hook_id];
  430. ch = chunk_from_cache(gc);
  431. ch->next = och->next;
  432. och->next = ch;
  433. }
  434. }
  435. ch->blk[ch->i++] = ptr;
  436. }
  437. void gc_unsafe_free(ptst_t *ptst, void *p, int alloc_id)
  438. {
  439. gc_t *gc = ptst->gc;
  440. chunk_t *ch;
  441. ch = gc->alloc[alloc_id];
  442. if ( ch->i < BLKS_PER_CHUNK )
  443. {
  444. ch->blk[ch->i++] = p;
  445. }
  446. else
  447. {
  448. gc_free(ptst, p, alloc_id);
  449. }
  450. }
  451. void gc_enter(ptst_t *ptst)
  452. {
  453. #ifdef MINIMAL_GC
  454. ptst->count++;
  455. MB();
  456. #else
  457. gc_t *gc = ptst->gc;
  458. int new_epoch, cnt;
  459. retry:
  460. cnt = ptst->count++;
  461. MB();
  462. if ( cnt == 1 )
  463. {
  464. new_epoch = gc_global.current;
  465. if ( gc->epoch != new_epoch )
  466. {
  467. gc->epoch = new_epoch;
  468. gc->entries_since_reclaim = 0;
  469. #ifdef YIELD_TO_HELP_PROGRESS
  470. gc->reclaim_attempts_since_yield = 0;
  471. #endif
  472. }
  473. else if ( gc->entries_since_reclaim++ == 100 )
  474. {
  475. ptst->count--;
  476. #ifdef YIELD_TO_HELP_PROGRESS
  477. if ( gc->reclaim_attempts_since_yield++ == 10000 )
  478. {
  479. gc->reclaim_attempts_since_yield = 0;
  480. sched_yield();
  481. }
  482. #endif
  483. gc->entries_since_reclaim = 0;
  484. gc_reclaim();
  485. goto retry;
  486. }
  487. }
  488. #endif
  489. }
  490. void gc_exit(ptst_t *ptst)
  491. {
  492. MB();
  493. ptst->count--;
  494. }
  495. gc_t *gc_init(void)
  496. {
  497. gc_t *gc;
  498. int i;
  499. gc = ALIGNED_ALLOC(sizeof(*gc));
  500. if ( gc == NULL ) MEM_FAIL((int)sizeof(*gc));
  501. memset(gc, 0, sizeof(*gc));
  502. #ifdef WEAK_MEM_ORDER
  503. /* Initialise shootdown state. */
  504. gc->async_page = mmap(NULL, gc_global.page_size, PROT_NONE,
  505. MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
  506. if ( gc->async_page == (void *)MAP_FAILED ) MEM_FAIL(gc_global.page_size);
  507. gc->async_page_state = 1;
  508. #endif
  509. gc->chunk_cache = get_empty_chunks(100);
  510. /* Get ourselves a set of allocation chunks. */
  511. for ( i = 0; i < gc_global.nr_sizes; i++ )
  512. {
  513. gc->alloc[i] = get_alloc_chunk(gc, i);
  514. }
  515. for ( ; i < MAX_SIZES; i++ )
  516. {
  517. gc->alloc[i] = chunk_from_cache(gc);
  518. }
  519. return(gc);
  520. }
  521. int gc_add_allocator(int alloc_size)
  522. {
  523. int ni, i = gc_global.nr_sizes;
  524. while ( (ni = CASIO(&gc_global.nr_sizes, i, i+1)) != i ) i = ni;
  525. gc_global.blk_sizes[i] = alloc_size;
  526. gc_global.alloc_size[i] = ALLOC_CHUNKS_PER_LIST;
  527. gc_global.alloc[i] = get_filled_chunks(ALLOC_CHUNKS_PER_LIST, alloc_size);
  528. return i;
  529. }
  530. void gc_remove_allocator(int alloc_id)
  531. {
  532. /* This is a no-op for now. */
  533. }
  534. int gc_add_hook(hook_fn_t fn)
  535. {
  536. int ni, i = gc_global.nr_hooks;
  537. while ( (ni = CASIO(&gc_global.nr_hooks, i, i+1)) != i ) i = ni;
  538. gc_global.hook_fns[i] = fn;
  539. return i;
  540. }
  541. void gc_remove_hook(int hook_id)
  542. {
  543. /* This is a no-op for now. */
  544. }
  545. void _destroy_gc_subsystem(void)
  546. {
  547. #ifdef PROFILE_GC
  548. printf("Total heap: %u bytes (%.2fMB) in %u allocations\n",
  549. gc_global.total_size, (double)gc_global.total_size / 1000000,
  550. gc_global.allocations);
  551. #endif
  552. }
  553. void _init_gc_subsystem(void)
  554. {
  555. memset(&gc_global, 0, sizeof(gc_global));
  556. gc_global.page_size = (unsigned int)sysconf(_SC_PAGESIZE);
  557. gc_global.free_chunks = alloc_more_chunks();
  558. gc_global.nr_hooks = 0;
  559. gc_global.nr_sizes = 0;
  560. }