/trunk/src/lockfree/gc.c
C | 672 lines | 443 code | 112 blank | 117 comment | 83 complexity | 3f541404ec8cb2ed64567b97e73bf388 MD5 | raw file
Possible License(s): BSD-3-Clause
1/****************************************************************************** 2 * gc.c 3 * 4 * A fully recycling epoch-based garbage collector. Works by counting 5 * threads in and out of critical regions, to work out when 6 * garbage queues can be fully deleted. 7 * 8 * Copyright (c) 2001-2003, K A Fraser 9 * 10 * All rights reserved. 11 * 12 * Redistribution and use in source and binary forms, with or without 13 * modification, are permitted provided that the following conditions are met: 14 * 15 * * Redistributions of source code must retain the above copyright 16 * notice, this list of conditions and the following disclaimer. 17 * 18 * * Redistributions in binary form must reproduce the above copyright 19 * notice, this list of conditions and the following disclaimer in the 20 * documentation and/or other materials provided with the distribution. 21 * 22 * * The name of the author may not be used to endorse or promote products 23 * derived from this software without specific prior written permission. 24 * 25 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 26 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 27 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 28 * DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, 29 * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 30 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 31 * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 32 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, 33 * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN 34 * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 35 * POSSIBILITY OF SUCH DAMAGE. 36 */ 37 38#include <assert.h> 39#include <stdio.h> 40#include <stdlib.h> 41#include <string.h> 42#include <sys/mman.h> 43#include <unistd.h> 44#include "portable_defns.h" 45#include "gc.h" 46 47/*#define MINIMAL_GC*/ 48/*#define YIELD_TO_HELP_PROGRESS*/ 49#define PROFILE_GC 50 51/* Recycled nodes are filled with this value if WEAK_MEM_ORDER. */ 52#define INVALID_BYTE 0 53#define INITIALISE_NODES(_p,_c) memset((_p), INVALID_BYTE, (_c)); 54 55/* Number of unique block sizes we can deal with. */ 56#define MAX_SIZES 20 57 58#define MAX_HOOKS 4 59 60/* 61 * The initial number of allocation chunks for each per-blocksize list. 62 * Popular allocation lists will steadily increase the allocation unit 63 * in line with demand. 64 */ 65#define ALLOC_CHUNKS_PER_LIST 10 66 67/* 68 * How many times should a thread call gc_enter(), seeing the same epoch 69 * each time, before it makes a reclaim attempt? 70 */ 71#define ENTRIES_PER_RECLAIM_ATTEMPT 100 72 73/* 74 * 0: current epoch -- threads are moving to this; 75 * -1: some threads may still throw garbage into this epoch; 76 * -2: no threads can see this epoch => we can zero garbage lists; 77 * -3: all threads see zeros in these garbage lists => move to alloc lists. 78 */ 79#ifdef WEAK_MEM_ORDER 80#define NR_EPOCHS 4 81#else 82#define NR_EPOCHS 3 83#endif 84 85/* 86 * A chunk amortises the cost of allocation from shared lists. It also 87 * helps when zeroing nodes, as it increases per-cacheline pointer density 88 * and means that node locations don't need to be brought into the cache 89 * (most architectures have a non-temporal store instruction). 90 */ 91#define BLKS_PER_CHUNK 100 92typedef struct chunk_st chunk_t; 93struct chunk_st 94{ 95 chunk_t *next; /* chunk chaining */ 96 unsigned int i; /* the next entry in blk[] to use */ 97 void *blk[BLKS_PER_CHUNK]; 98}; 99 100static struct gc_global_st 101{ 102 CACHE_PAD(0); 103 104 /* The current epoch. */ 105 VOLATILE unsigned int current; 106 CACHE_PAD(1); 107 108 /* Exclusive access to gc_reclaim(). */ 109 VOLATILE unsigned int inreclaim; 110 CACHE_PAD(2); 111 112 /* 113 * RUN-TIME CONSTANTS (to first approximation) 114 */ 115 116 /* Memory page size, in bytes. */ 117 unsigned int page_size; 118 119 /* Node sizes (run-time constants). */ 120 int nr_sizes; 121 int blk_sizes[MAX_SIZES]; 122 123 /* Registered epoch hooks. */ 124 int nr_hooks; 125 hook_fn_t hook_fns[MAX_HOOKS]; 126 CACHE_PAD(3); 127 128 /* 129 * DATA WE MAY HIT HARD 130 */ 131 132 /* Chain of free, empty chunks. */ 133 chunk_t * VOLATILE free_chunks; 134 135 /* Main allocation lists. */ 136 chunk_t * VOLATILE alloc[MAX_SIZES]; 137 VOLATILE unsigned int alloc_size[MAX_SIZES]; 138#ifdef PROFILE_GC 139 VOLATILE unsigned int total_size; 140 VOLATILE unsigned int allocations; 141#endif 142} gc_global; 143 144 145/* Per-thread state. */ 146struct gc_st 147{ 148 /* Epoch that this thread sees. */ 149 unsigned int epoch; 150 151 /* Number of calls to gc_entry() since last gc_reclaim() attempt. */ 152 unsigned int entries_since_reclaim; 153 154#ifdef YIELD_TO_HELP_PROGRESS 155 /* Number of calls to gc_reclaim() since we last yielded. */ 156 unsigned int reclaim_attempts_since_yield; 157#endif 158 159 /* Used by gc_async_barrier(). */ 160 void *async_page; 161 int async_page_state; 162 163 /* Garbage lists. */ 164 chunk_t *garbage[NR_EPOCHS][MAX_SIZES]; 165 chunk_t *garbage_tail[NR_EPOCHS][MAX_SIZES]; 166 chunk_t *chunk_cache; 167 168 /* Local allocation lists. */ 169 chunk_t *alloc[MAX_SIZES]; 170 unsigned int alloc_chunks[MAX_SIZES]; 171 172 /* Hook pointer lists. */ 173 chunk_t *hook[NR_EPOCHS][MAX_HOOKS]; 174}; 175 176 177#define MEM_FAIL(_s) \ 178do { \ 179 fprintf(stderr, "OUT OF MEMORY: %d bytes at line %d\n", (_s), __LINE__); \ 180 exit(1); \ 181} while ( 0 ) 182 183 184/* Allocate more empty chunks from the heap. */ 185#define CHUNKS_PER_ALLOC 1000 186static chunk_t *alloc_more_chunks(void) 187{ 188 int i; 189 chunk_t *h, *p; 190 191 h = p = ALIGNED_ALLOC(CHUNKS_PER_ALLOC * sizeof(*h)); 192 if ( h == NULL ) MEM_FAIL( (int)( CHUNKS_PER_ALLOC * sizeof(*h) ) ); 193 194 for ( i = 1; i < CHUNKS_PER_ALLOC; i++ ) 195 { 196 p->next = p + 1; 197 p++; 198 } 199 200 p->next = h; 201 202 return(h); 203} 204 205 206/* Put a chain of chunks onto a list. */ 207static void add_chunks_to_list(chunk_t *ch, chunk_t *head) 208{ 209 chunk_t *h_next, *new_h_next, *ch_next; 210 ch_next = ch->next; 211 new_h_next = head->next; 212 do { ch->next = h_next = new_h_next; WMB_NEAR_CAS(); } 213 while ( (new_h_next = CASPO(&head->next, h_next, ch_next)) != h_next ); 214} 215 216 217/* Allocate a chain of @n empty chunks. Pointers may be garbage. */ 218static chunk_t *get_empty_chunks(int n) 219{ 220 int i; 221 chunk_t *new_rh, *rh, *rt, *head; 222 223 retry: 224 head = gc_global.free_chunks; 225 new_rh = head->next; 226 do { 227 rh = new_rh; 228 rt = head; 229 WEAK_DEP_ORDER_RMB(); 230 for ( i = 0; i < n; i++ ) 231 { 232 if ( (rt = rt->next) == head ) 233 { 234 /* Allocate some more chunks. */ 235 add_chunks_to_list(alloc_more_chunks(), head); 236 goto retry; 237 } 238 } 239 } 240 while ( (new_rh = CASPO(&head->next, rh, rt->next)) != rh ); 241 242 rt->next = rh; 243 return(rh); 244} 245 246 247/* Get @n filled chunks, pointing at blocks of @sz bytes each. */ 248static chunk_t *get_filled_chunks(int n, int sz) 249{ 250 chunk_t *h, *p; 251 char *node; 252 int i; 253 254#ifdef PROFILE_GC 255 ADD_TO(gc_global.total_size, n * BLKS_PER_CHUNK * sz); 256 ADD_TO(gc_global.allocations, 1); 257#endif 258 259 node = ALIGNED_ALLOC(n * BLKS_PER_CHUNK * sz); 260 if ( node == NULL ) MEM_FAIL(n * BLKS_PER_CHUNK * sz); 261#ifdef WEAK_MEM_ORDER 262 INITIALISE_NODES(node, n * BLKS_PER_CHUNK * sz); 263#endif 264 265 h = p = get_empty_chunks(n); 266 do { 267 p->i = BLKS_PER_CHUNK; 268 for ( i = 0; i < BLKS_PER_CHUNK; i++ ) 269 { 270 p->blk[i] = node; 271 node += sz; 272 } 273 } 274 while ( (p = p->next) != h ); 275 276 return(h); 277} 278 279 280/* 281 * gc_async_barrier: Cause an asynchronous barrier in all other threads. We do 282 * this by causing a TLB shootdown to be propagated to all other processors. 283 * Each time such an action is required, this function calls: 284 * mprotect(async_page, <page size>, <new flags>) 285 * Each thread's state contains a memory page dedicated for this purpose. 286 */ 287#ifdef WEAK_MEM_ORDER 288static void gc_async_barrier(gc_t *gc) 289{ 290 mprotect(gc->async_page, gc_global.page_size, 291 gc->async_page_state ? PROT_READ : PROT_NONE); 292 gc->async_page_state = !gc->async_page_state; 293} 294#else 295#define gc_async_barrier(_g) ((void)0) 296#endif 297 298 299/* Grab a level @i allocation chunk from main chain. */ 300static chunk_t *get_alloc_chunk(gc_t *gc, int i) 301{ 302 chunk_t *alloc, *p, *new_p, *nh; 303 unsigned int sz; 304 305 alloc = gc_global.alloc[i]; 306 new_p = alloc->next; 307 308 do { 309 p = new_p; 310 while ( p == alloc ) 311 { 312 sz = gc_global.alloc_size[i]; 313 nh = get_filled_chunks(sz, gc_global.blk_sizes[i]); 314 ADD_TO(gc_global.alloc_size[i], sz >> 3); 315 gc_async_barrier(gc); 316 add_chunks_to_list(nh, alloc); 317 p = alloc->next; 318 } 319 WEAK_DEP_ORDER_RMB(); 320 } 321 while ( (new_p = CASPO(&alloc->next, p, p->next)) != p ); 322 323 p->next = p; 324 assert(p->i == BLKS_PER_CHUNK); 325 return(p); 326} 327 328 329#ifndef MINIMAL_GC 330/* 331 * gc_reclaim: Scans the list of struct gc_perthread looking for the lowest 332 * maximum epoch number seen by a thread that's in the list code. If it's the 333 * current epoch, the "nearly-free" lists from the previous epoch are 334 * reclaimed, and the epoch is incremented. 335 */ 336static void gc_reclaim(void) 337{ 338 ptst_t *ptst, *first_ptst, *our_ptst = NULL; 339 gc_t *gc = NULL; 340 unsigned long curr_epoch; 341 chunk_t *ch, *t; 342 int two_ago, three_ago, i, j; 343 344 /* Barrier to entering the reclaim critical section. */ 345 if ( gc_global.inreclaim || CASIO(&gc_global.inreclaim, 0, 1) ) return; 346 347 /* 348 * Grab first ptst structure *before* barrier -- prevent bugs 349 * on weak-ordered architectures. 350 */ 351 first_ptst = ptst_first(); 352 MB(); 353 curr_epoch = gc_global.current; 354 355 /* Have all threads seen the current epoch, or not in mutator code? */ 356 for ( ptst = first_ptst; ptst != NULL; ptst = ptst_next(ptst) ) 357 { 358 if ( (ptst->count > 1) && (ptst->gc->epoch != curr_epoch) ) goto out; 359 } 360 361 /* 362 * Three-epoch-old garbage lists move to allocation lists. 363 * Two-epoch-old garbage lists are cleaned out. 364 */ 365 two_ago = (curr_epoch+2) % NR_EPOCHS; 366 three_ago = (curr_epoch+1) % NR_EPOCHS; 367 if ( gc_global.nr_hooks != 0 ) 368 our_ptst = (ptst_t *)pthread_getspecific(ptst_key); 369 for ( ptst = first_ptst; ptst != NULL; ptst = ptst_next(ptst) ) 370 { 371 gc = ptst->gc; 372 373 for ( i = 0; i < gc_global.nr_sizes; i++ ) 374 { 375#ifdef WEAK_MEM_ORDER 376 int sz = gc_global.blk_sizes[i]; 377 if ( gc->garbage[two_ago][i] != NULL ) 378 { 379 chunk_t *head = gc->garbage[two_ago][i]; 380 ch = head; 381 do { 382 int j; 383 for ( j = 0; j < ch->i; j++ ) 384 INITIALISE_NODES(ch->blk[j], sz); 385 } 386 while ( (ch = ch->next) != head ); 387 } 388#endif 389 390 /* NB. Leave one chunk behind, as it is probably not yet full. */ 391 t = gc->garbage[three_ago][i]; 392 if ( (t == NULL) || ((ch = t->next) == t) ) continue; 393 gc->garbage_tail[three_ago][i]->next = ch; 394 gc->garbage_tail[three_ago][i] = t; 395 t->next = t; 396 add_chunks_to_list(ch, gc_global.alloc[i]); 397 } 398 399 for ( i = 0; i < gc_global.nr_hooks; i++ ) 400 { 401 hook_fn_t fn = gc_global.hook_fns[i]; 402 ch = gc->hook[three_ago][i]; 403 if ( ch == NULL ) continue; 404 gc->hook[three_ago][i] = NULL; 405 406 t = ch; 407 do { for ( j = 0; j < t->i; j++ ) fn(our_ptst, t->blk[j]); } 408 while ( (t = t->next) != ch ); 409 410 add_chunks_to_list(ch, gc_global.free_chunks); 411 } 412 } 413 414 /* Update current epoch. */ 415 WMB(); 416 gc_global.current = (curr_epoch+1) % NR_EPOCHS; 417 418 out: 419 gc_global.inreclaim = 0; 420} 421#endif /* MINIMAL_GC */ 422 423 424void *gc_alloc(ptst_t *ptst, int alloc_id) 425{ 426 gc_t *gc = ptst->gc; 427 chunk_t *ch; 428 429 ch = gc->alloc[alloc_id]; 430 if ( ch->i == 0 ) 431 { 432 if ( gc->alloc_chunks[alloc_id]++ == 100 ) 433 { 434 gc->alloc_chunks[alloc_id] = 0; 435 add_chunks_to_list(ch, gc_global.free_chunks); 436 gc->alloc[alloc_id] = ch = get_alloc_chunk(gc, alloc_id); 437 } 438 else 439 { 440 chunk_t *och = ch; 441 ch = get_alloc_chunk(gc, alloc_id); 442 ch->next = och->next; 443 och->next = ch; 444 gc->alloc[alloc_id] = ch; 445 } 446 } 447 448 return ch->blk[--ch->i]; 449} 450 451 452static chunk_t *chunk_from_cache(gc_t *gc) 453{ 454 chunk_t *ch = gc->chunk_cache, *p = ch->next; 455 456 if ( ch == p ) 457 { 458 gc->chunk_cache = get_empty_chunks(100); 459 } 460 else 461 { 462 ch->next = p->next; 463 p->next = p; 464 } 465 466 p->i = 0; 467 return(p); 468} 469 470 471void gc_free(ptst_t *ptst, void *p, int alloc_id) 472{ 473#ifndef MINIMAL_GC 474 gc_t *gc = ptst->gc; 475 chunk_t *prev, *new, *ch = gc->garbage[gc->epoch][alloc_id]; 476 477 if ( ch == NULL ) 478 { 479 gc->garbage[gc->epoch][alloc_id] = ch = chunk_from_cache(gc); 480 gc->garbage_tail[gc->epoch][alloc_id] = ch; 481 } 482 else if ( ch->i == BLKS_PER_CHUNK ) 483 { 484 prev = gc->garbage_tail[gc->epoch][alloc_id]; 485 new = chunk_from_cache(gc); 486 gc->garbage[gc->epoch][alloc_id] = new; 487 new->next = ch; 488 prev->next = new; 489 ch = new; 490 } 491 492 ch->blk[ch->i++] = p; 493#endif 494} 495 496 497void gc_add_ptr_to_hook_list(ptst_t *ptst, void *ptr, int hook_id) 498{ 499 gc_t *gc = ptst->gc; 500 chunk_t *och, *ch = gc->hook[gc->epoch][hook_id]; 501 502 if ( ch == NULL ) 503 { 504 gc->hook[gc->epoch][hook_id] = ch = chunk_from_cache(gc); 505 } 506 else 507 { 508 ch = ch->next; 509 if ( ch->i == BLKS_PER_CHUNK ) 510 { 511 och = gc->hook[gc->epoch][hook_id]; 512 ch = chunk_from_cache(gc); 513 ch->next = och->next; 514 och->next = ch; 515 } 516 } 517 518 ch->blk[ch->i++] = ptr; 519} 520 521 522void gc_unsafe_free(ptst_t *ptst, void *p, int alloc_id) 523{ 524 gc_t *gc = ptst->gc; 525 chunk_t *ch; 526 527 ch = gc->alloc[alloc_id]; 528 if ( ch->i < BLKS_PER_CHUNK ) 529 { 530 ch->blk[ch->i++] = p; 531 } 532 else 533 { 534 gc_free(ptst, p, alloc_id); 535 } 536} 537 538 539void gc_enter(ptst_t *ptst) 540{ 541#ifdef MINIMAL_GC 542 ptst->count++; 543 MB(); 544#else 545 gc_t *gc = ptst->gc; 546 int new_epoch, cnt; 547 548 retry: 549 cnt = ptst->count++; 550 MB(); 551 if ( cnt == 1 ) 552 { 553 new_epoch = gc_global.current; 554 if ( gc->epoch != new_epoch ) 555 { 556 gc->epoch = new_epoch; 557 gc->entries_since_reclaim = 0; 558#ifdef YIELD_TO_HELP_PROGRESS 559 gc->reclaim_attempts_since_yield = 0; 560#endif 561 } 562 else if ( gc->entries_since_reclaim++ == 100 ) 563 { 564 ptst->count--; 565#ifdef YIELD_TO_HELP_PROGRESS 566 if ( gc->reclaim_attempts_since_yield++ == 10000 ) 567 { 568 gc->reclaim_attempts_since_yield = 0; 569 sched_yield(); 570 } 571#endif 572 gc->entries_since_reclaim = 0; 573 gc_reclaim(); 574 goto retry; 575 } 576 } 577#endif 578} 579 580 581void gc_exit(ptst_t *ptst) 582{ 583 MB(); 584 ptst->count--; 585} 586 587 588gc_t *gc_init(void) 589{ 590 gc_t *gc; 591 int i; 592 593 gc = ALIGNED_ALLOC(sizeof(*gc)); 594 if ( gc == NULL ) MEM_FAIL((int)sizeof(*gc)); 595 memset(gc, 0, sizeof(*gc)); 596 597#ifdef WEAK_MEM_ORDER 598 /* Initialise shootdown state. */ 599 gc->async_page = mmap(NULL, gc_global.page_size, PROT_NONE, 600 MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); 601 if ( gc->async_page == (void *)MAP_FAILED ) MEM_FAIL(gc_global.page_size); 602 gc->async_page_state = 1; 603#endif 604 605 gc->chunk_cache = get_empty_chunks(100); 606 607 /* Get ourselves a set of allocation chunks. */ 608 for ( i = 0; i < gc_global.nr_sizes; i++ ) 609 { 610 gc->alloc[i] = get_alloc_chunk(gc, i); 611 } 612 for ( ; i < MAX_SIZES; i++ ) 613 { 614 gc->alloc[i] = chunk_from_cache(gc); 615 } 616 617 return(gc); 618} 619 620 621int gc_add_allocator(int alloc_size) 622{ 623 int ni, i = gc_global.nr_sizes; 624 while ( (ni = CASIO(&gc_global.nr_sizes, i, i+1)) != i ) i = ni; 625 gc_global.blk_sizes[i] = alloc_size; 626 gc_global.alloc_size[i] = ALLOC_CHUNKS_PER_LIST; 627 gc_global.alloc[i] = get_filled_chunks(ALLOC_CHUNKS_PER_LIST, alloc_size); 628 return i; 629} 630 631 632void gc_remove_allocator(int alloc_id) 633{ 634 /* This is a no-op for now. */ 635} 636 637 638int gc_add_hook(hook_fn_t fn) 639{ 640 int ni, i = gc_global.nr_hooks; 641 while ( (ni = CASIO(&gc_global.nr_hooks, i, i+1)) != i ) i = ni; 642 gc_global.hook_fns[i] = fn; 643 return i; 644} 645 646 647void gc_remove_hook(int hook_id) 648{ 649 /* This is a no-op for now. */ 650} 651 652 653void _destroy_gc_subsystem(void) 654{ 655#ifdef PROFILE_GC 656 printf("Total heap: %u bytes (%.2fMB) in %u allocations\n", 657 gc_global.total_size, (double)gc_global.total_size / 1000000, 658 gc_global.allocations); 659#endif 660} 661 662 663void _init_gc_subsystem(void) 664{ 665 memset(&gc_global, 0, sizeof(gc_global)); 666 667 gc_global.page_size = (unsigned int)sysconf(_SC_PAGESIZE); 668 gc_global.free_chunks = alloc_more_chunks(); 669 670 gc_global.nr_hooks = 0; 671 gc_global.nr_sizes = 0; 672}