PageRenderTime 1008ms CodeModel.GetById 329ms app.highlight 376ms RepoModel.GetById 296ms app.codeStats 0ms

/trunk/src/lockfree/gc.c

#
C | 672 lines | 443 code | 112 blank | 117 comment | 83 complexity | 3f541404ec8cb2ed64567b97e73bf388 MD5 | raw file
Possible License(s): BSD-3-Clause
  1/******************************************************************************
  2 * gc.c
  3 * 
  4 * A fully recycling epoch-based garbage collector. Works by counting
  5 * threads in and out of critical regions, to work out when
  6 * garbage queues can be fully deleted.
  7 * 
  8 * Copyright (c) 2001-2003, K A Fraser
  9 * 
 10 * All rights reserved.
 11 * 
 12 * Redistribution and use in source and binary forms, with or without
 13 * modification, are permitted provided that the following conditions are met:
 14 * 
 15 * * Redistributions of source code must retain the above copyright 
 16 * notice, this list of conditions and the following disclaimer.
 17 * 
 18 * * Redistributions in binary form must reproduce the above copyright 
 19 * notice, this list of conditions and the following disclaimer in the
 20 * documentation and/or other materials provided with the distribution.
 21 * 
 22 * * The name of the author may not be used to endorse or promote products
 23 * derived from this software without specific prior written permission.
 24 * 
 25 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 
 26 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 
 27 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 
 28 * DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, 
 29 * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 
 30 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 
 31 * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 
 32 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, 
 33 * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN 
 34 * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 
 35 * POSSIBILITY OF SUCH DAMAGE.
 36 */
 37
 38#include <assert.h>
 39#include <stdio.h>
 40#include <stdlib.h>
 41#include <string.h>
 42#include <sys/mman.h>
 43#include <unistd.h>
 44#include "portable_defns.h"
 45#include "gc.h"
 46
 47/*#define MINIMAL_GC*/
 48/*#define YIELD_TO_HELP_PROGRESS*/
 49#define PROFILE_GC
 50
 51/* Recycled nodes are filled with this value if WEAK_MEM_ORDER. */
 52#define INVALID_BYTE 0
 53#define INITIALISE_NODES(_p,_c) memset((_p), INVALID_BYTE, (_c));
 54
 55/* Number of unique block sizes we can deal with. */
 56#define MAX_SIZES 20
 57
 58#define MAX_HOOKS 4
 59
 60/*
 61 * The initial number of allocation chunks for each per-blocksize list.
 62 * Popular allocation lists will steadily increase the allocation unit
 63 * in line with demand.
 64 */
 65#define ALLOC_CHUNKS_PER_LIST 10
 66
 67/*
 68 * How many times should a thread call gc_enter(), seeing the same epoch
 69 * each time, before it makes a reclaim attempt?
 70 */
 71#define ENTRIES_PER_RECLAIM_ATTEMPT 100
 72
 73/*
 74 *  0: current epoch -- threads are moving to this;
 75 * -1: some threads may still throw garbage into this epoch;
 76 * -2: no threads can see this epoch => we can zero garbage lists;
 77 * -3: all threads see zeros in these garbage lists => move to alloc lists.
 78 */
 79#ifdef WEAK_MEM_ORDER
 80#define NR_EPOCHS 4
 81#else
 82#define NR_EPOCHS 3
 83#endif
 84
 85/*
 86 * A chunk amortises the cost of allocation from shared lists. It also
 87 * helps when zeroing nodes, as it increases per-cacheline pointer density
 88 * and means that node locations don't need to be brought into the cache 
 89 * (most architectures have a non-temporal store instruction).
 90 */
 91#define BLKS_PER_CHUNK 100
 92typedef struct chunk_st chunk_t;
 93struct chunk_st
 94{
 95    chunk_t *next;             /* chunk chaining                 */
 96    unsigned int i;            /* the next entry in blk[] to use */
 97    void *blk[BLKS_PER_CHUNK];
 98};
 99
100static struct gc_global_st
101{
102    CACHE_PAD(0);
103
104    /* The current epoch. */
105    VOLATILE unsigned int current;
106    CACHE_PAD(1);
107
108    /* Exclusive access to gc_reclaim(). */
109    VOLATILE unsigned int inreclaim;
110    CACHE_PAD(2);
111
112    /*
113     * RUN-TIME CONSTANTS (to first approximation)
114     */
115
116    /* Memory page size, in bytes. */
117    unsigned int page_size;
118
119    /* Node sizes (run-time constants). */
120    int nr_sizes;
121    int blk_sizes[MAX_SIZES];
122
123    /* Registered epoch hooks. */
124    int nr_hooks;
125    hook_fn_t hook_fns[MAX_HOOKS];
126    CACHE_PAD(3);
127
128    /*
129     * DATA WE MAY HIT HARD
130     */
131
132    /* Chain of free, empty chunks. */
133    chunk_t * VOLATILE free_chunks;
134
135    /* Main allocation lists. */
136    chunk_t * VOLATILE alloc[MAX_SIZES];
137    VOLATILE unsigned int alloc_size[MAX_SIZES];
138#ifdef PROFILE_GC
139    VOLATILE unsigned int total_size;
140    VOLATILE unsigned int allocations;
141#endif
142} gc_global;
143
144
145/* Per-thread state. */
146struct gc_st
147{
148    /* Epoch that this thread sees. */
149    unsigned int epoch;
150
151    /* Number of calls to gc_entry() since last gc_reclaim() attempt. */
152    unsigned int entries_since_reclaim;
153
154#ifdef YIELD_TO_HELP_PROGRESS
155    /* Number of calls to gc_reclaim() since we last yielded. */
156    unsigned int reclaim_attempts_since_yield;
157#endif
158
159    /* Used by gc_async_barrier(). */
160    void *async_page;
161    int   async_page_state;
162
163    /* Garbage lists. */
164    chunk_t *garbage[NR_EPOCHS][MAX_SIZES];
165    chunk_t *garbage_tail[NR_EPOCHS][MAX_SIZES];
166    chunk_t *chunk_cache;
167
168    /* Local allocation lists. */
169    chunk_t *alloc[MAX_SIZES];
170    unsigned int alloc_chunks[MAX_SIZES];
171
172    /* Hook pointer lists. */
173    chunk_t *hook[NR_EPOCHS][MAX_HOOKS];
174};
175
176
177#define MEM_FAIL(_s)                                                         \
178do {                                                                         \
179    fprintf(stderr, "OUT OF MEMORY: %d bytes at line %d\n", (_s), __LINE__); \
180    exit(1);                                                                 \
181} while ( 0 )
182
183
184/* Allocate more empty chunks from the heap. */
185#define CHUNKS_PER_ALLOC 1000
186static chunk_t *alloc_more_chunks(void)
187{
188    int i;
189    chunk_t *h, *p;
190
191    h = p = ALIGNED_ALLOC(CHUNKS_PER_ALLOC * sizeof(*h));
192    if ( h == NULL ) MEM_FAIL( (int)( CHUNKS_PER_ALLOC * sizeof(*h) ) );
193
194    for ( i = 1; i < CHUNKS_PER_ALLOC; i++ )
195    {
196        p->next = p + 1;
197        p++;
198    }
199
200    p->next = h;
201
202    return(h);
203}
204
205
206/* Put a chain of chunks onto a list. */
207static void add_chunks_to_list(chunk_t *ch, chunk_t *head)
208{
209    chunk_t *h_next, *new_h_next, *ch_next;
210    ch_next    = ch->next;
211    new_h_next = head->next;
212    do { ch->next = h_next = new_h_next; WMB_NEAR_CAS(); }
213    while ( (new_h_next = CASPO(&head->next, h_next, ch_next)) != h_next );
214}
215
216
217/* Allocate a chain of @n empty chunks. Pointers may be garbage. */
218static chunk_t *get_empty_chunks(int n)
219{
220    int i;
221    chunk_t *new_rh, *rh, *rt, *head;
222
223 retry:
224    head = gc_global.free_chunks;
225    new_rh = head->next;
226    do {
227        rh = new_rh;
228        rt = head;
229        WEAK_DEP_ORDER_RMB();
230        for ( i = 0; i < n; i++ )
231        {
232            if ( (rt = rt->next) == head )
233            {
234                /* Allocate some more chunks. */
235                add_chunks_to_list(alloc_more_chunks(), head);
236                goto retry;
237            }
238        }
239    }
240    while ( (new_rh = CASPO(&head->next, rh, rt->next)) != rh );
241
242    rt->next = rh;
243    return(rh);
244}
245
246
247/* Get @n filled chunks, pointing at blocks of @sz bytes each. */
248static chunk_t *get_filled_chunks(int n, int sz)
249{
250    chunk_t *h, *p;
251    char *node;
252    int i;
253
254#ifdef PROFILE_GC
255    ADD_TO(gc_global.total_size, n * BLKS_PER_CHUNK * sz);
256    ADD_TO(gc_global.allocations, 1);
257#endif
258
259    node = ALIGNED_ALLOC(n * BLKS_PER_CHUNK * sz);
260    if ( node == NULL ) MEM_FAIL(n * BLKS_PER_CHUNK * sz);
261#ifdef WEAK_MEM_ORDER
262    INITIALISE_NODES(node, n * BLKS_PER_CHUNK * sz);
263#endif
264
265    h = p = get_empty_chunks(n);
266    do {
267        p->i = BLKS_PER_CHUNK;
268        for ( i = 0; i < BLKS_PER_CHUNK; i++ )
269        {
270            p->blk[i] = node;
271            node += sz;
272        }
273    }
274    while ( (p = p->next) != h );
275
276    return(h);
277}
278
279
280/*
281 * gc_async_barrier: Cause an asynchronous barrier in all other threads. We do 
282 * this by causing a TLB shootdown to be propagated to all other processors. 
283 * Each time such an action is required, this function calls:
284 *   mprotect(async_page, <page size>, <new flags>)
285 * Each thread's state contains a memory page dedicated for this purpose.
286 */
287#ifdef WEAK_MEM_ORDER
288static void gc_async_barrier(gc_t *gc)
289{
290    mprotect(gc->async_page, gc_global.page_size,
291             gc->async_page_state ? PROT_READ : PROT_NONE);
292    gc->async_page_state = !gc->async_page_state;
293}
294#else
295#define gc_async_barrier(_g) ((void)0)
296#endif
297
298
299/* Grab a level @i allocation chunk from main chain. */
300static chunk_t *get_alloc_chunk(gc_t *gc, int i)
301{
302    chunk_t *alloc, *p, *new_p, *nh;
303    unsigned int sz;
304
305    alloc = gc_global.alloc[i];
306    new_p = alloc->next;
307
308    do {
309        p = new_p;
310        while ( p == alloc )
311        {
312            sz = gc_global.alloc_size[i];
313            nh = get_filled_chunks(sz, gc_global.blk_sizes[i]);
314            ADD_TO(gc_global.alloc_size[i], sz >> 3);
315            gc_async_barrier(gc);
316            add_chunks_to_list(nh, alloc);
317            p = alloc->next;
318        }
319        WEAK_DEP_ORDER_RMB();
320    }
321    while ( (new_p = CASPO(&alloc->next, p, p->next)) != p );
322
323    p->next = p;
324    assert(p->i == BLKS_PER_CHUNK);
325    return(p);
326}
327
328
329#ifndef MINIMAL_GC
330/*
331 * gc_reclaim: Scans the list of struct gc_perthread looking for the lowest
332 * maximum epoch number seen by a thread that's in the list code. If it's the
333 * current epoch, the "nearly-free" lists from the previous epoch are 
334 * reclaimed, and the epoch is incremented.
335 */
336static void gc_reclaim(void)
337{
338    ptst_t       *ptst, *first_ptst, *our_ptst = NULL;
339    gc_t         *gc = NULL;
340    unsigned long curr_epoch;
341    chunk_t      *ch, *t;
342    int           two_ago, three_ago, i, j;
343    
344    /* Barrier to entering the reclaim critical section. */
345    if ( gc_global.inreclaim || CASIO(&gc_global.inreclaim, 0, 1) ) return;
346
347    /*
348     * Grab first ptst structure *before* barrier -- prevent bugs
349     * on weak-ordered architectures.
350     */
351    first_ptst = ptst_first();
352    MB();
353    curr_epoch = gc_global.current;
354
355    /* Have all threads seen the current epoch, or not in mutator code? */
356    for ( ptst = first_ptst; ptst != NULL; ptst = ptst_next(ptst) )
357    {
358        if ( (ptst->count > 1) && (ptst->gc->epoch != curr_epoch) ) goto out;
359    }
360
361    /*
362     * Three-epoch-old garbage lists move to allocation lists.
363     * Two-epoch-old garbage lists are cleaned out.
364     */
365    two_ago   = (curr_epoch+2) % NR_EPOCHS;
366    three_ago = (curr_epoch+1) % NR_EPOCHS;
367    if ( gc_global.nr_hooks != 0 )
368        our_ptst = (ptst_t *)pthread_getspecific(ptst_key);
369    for ( ptst = first_ptst; ptst != NULL; ptst = ptst_next(ptst) )
370    {
371        gc = ptst->gc;
372
373        for ( i = 0; i < gc_global.nr_sizes; i++ )
374        {
375#ifdef WEAK_MEM_ORDER
376            int sz = gc_global.blk_sizes[i];
377            if ( gc->garbage[two_ago][i] != NULL )
378            {
379                chunk_t *head = gc->garbage[two_ago][i];
380                ch = head;
381                do {
382                    int j;
383                    for ( j = 0; j < ch->i; j++ )
384                        INITIALISE_NODES(ch->blk[j], sz);
385                }
386                while ( (ch = ch->next) != head );
387            }
388#endif
389
390            /* NB. Leave one chunk behind, as it is probably not yet full. */
391            t = gc->garbage[three_ago][i];
392            if ( (t == NULL) || ((ch = t->next) == t) ) continue;
393            gc->garbage_tail[three_ago][i]->next = ch;
394            gc->garbage_tail[three_ago][i] = t;
395            t->next = t;
396            add_chunks_to_list(ch, gc_global.alloc[i]);
397        }
398
399        for ( i = 0; i < gc_global.nr_hooks; i++ )
400        {
401            hook_fn_t fn = gc_global.hook_fns[i];
402            ch = gc->hook[three_ago][i];
403            if ( ch == NULL ) continue;
404            gc->hook[three_ago][i] = NULL;
405
406            t = ch;
407            do { for ( j = 0; j < t->i; j++ ) fn(our_ptst, t->blk[j]); }
408            while ( (t = t->next) != ch );
409
410            add_chunks_to_list(ch, gc_global.free_chunks);
411        }
412    }
413
414    /* Update current epoch. */
415    WMB();
416    gc_global.current = (curr_epoch+1) % NR_EPOCHS;
417
418 out:
419    gc_global.inreclaim = 0;
420}
421#endif /* MINIMAL_GC */
422
423
424void *gc_alloc(ptst_t *ptst, int alloc_id)
425{
426    gc_t *gc = ptst->gc;
427    chunk_t *ch;
428
429    ch = gc->alloc[alloc_id];
430    if ( ch->i == 0 )
431    {
432        if ( gc->alloc_chunks[alloc_id]++ == 100 )
433        {
434            gc->alloc_chunks[alloc_id] = 0;
435            add_chunks_to_list(ch, gc_global.free_chunks);
436            gc->alloc[alloc_id] = ch = get_alloc_chunk(gc, alloc_id);
437        }
438        else
439        {
440            chunk_t *och = ch;
441            ch = get_alloc_chunk(gc, alloc_id);
442            ch->next  = och->next;
443            och->next = ch;
444            gc->alloc[alloc_id] = ch;        
445        }
446    }
447
448    return ch->blk[--ch->i];
449}
450
451
452static chunk_t *chunk_from_cache(gc_t *gc)
453{
454    chunk_t *ch = gc->chunk_cache, *p = ch->next;
455
456    if ( ch == p )
457    {
458        gc->chunk_cache = get_empty_chunks(100);
459    }
460    else
461    {
462        ch->next = p->next;
463        p->next  = p;
464    }
465
466    p->i = 0;
467    return(p);
468}
469
470
471void gc_free(ptst_t *ptst, void *p, int alloc_id) 
472{
473#ifndef MINIMAL_GC
474    gc_t *gc = ptst->gc;
475    chunk_t *prev, *new, *ch = gc->garbage[gc->epoch][alloc_id];
476
477    if ( ch == NULL )
478    {
479        gc->garbage[gc->epoch][alloc_id] = ch = chunk_from_cache(gc);
480        gc->garbage_tail[gc->epoch][alloc_id] = ch;
481    }
482    else if ( ch->i == BLKS_PER_CHUNK )
483    {
484        prev = gc->garbage_tail[gc->epoch][alloc_id];
485        new  = chunk_from_cache(gc);
486        gc->garbage[gc->epoch][alloc_id] = new;
487        new->next  = ch;
488        prev->next = new;
489        ch = new;
490    }
491
492    ch->blk[ch->i++] = p;
493#endif
494}
495
496
497void gc_add_ptr_to_hook_list(ptst_t *ptst, void *ptr, int hook_id)
498{
499    gc_t *gc = ptst->gc;
500    chunk_t *och, *ch = gc->hook[gc->epoch][hook_id];
501
502    if ( ch == NULL )
503    {
504        gc->hook[gc->epoch][hook_id] = ch = chunk_from_cache(gc);
505    }
506    else
507    {
508        ch = ch->next;
509        if ( ch->i == BLKS_PER_CHUNK )
510        {
511            och       = gc->hook[gc->epoch][hook_id];
512            ch        = chunk_from_cache(gc);
513            ch->next  = och->next;
514            och->next = ch;
515        }
516    }
517
518    ch->blk[ch->i++] = ptr;
519}
520
521
522void gc_unsafe_free(ptst_t *ptst, void *p, int alloc_id)
523{
524    gc_t *gc = ptst->gc;
525    chunk_t *ch;
526
527    ch = gc->alloc[alloc_id];
528    if ( ch->i < BLKS_PER_CHUNK )
529    {
530        ch->blk[ch->i++] = p;
531    }
532    else
533    {
534        gc_free(ptst, p, alloc_id);
535    }
536}
537
538
539void gc_enter(ptst_t *ptst)
540{
541#ifdef MINIMAL_GC
542    ptst->count++;
543    MB();
544#else
545    gc_t *gc = ptst->gc;
546    int new_epoch, cnt;
547 
548 retry:
549    cnt = ptst->count++;
550    MB();
551    if ( cnt == 1 )
552    {
553        new_epoch = gc_global.current;
554        if ( gc->epoch != new_epoch )
555        {
556            gc->epoch = new_epoch;
557            gc->entries_since_reclaim        = 0;
558#ifdef YIELD_TO_HELP_PROGRESS
559            gc->reclaim_attempts_since_yield = 0;
560#endif
561        }
562        else if ( gc->entries_since_reclaim++ == 100 )
563        {
564            ptst->count--;
565#ifdef YIELD_TO_HELP_PROGRESS
566            if ( gc->reclaim_attempts_since_yield++ == 10000 )
567            {
568                gc->reclaim_attempts_since_yield = 0;
569                sched_yield();
570            }
571#endif
572            gc->entries_since_reclaim = 0;
573            gc_reclaim();
574            goto retry;    
575        }
576    }
577#endif
578}
579
580
581void gc_exit(ptst_t *ptst)
582{
583    MB();
584    ptst->count--;
585}
586
587
588gc_t *gc_init(void)
589{
590    gc_t *gc;
591    int   i;
592
593    gc = ALIGNED_ALLOC(sizeof(*gc));
594    if ( gc == NULL ) MEM_FAIL((int)sizeof(*gc));
595    memset(gc, 0, sizeof(*gc));
596
597#ifdef WEAK_MEM_ORDER
598    /* Initialise shootdown state. */
599    gc->async_page = mmap(NULL, gc_global.page_size, PROT_NONE, 
600                          MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
601    if ( gc->async_page == (void *)MAP_FAILED ) MEM_FAIL(gc_global.page_size);
602    gc->async_page_state = 1;
603#endif
604
605    gc->chunk_cache = get_empty_chunks(100);
606
607    /* Get ourselves a set of allocation chunks. */
608    for ( i = 0; i < gc_global.nr_sizes; i++ )
609    {
610        gc->alloc[i] = get_alloc_chunk(gc, i);
611    }
612    for ( ; i < MAX_SIZES; i++ )
613    {
614        gc->alloc[i] = chunk_from_cache(gc);
615    }
616
617    return(gc);
618}
619
620
621int gc_add_allocator(int alloc_size)
622{
623    int ni, i = gc_global.nr_sizes;
624    while ( (ni = CASIO(&gc_global.nr_sizes, i, i+1)) != i ) i = ni;
625    gc_global.blk_sizes[i]  = alloc_size;
626    gc_global.alloc_size[i] = ALLOC_CHUNKS_PER_LIST;
627    gc_global.alloc[i] = get_filled_chunks(ALLOC_CHUNKS_PER_LIST, alloc_size);
628    return i;
629}
630
631
632void gc_remove_allocator(int alloc_id)
633{
634    /* This is a no-op for now. */
635}
636
637
638int gc_add_hook(hook_fn_t fn)
639{
640    int ni, i = gc_global.nr_hooks;
641    while ( (ni = CASIO(&gc_global.nr_hooks, i, i+1)) != i ) i = ni;
642    gc_global.hook_fns[i] = fn;
643    return i;    
644}
645
646
647void gc_remove_hook(int hook_id)
648{
649    /* This is a no-op for now. */
650}
651
652
653void _destroy_gc_subsystem(void)
654{
655#ifdef PROFILE_GC
656    printf("Total heap: %u bytes (%.2fMB) in %u allocations\n",
657           gc_global.total_size, (double)gc_global.total_size / 1000000,
658           gc_global.allocations);
659#endif
660}
661
662
663void _init_gc_subsystem(void)
664{
665    memset(&gc_global, 0, sizeof(gc_global));
666
667    gc_global.page_size   = (unsigned int)sysconf(_SC_PAGESIZE);
668    gc_global.free_chunks = alloc_more_chunks();
669
670    gc_global.nr_hooks = 0;
671    gc_global.nr_sizes = 0;
672}