PageRenderTime 81ms CodeModel.GetById 6ms app.highlight 53ms RepoModel.GetById 1ms app.codeStats 1ms

/Source/JavaScriptCore/wtf/FastMalloc.cpp

https://bitbucket.org/slukk/atsmp_4.2-external_webkit
C++ | 4626 lines | 3257 code | 645 blank | 724 comment | 544 complexity | 0acf3c415fe5f364536a7e2c9c2efcf3 MD5 | raw file

Large files files are truncated, but you can click here to view the full file

   1// Copyright (c) 2005, 2007, Google Inc.
   2// All rights reserved.
   3// Copyright (C) 2005, 2006, 2007, 2008, 2009, 2011 Apple Inc. All rights reserved.
   4// 
   5// Redistribution and use in source and binary forms, with or without
   6// modification, are permitted provided that the following conditions are
   7// met:
   8// 
   9//     * Redistributions of source code must retain the above copyright
  10// notice, this list of conditions and the following disclaimer.
  11//     * Redistributions in binary form must reproduce the above
  12// copyright notice, this list of conditions and the following disclaimer
  13// in the documentation and/or other materials provided with the
  14// distribution.
  15//     * Neither the name of Google Inc. nor the names of its
  16// contributors may be used to endorse or promote products derived from
  17// this software without specific prior written permission.
  18// 
  19// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  20// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  21// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  22// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  23// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  24// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  25// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  26// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  27// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  28// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  29// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  30
  31// ---
  32// Author: Sanjay Ghemawat <opensource@google.com>
  33//
  34// A malloc that uses a per-thread cache to satisfy small malloc requests.
  35// (The time for malloc/free of a small object drops from 300 ns to 50 ns.)
  36//
  37// See doc/tcmalloc.html for a high-level
  38// description of how this malloc works.
  39//
  40// SYNCHRONIZATION
  41//  1. The thread-specific lists are accessed without acquiring any locks.
  42//     This is safe because each such list is only accessed by one thread.
  43//  2. We have a lock per central free-list, and hold it while manipulating
  44//     the central free list for a particular size.
  45//  3. The central page allocator is protected by "pageheap_lock".
  46//  4. The pagemap (which maps from page-number to descriptor),
  47//     can be read without holding any locks, and written while holding
  48//     the "pageheap_lock".
  49//  5. To improve performance, a subset of the information one can get
  50//     from the pagemap is cached in a data structure, pagemap_cache_,
  51//     that atomically reads and writes its entries.  This cache can be
  52//     read and written without locking.
  53//
  54//     This multi-threaded access to the pagemap is safe for fairly
  55//     subtle reasons.  We basically assume that when an object X is
  56//     allocated by thread A and deallocated by thread B, there must
  57//     have been appropriate synchronization in the handoff of object
  58//     X from thread A to thread B.  The same logic applies to pagemap_cache_.
  59//
  60// THE PAGEID-TO-SIZECLASS CACHE
  61// Hot PageID-to-sizeclass mappings are held by pagemap_cache_.  If this cache
  62// returns 0 for a particular PageID then that means "no information," not that
  63// the sizeclass is 0.  The cache may have stale information for pages that do
  64// not hold the beginning of any free()'able object.  Staleness is eliminated
  65// in Populate() for pages with sizeclass > 0 objects, and in do_malloc() and
  66// do_memalign() for all other relevant pages.
  67//
  68// TODO: Bias reclamation to larger addresses
  69// TODO: implement mallinfo/mallopt
  70// TODO: Better testing
  71//
  72// 9/28/2003 (new page-level allocator replaces ptmalloc2):
  73// * malloc/free of small objects goes from ~300 ns to ~50 ns.
  74// * allocation of a reasonably complicated struct
  75//   goes from about 1100 ns to about 300 ns.
  76
  77#include "config.h"
  78#include "FastMalloc.h"
  79
  80#include "Assertions.h"
  81#include <limits>
  82#if ENABLE(JSC_MULTIPLE_THREADS)
  83#include <pthread.h>
  84#endif
  85#include <wtf/StdLibExtras.h>
  86
  87#ifndef NO_TCMALLOC_SAMPLES
  88#ifdef WTF_CHANGES
  89#define NO_TCMALLOC_SAMPLES
  90#endif
  91#endif
  92
  93#if !(defined(USE_SYSTEM_MALLOC) && USE_SYSTEM_MALLOC) && defined(NDEBUG)
  94#define FORCE_SYSTEM_MALLOC 0
  95#else
  96#define FORCE_SYSTEM_MALLOC 1
  97#endif
  98
  99// Use a background thread to periodically scavenge memory to release back to the system
 100// https://bugs.webkit.org/show_bug.cgi?id=27900: don't turn this on for Tiger until we have figured out why it caused a crash.
 101#if defined(BUILDING_ON_TIGER)
 102#define USE_BACKGROUND_THREAD_TO_SCAVENGE_MEMORY 0
 103#else
 104#define USE_BACKGROUND_THREAD_TO_SCAVENGE_MEMORY 1
 105#endif
 106
 107#ifndef NDEBUG
 108namespace WTF {
 109
 110#if ENABLE(JSC_MULTIPLE_THREADS)
 111static pthread_key_t isForbiddenKey;
 112static pthread_once_t isForbiddenKeyOnce = PTHREAD_ONCE_INIT;
 113static void initializeIsForbiddenKey()
 114{
 115  pthread_key_create(&isForbiddenKey, 0);
 116}
 117
 118#if !ASSERT_DISABLED
 119static bool isForbidden()
 120{
 121    pthread_once(&isForbiddenKeyOnce, initializeIsForbiddenKey);
 122    return !!pthread_getspecific(isForbiddenKey);
 123}
 124#endif
 125
 126void fastMallocForbid()
 127{
 128    pthread_once(&isForbiddenKeyOnce, initializeIsForbiddenKey);
 129    pthread_setspecific(isForbiddenKey, &isForbiddenKey);
 130}
 131
 132void fastMallocAllow()
 133{
 134    pthread_once(&isForbiddenKeyOnce, initializeIsForbiddenKey);
 135    pthread_setspecific(isForbiddenKey, 0);
 136}
 137
 138#else
 139
 140static bool staticIsForbidden;
 141static bool isForbidden()
 142{
 143    return staticIsForbidden;
 144}
 145
 146void fastMallocForbid()
 147{
 148    staticIsForbidden = true;
 149}
 150
 151void fastMallocAllow()
 152{
 153    staticIsForbidden = false;
 154}
 155#endif // ENABLE(JSC_MULTIPLE_THREADS)
 156
 157} // namespace WTF
 158#endif // NDEBUG
 159
 160#include <string.h>
 161
 162namespace WTF {
 163
 164#if ENABLE(FAST_MALLOC_MATCH_VALIDATION)
 165
 166namespace Internal {
 167
 168void fastMallocMatchFailed(void*)
 169{
 170    CRASH();
 171}
 172
 173} // namespace Internal
 174
 175#endif
 176
 177void* fastZeroedMalloc(size_t n) 
 178{
 179    void* result = fastMalloc(n);
 180    memset(result, 0, n);
 181    return result;
 182}
 183
 184char* fastStrDup(const char* src)
 185{
 186    size_t len = strlen(src) + 1;
 187    char* dup = static_cast<char*>(fastMalloc(len));
 188    memcpy(dup, src, len);
 189    return dup;
 190}
 191
 192TryMallocReturnValue tryFastZeroedMalloc(size_t n) 
 193{
 194    void* result;
 195    if (!tryFastMalloc(n).getValue(result))
 196        return 0;
 197    memset(result, 0, n);
 198    return result;
 199}
 200
 201} // namespace WTF
 202
 203#if FORCE_SYSTEM_MALLOC
 204
 205#if PLATFORM(BREWMP)
 206#include "brew/SystemMallocBrew.h"
 207#endif
 208
 209#if OS(DARWIN)
 210#include <malloc/malloc.h>
 211#elif OS(WINDOWS)
 212#include <malloc.h>
 213#endif
 214
 215namespace WTF {
 216
 217TryMallocReturnValue tryFastMalloc(size_t n) 
 218{
 219    ASSERT(!isForbidden());
 220
 221#if ENABLE(FAST_MALLOC_MATCH_VALIDATION)
 222    if (std::numeric_limits<size_t>::max() - sizeof(AllocAlignmentInteger) <= n)  // If overflow would occur...
 223        return 0;
 224
 225    void* result = malloc(n + sizeof(AllocAlignmentInteger));
 226    if (!result)
 227        return 0;
 228
 229    *static_cast<AllocAlignmentInteger*>(result) = Internal::AllocTypeMalloc;
 230    result = static_cast<AllocAlignmentInteger*>(result) + 1;
 231
 232    return result;
 233#else
 234    return malloc(n);
 235#endif
 236}
 237
 238void* fastMalloc(size_t n) 
 239{
 240    ASSERT(!isForbidden());
 241
 242#if ENABLE(FAST_MALLOC_MATCH_VALIDATION)
 243    TryMallocReturnValue returnValue = tryFastMalloc(n);
 244    void* result;
 245    if (!returnValue.getValue(result))
 246        CRASH();
 247#else
 248    void* result = malloc(n);
 249#endif
 250
 251    if (!result) {
 252#if PLATFORM(BREWMP)
 253        // The behavior of malloc(0) is implementation defined.
 254        // To make sure that fastMalloc never returns 0, retry with fastMalloc(1).
 255        if (!n)
 256            return fastMalloc(1);
 257#endif
 258        CRASH();
 259    }
 260
 261    return result;
 262}
 263
 264TryMallocReturnValue tryFastCalloc(size_t n_elements, size_t element_size)
 265{
 266    ASSERT(!isForbidden());
 267
 268#if ENABLE(FAST_MALLOC_MATCH_VALIDATION)
 269    size_t totalBytes = n_elements * element_size;
 270    if (n_elements > 1 && element_size && (totalBytes / element_size) != n_elements || (std::numeric_limits<size_t>::max() - sizeof(AllocAlignmentInteger) <= totalBytes))
 271        return 0;
 272
 273    totalBytes += sizeof(AllocAlignmentInteger);
 274    void* result = malloc(totalBytes);
 275    if (!result)
 276        return 0;
 277
 278    memset(result, 0, totalBytes);
 279    *static_cast<AllocAlignmentInteger*>(result) = Internal::AllocTypeMalloc;
 280    result = static_cast<AllocAlignmentInteger*>(result) + 1;
 281    return result;
 282#else
 283    return calloc(n_elements, element_size);
 284#endif
 285}
 286
 287void* fastCalloc(size_t n_elements, size_t element_size)
 288{
 289    ASSERT(!isForbidden());
 290
 291#if ENABLE(FAST_MALLOC_MATCH_VALIDATION)
 292    TryMallocReturnValue returnValue = tryFastCalloc(n_elements, element_size);
 293    void* result;
 294    if (!returnValue.getValue(result))
 295        CRASH();
 296#else
 297    void* result = calloc(n_elements, element_size);
 298#endif
 299
 300    if (!result) {
 301#if PLATFORM(BREWMP)
 302        // If either n_elements or element_size is 0, the behavior of calloc is implementation defined.
 303        // To make sure that fastCalloc never returns 0, retry with fastCalloc(1, 1).
 304        if (!n_elements || !element_size)
 305            return fastCalloc(1, 1);
 306#endif
 307        CRASH();
 308    }
 309
 310    return result;
 311}
 312
 313void fastFree(void* p)
 314{
 315    ASSERT(!isForbidden());
 316
 317#if ENABLE(FAST_MALLOC_MATCH_VALIDATION)
 318    if (!p)
 319        return;
 320
 321    AllocAlignmentInteger* header = Internal::fastMallocMatchValidationValue(p);
 322    if (*header != Internal::AllocTypeMalloc)
 323        Internal::fastMallocMatchFailed(p);
 324    free(header);
 325#else
 326    free(p);
 327#endif
 328}
 329
 330TryMallocReturnValue tryFastRealloc(void* p, size_t n)
 331{
 332    ASSERT(!isForbidden());
 333
 334#if ENABLE(FAST_MALLOC_MATCH_VALIDATION)
 335    if (p) {
 336        if (std::numeric_limits<size_t>::max() - sizeof(AllocAlignmentInteger) <= n)  // If overflow would occur...
 337            return 0;
 338        AllocAlignmentInteger* header = Internal::fastMallocMatchValidationValue(p);
 339        if (*header != Internal::AllocTypeMalloc)
 340            Internal::fastMallocMatchFailed(p);
 341        void* result = realloc(header, n + sizeof(AllocAlignmentInteger));
 342        if (!result)
 343            return 0;
 344
 345        // This should not be needed because the value is already there:
 346        // *static_cast<AllocAlignmentInteger*>(result) = Internal::AllocTypeMalloc;
 347        result = static_cast<AllocAlignmentInteger*>(result) + 1;
 348        return result;
 349    } else {
 350        return fastMalloc(n);
 351    }
 352#else
 353    return realloc(p, n);
 354#endif
 355}
 356
 357void* fastRealloc(void* p, size_t n)
 358{
 359    ASSERT(!isForbidden());
 360
 361#if ENABLE(FAST_MALLOC_MATCH_VALIDATION)
 362    TryMallocReturnValue returnValue = tryFastRealloc(p, n);
 363    void* result;
 364    if (!returnValue.getValue(result))
 365        CRASH();
 366#else
 367    void* result = realloc(p, n);
 368#endif
 369
 370    if (!result)
 371        CRASH();
 372    return result;
 373}
 374
 375void releaseFastMallocFreeMemory() { }
 376    
 377FastMallocStatistics fastMallocStatistics()
 378{
 379    FastMallocStatistics statistics = { 0, 0, 0 };
 380    return statistics;
 381}
 382
 383size_t fastMallocSize(const void* p)
 384{
 385#if OS(DARWIN)
 386    return malloc_size(p);
 387#elif OS(WINDOWS) && !PLATFORM(BREWMP)
 388    // Brew MP uses its own memory allocator, so _msize does not work on the Brew MP simulator.
 389    return _msize(const_cast<void*>(p));
 390#else
 391    return 1;
 392#endif
 393}
 394
 395} // namespace WTF
 396
 397#if OS(DARWIN)
 398// This symbol is present in the JavaScriptCore exports file even when FastMalloc is disabled.
 399// It will never be used in this case, so it's type and value are less interesting than its presence.
 400extern "C" const int jscore_fastmalloc_introspection = 0;
 401#endif
 402
 403#else // FORCE_SYSTEM_MALLOC
 404
 405#if HAVE(STDINT_H)
 406#include <stdint.h>
 407#elif HAVE(INTTYPES_H)
 408#include <inttypes.h>
 409#else
 410#include <sys/types.h>
 411#endif
 412
 413#include "AlwaysInline.h"
 414#include "Assertions.h"
 415#include "TCPackedCache.h"
 416#include "TCPageMap.h"
 417#include "TCSpinLock.h"
 418#include "TCSystemAlloc.h"
 419#include <algorithm>
 420#include <limits>
 421#include <pthread.h>
 422#include <stdarg.h>
 423#include <stddef.h>
 424#include <stdio.h>
 425#if HAVE(ERRNO_H)
 426#include <errno.h>
 427#endif
 428#if OS(UNIX)
 429#include <unistd.h>
 430#endif
 431#if OS(WINDOWS)
 432#ifndef WIN32_LEAN_AND_MEAN
 433#define WIN32_LEAN_AND_MEAN
 434#endif
 435#include <windows.h>
 436#endif
 437
 438#ifdef WTF_CHANGES
 439
 440#if OS(DARWIN)
 441#include "MallocZoneSupport.h"
 442#include <wtf/HashSet.h>
 443#include <wtf/Vector.h>
 444#endif
 445
 446#if HAVE(HEADER_DETECTION_H)
 447#include "HeaderDetection.h"
 448#endif
 449
 450#if HAVE(DISPATCH_H)
 451#include <dispatch/dispatch.h>
 452#endif
 453
 454#if HAVE(PTHREAD_MACHDEP_H)
 455#include <System/pthread_machdep.h>
 456
 457#if defined(__PTK_FRAMEWORK_JAVASCRIPTCORE_KEY0)
 458#define WTF_USE_PTHREAD_GETSPECIFIC_DIRECT 1
 459#endif
 460#endif
 461
 462#ifndef PRIuS
 463#define PRIuS "zu"
 464#endif
 465
 466// Calling pthread_getspecific through a global function pointer is faster than a normal
 467// call to the function on Mac OS X, and it's used in performance-critical code. So we
 468// use a function pointer. But that's not necessarily faster on other platforms, and we had
 469// problems with this technique on Windows, so we'll do this only on Mac OS X.
 470#if OS(DARWIN)
 471#if !USE(PTHREAD_GETSPECIFIC_DIRECT)
 472static void* (*pthread_getspecific_function_pointer)(pthread_key_t) = pthread_getspecific;
 473#define pthread_getspecific(key) pthread_getspecific_function_pointer(key)
 474#else
 475#define pthread_getspecific(key) _pthread_getspecific_direct(key)
 476#define pthread_setspecific(key, val) _pthread_setspecific_direct(key, (val))
 477#endif
 478#endif
 479
 480#define DEFINE_VARIABLE(type, name, value, meaning) \
 481  namespace FLAG__namespace_do_not_use_directly_use_DECLARE_##type##_instead {  \
 482  type FLAGS_##name(value);                                \
 483  char FLAGS_no##name;                                                        \
 484  }                                                                           \
 485  using FLAG__namespace_do_not_use_directly_use_DECLARE_##type##_instead::FLAGS_##name
 486  
 487#define DEFINE_int64(name, value, meaning) \
 488  DEFINE_VARIABLE(int64_t, name, value, meaning)
 489  
 490#define DEFINE_double(name, value, meaning) \
 491  DEFINE_VARIABLE(double, name, value, meaning)
 492
 493namespace WTF {
 494
 495#define malloc fastMalloc
 496#define calloc fastCalloc
 497#define free fastFree
 498#define realloc fastRealloc
 499
 500#define MESSAGE LOG_ERROR
 501#define CHECK_CONDITION ASSERT
 502
 503#if OS(DARWIN)
 504struct Span;
 505class TCMalloc_Central_FreeListPadded;
 506class TCMalloc_PageHeap;
 507class TCMalloc_ThreadCache;
 508template <typename T> class PageHeapAllocator;
 509
 510class FastMallocZone {
 511public:
 512    static void init();
 513
 514    static kern_return_t enumerate(task_t, void*, unsigned typeMmask, vm_address_t zoneAddress, memory_reader_t, vm_range_recorder_t);
 515    static size_t goodSize(malloc_zone_t*, size_t size) { return size; }
 516    static boolean_t check(malloc_zone_t*) { return true; }
 517    static void  print(malloc_zone_t*, boolean_t) { }
 518    static void log(malloc_zone_t*, void*) { }
 519    static void forceLock(malloc_zone_t*) { }
 520    static void forceUnlock(malloc_zone_t*) { }
 521    static void statistics(malloc_zone_t*, malloc_statistics_t* stats) { memset(stats, 0, sizeof(malloc_statistics_t)); }
 522
 523private:
 524    FastMallocZone(TCMalloc_PageHeap*, TCMalloc_ThreadCache**, TCMalloc_Central_FreeListPadded*, PageHeapAllocator<Span>*, PageHeapAllocator<TCMalloc_ThreadCache>*);
 525    static size_t size(malloc_zone_t*, const void*);
 526    static void* zoneMalloc(malloc_zone_t*, size_t);
 527    static void* zoneCalloc(malloc_zone_t*, size_t numItems, size_t size);
 528    static void zoneFree(malloc_zone_t*, void*);
 529    static void* zoneRealloc(malloc_zone_t*, void*, size_t);
 530    static void* zoneValloc(malloc_zone_t*, size_t) { LOG_ERROR("valloc is not supported"); return 0; }
 531    static void zoneDestroy(malloc_zone_t*) { }
 532
 533    malloc_zone_t m_zone;
 534    TCMalloc_PageHeap* m_pageHeap;
 535    TCMalloc_ThreadCache** m_threadHeaps;
 536    TCMalloc_Central_FreeListPadded* m_centralCaches;
 537    PageHeapAllocator<Span>* m_spanAllocator;
 538    PageHeapAllocator<TCMalloc_ThreadCache>* m_pageHeapAllocator;
 539};
 540
 541#endif
 542
 543#endif
 544
 545#ifndef WTF_CHANGES
 546// This #ifdef should almost never be set.  Set NO_TCMALLOC_SAMPLES if
 547// you're porting to a system where you really can't get a stacktrace.
 548#ifdef NO_TCMALLOC_SAMPLES
 549// We use #define so code compiles even if you #include stacktrace.h somehow.
 550# define GetStackTrace(stack, depth, skip)  (0)
 551#else
 552# include <google/stacktrace.h>
 553#endif
 554#endif
 555
 556// Even if we have support for thread-local storage in the compiler
 557// and linker, the OS may not support it.  We need to check that at
 558// runtime.  Right now, we have to keep a manual set of "bad" OSes.
 559#if defined(HAVE_TLS)
 560  static bool kernel_supports_tls = false;      // be conservative
 561  static inline bool KernelSupportsTLS() {
 562    return kernel_supports_tls;
 563  }
 564# if !HAVE_DECL_UNAME   // if too old for uname, probably too old for TLS
 565    static void CheckIfKernelSupportsTLS() {
 566      kernel_supports_tls = false;
 567    }
 568# else
 569#   include <sys/utsname.h>    // DECL_UNAME checked for <sys/utsname.h> too
 570    static void CheckIfKernelSupportsTLS() {
 571      struct utsname buf;
 572      if (uname(&buf) != 0) {   // should be impossible
 573        MESSAGE("uname failed assuming no TLS support (errno=%d)\n", errno);
 574        kernel_supports_tls = false;
 575      } else if (strcasecmp(buf.sysname, "linux") == 0) {
 576        // The linux case: the first kernel to support TLS was 2.6.0
 577        if (buf.release[0] < '2' && buf.release[1] == '.')    // 0.x or 1.x
 578          kernel_supports_tls = false;
 579        else if (buf.release[0] == '2' && buf.release[1] == '.' &&
 580                 buf.release[2] >= '0' && buf.release[2] < '6' &&
 581                 buf.release[3] == '.')                       // 2.0 - 2.5
 582          kernel_supports_tls = false;
 583        else
 584          kernel_supports_tls = true;
 585      } else {        // some other kernel, we'll be optimisitic
 586        kernel_supports_tls = true;
 587      }
 588      // TODO(csilvers): VLOG(1) the tls status once we support RAW_VLOG
 589    }
 590#  endif  // HAVE_DECL_UNAME
 591#endif    // HAVE_TLS
 592
 593// __THROW is defined in glibc systems.  It means, counter-intuitively,
 594// "This function will never throw an exception."  It's an optional
 595// optimization tool, but we may need to use it to match glibc prototypes.
 596#ifndef __THROW    // I guess we're not on a glibc system
 597# define __THROW   // __THROW is just an optimization, so ok to make it ""
 598#endif
 599
 600//-------------------------------------------------------------------
 601// Configuration
 602//-------------------------------------------------------------------
 603
 604// Not all possible combinations of the following parameters make
 605// sense.  In particular, if kMaxSize increases, you may have to
 606// increase kNumClasses as well.
 607static const size_t kPageShift  = 12;
 608static const size_t kPageSize   = 1 << kPageShift;
 609static const size_t kMaxSize    = 8u * kPageSize;
 610static const size_t kAlignShift = 3;
 611static const size_t kAlignment  = 1 << kAlignShift;
 612static const size_t kNumClasses = 68;
 613
 614// Allocates a big block of memory for the pagemap once we reach more than
 615// 128MB
 616static const size_t kPageMapBigAllocationThreshold = 128 << 20;
 617
 618// Minimum number of pages to fetch from system at a time.  Must be
 619// significantly bigger than kPageSize to amortize system-call
 620// overhead, and also to reduce external fragementation.  Also, we
 621// should keep this value big because various incarnations of Linux
 622// have small limits on the number of mmap() regions per
 623// address-space.
 624static const size_t kMinSystemAlloc = 1 << (20 - kPageShift);
 625
 626// Number of objects to move between a per-thread list and a central
 627// list in one shot.  We want this to be not too small so we can
 628// amortize the lock overhead for accessing the central list.  Making
 629// it too big may temporarily cause unnecessary memory wastage in the
 630// per-thread free list until the scavenger cleans up the list.
 631static int num_objects_to_move[kNumClasses];
 632
 633// Maximum length we allow a per-thread free-list to have before we
 634// move objects from it into the corresponding central free-list.  We
 635// want this big to avoid locking the central free-list too often.  It
 636// should not hurt to make this list somewhat big because the
 637// scavenging code will shrink it down when its contents are not in use.
 638static const int kMaxFreeListLength = 256;
 639
 640// Lower and upper bounds on the per-thread cache sizes
 641static const size_t kMinThreadCacheSize = kMaxSize * 2;
 642static const size_t kMaxThreadCacheSize = 2 << 20;
 643
 644// Default bound on the total amount of thread caches
 645static const size_t kDefaultOverallThreadCacheSize = 16 << 20;
 646
 647// For all span-lengths < kMaxPages we keep an exact-size list.
 648// REQUIRED: kMaxPages >= kMinSystemAlloc;
 649static const size_t kMaxPages = kMinSystemAlloc;
 650
 651/* The smallest prime > 2^n */
 652static int primes_list[] = {
 653    // Small values might cause high rates of sampling
 654    // and hence commented out.
 655    // 2, 5, 11, 17, 37, 67, 131, 257,
 656    // 521, 1031, 2053, 4099, 8209, 16411,
 657    32771, 65537, 131101, 262147, 524309, 1048583,
 658    2097169, 4194319, 8388617, 16777259, 33554467 };
 659
 660// Twice the approximate gap between sampling actions.
 661// I.e., we take one sample approximately once every
 662//      tcmalloc_sample_parameter/2
 663// bytes of allocation, i.e., ~ once every 128KB.
 664// Must be a prime number.
 665#ifdef NO_TCMALLOC_SAMPLES
 666DEFINE_int64(tcmalloc_sample_parameter, 0,
 667             "Unused: code is compiled with NO_TCMALLOC_SAMPLES");
 668static size_t sample_period = 0;
 669#else
 670DEFINE_int64(tcmalloc_sample_parameter, 262147,
 671         "Twice the approximate gap between sampling actions."
 672         " Must be a prime number. Otherwise will be rounded up to a "
 673         " larger prime number");
 674static size_t sample_period = 262147;
 675#endif
 676
 677// Protects sample_period above
 678static SpinLock sample_period_lock = SPINLOCK_INITIALIZER;
 679
 680// Parameters for controlling how fast memory is returned to the OS.
 681
 682DEFINE_double(tcmalloc_release_rate, 1,
 683              "Rate at which we release unused memory to the system.  "
 684              "Zero means we never release memory back to the system.  "
 685              "Increase this flag to return memory faster; decrease it "
 686              "to return memory slower.  Reasonable rates are in the "
 687              "range [0,10]");
 688
 689//-------------------------------------------------------------------
 690// Mapping from size to size_class and vice versa
 691//-------------------------------------------------------------------
 692
 693// Sizes <= 1024 have an alignment >= 8.  So for such sizes we have an
 694// array indexed by ceil(size/8).  Sizes > 1024 have an alignment >= 128.
 695// So for these larger sizes we have an array indexed by ceil(size/128).
 696//
 697// We flatten both logical arrays into one physical array and use
 698// arithmetic to compute an appropriate index.  The constants used by
 699// ClassIndex() were selected to make the flattening work.
 700//
 701// Examples:
 702//   Size       Expression                      Index
 703//   -------------------------------------------------------
 704//   0          (0 + 7) / 8                     0
 705//   1          (1 + 7) / 8                     1
 706//   ...
 707//   1024       (1024 + 7) / 8                  128
 708//   1025       (1025 + 127 + (120<<7)) / 128   129
 709//   ...
 710//   32768      (32768 + 127 + (120<<7)) / 128  376
 711static const size_t kMaxSmallSize = 1024;
 712static const int shift_amount[2] = { 3, 7 };  // For divides by 8 or 128
 713static const int add_amount[2] = { 7, 127 + (120 << 7) };
 714static unsigned char class_array[377];
 715
 716// Compute index of the class_array[] entry for a given size
 717static inline int ClassIndex(size_t s) {
 718  const int i = (s > kMaxSmallSize);
 719  return static_cast<int>((s + add_amount[i]) >> shift_amount[i]);
 720}
 721
 722// Mapping from size class to max size storable in that class
 723static size_t class_to_size[kNumClasses];
 724
 725// Mapping from size class to number of pages to allocate at a time
 726static size_t class_to_pages[kNumClasses];
 727
 728// TransferCache is used to cache transfers of num_objects_to_move[size_class]
 729// back and forth between thread caches and the central cache for a given size
 730// class.
 731struct TCEntry {
 732  void *head;  // Head of chain of objects.
 733  void *tail;  // Tail of chain of objects.
 734};
 735// A central cache freelist can have anywhere from 0 to kNumTransferEntries
 736// slots to put link list chains into.  To keep memory usage bounded the total
 737// number of TCEntries across size classes is fixed.  Currently each size
 738// class is initially given one TCEntry which also means that the maximum any
 739// one class can have is kNumClasses.
 740static const int kNumTransferEntries = kNumClasses;
 741
 742// Note: the following only works for "n"s that fit in 32-bits, but
 743// that is fine since we only use it for small sizes.
 744static inline int LgFloor(size_t n) {
 745  int log = 0;
 746  for (int i = 4; i >= 0; --i) {
 747    int shift = (1 << i);
 748    size_t x = n >> shift;
 749    if (x != 0) {
 750      n = x;
 751      log += shift;
 752    }
 753  }
 754  ASSERT(n == 1);
 755  return log;
 756}
 757
 758// Some very basic linked list functions for dealing with using void * as
 759// storage.
 760
 761static inline void *SLL_Next(void *t) {
 762  return *(reinterpret_cast<void**>(t));
 763}
 764
 765static inline void SLL_SetNext(void *t, void *n) {
 766  *(reinterpret_cast<void**>(t)) = n;
 767}
 768
 769static inline void SLL_Push(void **list, void *element) {
 770  SLL_SetNext(element, *list);
 771  *list = element;
 772}
 773
 774static inline void *SLL_Pop(void **list) {
 775  void *result = *list;
 776  *list = SLL_Next(*list);
 777  return result;
 778}
 779
 780
 781// Remove N elements from a linked list to which head points.  head will be
 782// modified to point to the new head.  start and end will point to the first
 783// and last nodes of the range.  Note that end will point to NULL after this
 784// function is called.
 785static inline void SLL_PopRange(void **head, int N, void **start, void **end) {
 786  if (N == 0) {
 787    *start = NULL;
 788    *end = NULL;
 789    return;
 790  }
 791
 792  void *tmp = *head;
 793  for (int i = 1; i < N; ++i) {
 794    tmp = SLL_Next(tmp);
 795  }
 796
 797  *start = *head;
 798  *end = tmp;
 799  *head = SLL_Next(tmp);
 800  // Unlink range from list.
 801  SLL_SetNext(tmp, NULL);
 802}
 803
 804static inline void SLL_PushRange(void **head, void *start, void *end) {
 805  if (!start) return;
 806  SLL_SetNext(end, *head);
 807  *head = start;
 808}
 809
 810static inline size_t SLL_Size(void *head) {
 811  int count = 0;
 812  while (head) {
 813    count++;
 814    head = SLL_Next(head);
 815  }
 816  return count;
 817}
 818
 819// Setup helper functions.
 820
 821static ALWAYS_INLINE size_t SizeClass(size_t size) {
 822  return class_array[ClassIndex(size)];
 823}
 824
 825// Get the byte-size for a specified class
 826static ALWAYS_INLINE size_t ByteSizeForClass(size_t cl) {
 827  return class_to_size[cl];
 828}
 829static int NumMoveSize(size_t size) {
 830  if (size == 0) return 0;
 831  // Use approx 64k transfers between thread and central caches.
 832  int num = static_cast<int>(64.0 * 1024.0 / size);
 833  if (num < 2) num = 2;
 834  // Clamp well below kMaxFreeListLength to avoid ping pong between central
 835  // and thread caches.
 836  if (num > static_cast<int>(0.8 * kMaxFreeListLength))
 837    num = static_cast<int>(0.8 * kMaxFreeListLength);
 838
 839  // Also, avoid bringing in too many objects into small object free
 840  // lists.  There are lots of such lists, and if we allow each one to
 841  // fetch too many at a time, we end up having to scavenge too often
 842  // (especially when there are lots of threads and each thread gets a
 843  // small allowance for its thread cache).
 844  //
 845  // TODO: Make thread cache free list sizes dynamic so that we do not
 846  // have to equally divide a fixed resource amongst lots of threads.
 847  if (num > 32) num = 32;
 848
 849  return num;
 850}
 851
 852// Initialize the mapping arrays
 853static void InitSizeClasses() {
 854  // Do some sanity checking on add_amount[]/shift_amount[]/class_array[]
 855  if (ClassIndex(0) < 0) {
 856    MESSAGE("Invalid class index %d for size 0\n", ClassIndex(0));
 857    CRASH();
 858  }
 859  if (static_cast<size_t>(ClassIndex(kMaxSize)) >= sizeof(class_array)) {
 860    MESSAGE("Invalid class index %d for kMaxSize\n", ClassIndex(kMaxSize));
 861    CRASH();
 862  }
 863
 864  // Compute the size classes we want to use
 865  size_t sc = 1;   // Next size class to assign
 866  unsigned char alignshift = kAlignShift;
 867  int last_lg = -1;
 868  for (size_t size = kAlignment; size <= kMaxSize; size += (1 << alignshift)) {
 869    int lg = LgFloor(size);
 870    if (lg > last_lg) {
 871      // Increase alignment every so often.
 872      //
 873      // Since we double the alignment every time size doubles and
 874      // size >= 128, this means that space wasted due to alignment is
 875      // at most 16/128 i.e., 12.5%.  Plus we cap the alignment at 256
 876      // bytes, so the space wasted as a percentage starts falling for
 877      // sizes > 2K.
 878      if ((lg >= 7) && (alignshift < 8)) {
 879        alignshift++;
 880      }
 881      last_lg = lg;
 882    }
 883
 884    // Allocate enough pages so leftover is less than 1/8 of total.
 885    // This bounds wasted space to at most 12.5%.
 886    size_t psize = kPageSize;
 887    while ((psize % size) > (psize >> 3)) {
 888      psize += kPageSize;
 889    }
 890    const size_t my_pages = psize >> kPageShift;
 891
 892    if (sc > 1 && my_pages == class_to_pages[sc-1]) {
 893      // See if we can merge this into the previous class without
 894      // increasing the fragmentation of the previous class.
 895      const size_t my_objects = (my_pages << kPageShift) / size;
 896      const size_t prev_objects = (class_to_pages[sc-1] << kPageShift)
 897                                  / class_to_size[sc-1];
 898      if (my_objects == prev_objects) {
 899        // Adjust last class to include this size
 900        class_to_size[sc-1] = size;
 901        continue;
 902      }
 903    }
 904
 905    // Add new class
 906    class_to_pages[sc] = my_pages;
 907    class_to_size[sc] = size;
 908    sc++;
 909  }
 910  if (sc != kNumClasses) {
 911    MESSAGE("wrong number of size classes: found %" PRIuS " instead of %d\n",
 912            sc, int(kNumClasses));
 913    CRASH();
 914  }
 915
 916  // Initialize the mapping arrays
 917  int next_size = 0;
 918  for (unsigned char c = 1; c < kNumClasses; c++) {
 919    const size_t max_size_in_class = class_to_size[c];
 920    for (size_t s = next_size; s <= max_size_in_class; s += kAlignment) {
 921      class_array[ClassIndex(s)] = c;
 922    }
 923    next_size = static_cast<int>(max_size_in_class + kAlignment);
 924  }
 925
 926  // Double-check sizes just to be safe
 927  for (size_t size = 0; size <= kMaxSize; size++) {
 928    const size_t sc = SizeClass(size);
 929    if (sc == 0) {
 930      MESSAGE("Bad size class %" PRIuS " for %" PRIuS "\n", sc, size);
 931      CRASH();
 932    }
 933    if (sc > 1 && size <= class_to_size[sc-1]) {
 934      MESSAGE("Allocating unnecessarily large class %" PRIuS " for %" PRIuS
 935              "\n", sc, size);
 936      CRASH();
 937    }
 938    if (sc >= kNumClasses) {
 939      MESSAGE("Bad size class %" PRIuS " for %" PRIuS "\n", sc, size);
 940      CRASH();
 941    }
 942    const size_t s = class_to_size[sc];
 943    if (size > s) {
 944     MESSAGE("Bad size %" PRIuS " for %" PRIuS " (sc = %" PRIuS ")\n", s, size, sc);
 945      CRASH();
 946    }
 947    if (s == 0) {
 948      MESSAGE("Bad size %" PRIuS " for %" PRIuS " (sc = %" PRIuS ")\n", s, size, sc);
 949      CRASH();
 950    }
 951  }
 952
 953  // Initialize the num_objects_to_move array.
 954  for (size_t cl = 1; cl  < kNumClasses; ++cl) {
 955    num_objects_to_move[cl] = NumMoveSize(ByteSizeForClass(cl));
 956  }
 957
 958#ifndef WTF_CHANGES
 959  if (false) {
 960    // Dump class sizes and maximum external wastage per size class
 961    for (size_t cl = 1; cl  < kNumClasses; ++cl) {
 962      const int alloc_size = class_to_pages[cl] << kPageShift;
 963      const int alloc_objs = alloc_size / class_to_size[cl];
 964      const int min_used = (class_to_size[cl-1] + 1) * alloc_objs;
 965      const int max_waste = alloc_size - min_used;
 966      MESSAGE("SC %3d [ %8d .. %8d ] from %8d ; %2.0f%% maxwaste\n",
 967              int(cl),
 968              int(class_to_size[cl-1] + 1),
 969              int(class_to_size[cl]),
 970              int(class_to_pages[cl] << kPageShift),
 971              max_waste * 100.0 / alloc_size
 972              );
 973    }
 974  }
 975#endif
 976}
 977
 978// -------------------------------------------------------------------------
 979// Simple allocator for objects of a specified type.  External locking
 980// is required before accessing one of these objects.
 981// -------------------------------------------------------------------------
 982
 983// Metadata allocator -- keeps stats about how many bytes allocated
 984static uint64_t metadata_system_bytes = 0;
 985static void* MetaDataAlloc(size_t bytes) {
 986  void* result = TCMalloc_SystemAlloc(bytes, 0);
 987  if (result != NULL) {
 988    metadata_system_bytes += bytes;
 989  }
 990  return result;
 991}
 992
 993template <class T>
 994class PageHeapAllocator {
 995 private:
 996  // How much to allocate from system at a time
 997  static const size_t kAllocIncrement = 32 << 10;
 998
 999  // Aligned size of T
1000  static const size_t kAlignedSize
1001  = (((sizeof(T) + kAlignment - 1) / kAlignment) * kAlignment);
1002
1003  // Free area from which to carve new objects
1004  char* free_area_;
1005  size_t free_avail_;
1006
1007  // Linked list of all regions allocated by this allocator
1008  void* allocated_regions_;
1009
1010  // Free list of already carved objects
1011  void* free_list_;
1012
1013  // Number of allocated but unfreed objects
1014  int inuse_;
1015
1016 public:
1017  void Init() {
1018    ASSERT(kAlignedSize <= kAllocIncrement);
1019    inuse_ = 0;
1020    allocated_regions_ = 0;
1021    free_area_ = NULL;
1022    free_avail_ = 0;
1023    free_list_ = NULL;
1024  }
1025
1026  T* New() {
1027    // Consult free list
1028    void* result;
1029    if (free_list_ != NULL) {
1030      result = free_list_;
1031      free_list_ = *(reinterpret_cast<void**>(result));
1032    } else {
1033      if (free_avail_ < kAlignedSize) {
1034        // Need more room
1035        char* new_allocation = reinterpret_cast<char*>(MetaDataAlloc(kAllocIncrement));
1036        if (!new_allocation)
1037          CRASH();
1038
1039        *reinterpret_cast_ptr<void**>(new_allocation) = allocated_regions_;
1040        allocated_regions_ = new_allocation;
1041        free_area_ = new_allocation + kAlignedSize;
1042        free_avail_ = kAllocIncrement - kAlignedSize;
1043      }
1044      result = free_area_;
1045      free_area_ += kAlignedSize;
1046      free_avail_ -= kAlignedSize;
1047    }
1048    inuse_++;
1049    return reinterpret_cast<T*>(result);
1050  }
1051
1052  void Delete(T* p) {
1053    *(reinterpret_cast<void**>(p)) = free_list_;
1054    free_list_ = p;
1055    inuse_--;
1056  }
1057
1058  int inuse() const { return inuse_; }
1059
1060#if defined(WTF_CHANGES) && OS(DARWIN)
1061  template <class Recorder>
1062  void recordAdministrativeRegions(Recorder& recorder, const RemoteMemoryReader& reader)
1063  {
1064      for (void* adminAllocation = allocated_regions_; adminAllocation; adminAllocation = reader.nextEntryInLinkedList(reinterpret_cast<void**>(adminAllocation)))
1065          recorder.recordRegion(reinterpret_cast<vm_address_t>(adminAllocation), kAllocIncrement);
1066  }
1067#endif
1068};
1069
1070// -------------------------------------------------------------------------
1071// Span - a contiguous run of pages
1072// -------------------------------------------------------------------------
1073
1074// Type that can hold a page number
1075typedef uintptr_t PageID;
1076
1077// Type that can hold the length of a run of pages
1078typedef uintptr_t Length;
1079
1080static const Length kMaxValidPages = (~static_cast<Length>(0)) >> kPageShift;
1081
1082// Convert byte size into pages.  This won't overflow, but may return
1083// an unreasonably large value if bytes is huge enough.
1084static inline Length pages(size_t bytes) {
1085  return (bytes >> kPageShift) +
1086      ((bytes & (kPageSize - 1)) > 0 ? 1 : 0);
1087}
1088
1089// Convert a user size into the number of bytes that will actually be
1090// allocated
1091static size_t AllocationSize(size_t bytes) {
1092  if (bytes > kMaxSize) {
1093    // Large object: we allocate an integral number of pages
1094    ASSERT(bytes <= (kMaxValidPages << kPageShift));
1095    return pages(bytes) << kPageShift;
1096  } else {
1097    // Small object: find the size class to which it belongs
1098    return ByteSizeForClass(SizeClass(bytes));
1099  }
1100}
1101
1102// Information kept for a span (a contiguous run of pages).
1103struct Span {
1104  PageID        start;          // Starting page number
1105  Length        length;         // Number of pages in span
1106  Span*         next;           // Used when in link list
1107  Span*         prev;           // Used when in link list
1108  void*         objects;        // Linked list of free objects
1109  unsigned int  free : 1;       // Is the span free
1110#ifndef NO_TCMALLOC_SAMPLES
1111  unsigned int  sample : 1;     // Sampled object?
1112#endif
1113  unsigned int  sizeclass : 8;  // Size-class for small objects (or 0)
1114  unsigned int  refcount : 11;  // Number of non-free objects
1115  bool decommitted : 1;
1116
1117#undef SPAN_HISTORY
1118#ifdef SPAN_HISTORY
1119  // For debugging, we can keep a log events per span
1120  int nexthistory;
1121  char history[64];
1122  int value[64];
1123#endif
1124};
1125
1126#define ASSERT_SPAN_COMMITTED(span) ASSERT(!span->decommitted)
1127
1128#ifdef SPAN_HISTORY
1129void Event(Span* span, char op, int v = 0) {
1130  span->history[span->nexthistory] = op;
1131  span->value[span->nexthistory] = v;
1132  span->nexthistory++;
1133  if (span->nexthistory == sizeof(span->history)) span->nexthistory = 0;
1134}
1135#else
1136#define Event(s,o,v) ((void) 0)
1137#endif
1138
1139// Allocator/deallocator for spans
1140static PageHeapAllocator<Span> span_allocator;
1141static Span* NewSpan(PageID p, Length len) {
1142  Span* result = span_allocator.New();
1143  memset(result, 0, sizeof(*result));
1144  result->start = p;
1145  result->length = len;
1146#ifdef SPAN_HISTORY
1147  result->nexthistory = 0;
1148#endif
1149  return result;
1150}
1151
1152static inline void DeleteSpan(Span* span) {
1153#ifndef NDEBUG
1154  // In debug mode, trash the contents of deleted Spans
1155  memset(span, 0x3f, sizeof(*span));
1156#endif
1157  span_allocator.Delete(span);
1158}
1159
1160// -------------------------------------------------------------------------
1161// Doubly linked list of spans.
1162// -------------------------------------------------------------------------
1163
1164static inline void DLL_Init(Span* list) {
1165  list->next = list;
1166  list->prev = list;
1167}
1168
1169static inline void DLL_Remove(Span* span) {
1170  span->prev->next = span->next;
1171  span->next->prev = span->prev;
1172  span->prev = NULL;
1173  span->next = NULL;
1174}
1175
1176static ALWAYS_INLINE bool DLL_IsEmpty(const Span* list) {
1177  return list->next == list;
1178}
1179
1180static int DLL_Length(const Span* list) {
1181  int result = 0;
1182  for (Span* s = list->next; s != list; s = s->next) {
1183    result++;
1184  }
1185  return result;
1186}
1187
1188#if 0 /* Not needed at the moment -- causes compiler warnings if not used */
1189static void DLL_Print(const char* label, const Span* list) {
1190  MESSAGE("%-10s %p:", label, list);
1191  for (const Span* s = list->next; s != list; s = s->next) {
1192    MESSAGE(" <%p,%u,%u>", s, s->start, s->length);
1193  }
1194  MESSAGE("\n");
1195}
1196#endif
1197
1198static inline void DLL_Prepend(Span* list, Span* span) {
1199  ASSERT(span->next == NULL);
1200  ASSERT(span->prev == NULL);
1201  span->next = list->next;
1202  span->prev = list;
1203  list->next->prev = span;
1204  list->next = span;
1205}
1206
1207// -------------------------------------------------------------------------
1208// Stack traces kept for sampled allocations
1209//   The following state is protected by pageheap_lock_.
1210// -------------------------------------------------------------------------
1211
1212// size/depth are made the same size as a pointer so that some generic
1213// code below can conveniently cast them back and forth to void*.
1214static const int kMaxStackDepth = 31;
1215struct StackTrace {
1216  uintptr_t size;          // Size of object
1217  uintptr_t depth;         // Number of PC values stored in array below
1218  void*     stack[kMaxStackDepth];
1219};
1220static PageHeapAllocator<StackTrace> stacktrace_allocator;
1221static Span sampled_objects;
1222
1223// -------------------------------------------------------------------------
1224// Map from page-id to per-page data
1225// -------------------------------------------------------------------------
1226
1227// We use PageMap2<> for 32-bit and PageMap3<> for 64-bit machines.
1228// We also use a simple one-level cache for hot PageID-to-sizeclass mappings,
1229// because sometimes the sizeclass is all the information we need.
1230
1231// Selector class -- general selector uses 3-level map
1232template <int BITS> class MapSelector {
1233 public:
1234  typedef TCMalloc_PageMap3<BITS-kPageShift> Type;
1235  typedef PackedCache<BITS, uint64_t> CacheType;
1236};
1237
1238#if defined(WTF_CHANGES)
1239#if CPU(X86_64)
1240// On all known X86-64 platforms, the upper 16 bits are always unused and therefore 
1241// can be excluded from the PageMap key.
1242// See http://en.wikipedia.org/wiki/X86-64#Virtual_address_space_details
1243
1244static const size_t kBitsUnusedOn64Bit = 16;
1245#else
1246static const size_t kBitsUnusedOn64Bit = 0;
1247#endif
1248
1249// A three-level map for 64-bit machines
1250template <> class MapSelector<64> {
1251 public:
1252  typedef TCMalloc_PageMap3<64 - kPageShift - kBitsUnusedOn64Bit> Type;
1253  typedef PackedCache<64, uint64_t> CacheType;
1254};
1255#endif
1256
1257// A two-level map for 32-bit machines
1258template <> class MapSelector<32> {
1259 public:
1260  typedef TCMalloc_PageMap2<32 - kPageShift> Type;
1261  typedef PackedCache<32 - kPageShift, uint16_t> CacheType;
1262};
1263
1264// -------------------------------------------------------------------------
1265// Page-level allocator
1266//  * Eager coalescing
1267//
1268// Heap for page-level allocation.  We allow allocating and freeing a
1269// contiguous runs of pages (called a "span").
1270// -------------------------------------------------------------------------
1271
1272#if USE_BACKGROUND_THREAD_TO_SCAVENGE_MEMORY
1273// The page heap maintains a free list for spans that are no longer in use by
1274// the central cache or any thread caches. We use a background thread to
1275// periodically scan the free list and release a percentage of it back to the OS.
1276
1277// If free_committed_pages_ exceeds kMinimumFreeCommittedPageCount, the
1278// background thread:
1279//     - wakes up
1280//     - pauses for kScavengeDelayInSeconds
1281//     - returns to the OS a percentage of the memory that remained unused during
1282//       that pause (kScavengePercentage * min_free_committed_pages_since_last_scavenge_)
1283// The goal of this strategy is to reduce memory pressure in a timely fashion
1284// while avoiding thrashing the OS allocator.
1285
1286// Time delay before the page heap scavenger will consider returning pages to
1287// the OS.
1288static const int kScavengeDelayInSeconds = 2;
1289
1290// Approximate percentage of free committed pages to return to the OS in one
1291// scavenge.
1292static const float kScavengePercentage = .5f;
1293
1294// number of span lists to keep spans in when memory is returned.
1295static const int kMinSpanListsWithSpans = 32;
1296
1297// Number of free committed pages that we want to keep around.  The minimum number of pages used when there
1298// is 1 span in each of the first kMinSpanListsWithSpans spanlists.  Currently 528 pages.
1299static const size_t kMinimumFreeCommittedPageCount = kMinSpanListsWithSpans * ((1.0f+kMinSpanListsWithSpans) / 2.0f);
1300
1301#endif
1302
1303class TCMalloc_PageHeap {
1304 public:
1305  void init();
1306
1307  // Allocate a run of "n" pages.  Returns zero if out of memory.
1308  Span* New(Length n);
1309
1310  // Delete the span "[p, p+n-1]".
1311  // REQUIRES: span was returned by earlier call to New() and
1312  //           has not yet been deleted.
1313  void Delete(Span* span);
1314
1315  // Mark an allocated span as being used for small objects of the
1316  // specified size-class.
1317  // REQUIRES: span was returned by an earlier call to New()
1318  //           and has not yet been deleted.
1319  void RegisterSizeClass(Span* span, size_t sc);
1320
1321  // Split an allocated span into two spans: one of length "n" pages
1322  // followed by another span of length "span->length - n" pages.
1323  // Modifies "*span" to point to the first span of length "n" pages.
1324  // Returns a pointer to the second span.
1325  //
1326  // REQUIRES: "0 < n < span->length"
1327  // REQUIRES: !span->free
1328  // REQUIRES: span->sizeclass == 0
1329  Span* Split(Span* span, Length n);
1330
1331  // Return the descriptor for the specified page.
1332  inline Span* GetDescriptor(PageID p) const {
1333    return reinterpret_cast<Span*>(pagemap_.get(p));
1334  }
1335
1336#ifdef WTF_CHANGES
1337  inline Span* GetDescriptorEnsureSafe(PageID p)
1338  {
1339      pagemap_.Ensure(p, 1);
1340      return GetDescriptor(p);
1341  }
1342    
1343  size_t ReturnedBytes() const;
1344#endif
1345
1346  // Dump state to stderr
1347#ifndef WTF_CHANGES
1348  void Dump(TCMalloc_Printer* out);
1349#endif
1350
1351  // Return number of bytes allocated from system
1352  inline uint64_t SystemBytes() const { return system_bytes_; }
1353
1354  // Return number of free bytes in heap
1355  uint64_t FreeBytes() const {
1356    return (static_cast<uint64_t>(free_pages_) << kPageShift);
1357  }
1358
1359  bool Check();
1360  bool CheckList(Span* list, Length min_pages, Length max_pages);
1361
1362  // Release all pages on the free list for reuse by the OS:
1363  void ReleaseFreePages();
1364
1365  // Return 0 if we have no information, or else the correct sizeclass for p.
1366  // Reads and writes to pagemap_cache_ do not require locking.
1367  // The entries are 64 bits on 64-bit hardware and 16 bits on
1368  // 32-bit hardware, and we don't mind raciness as long as each read of
1369  // an entry yields a valid entry, not a partially updated entry.
1370  size_t GetSizeClassIfCached(PageID p) const {
1371    return pagemap_cache_.GetOrDefault(p, 0);
1372  }
1373  void CacheSizeClass(PageID p, size_t cl) const { pagemap_cache_.Put(p, cl); }
1374
1375 private:
1376  // Pick the appropriate map and cache types based on pointer size
1377  typedef MapSelector<8*sizeof(uintptr_t)>::Type PageMap;
1378  typedef MapSelector<8*sizeof(uintptr_t)>::CacheType PageMapCache;
1379  PageMap pagemap_;
1380  mutable PageMapCache pagemap_cache_;
1381
1382  // We segregate spans of a given size into two circular linked
1383  // lists: one for normal spans, and one for spans whose memory
1384  // has been returned to the system.
1385  struct SpanList {
1386    Span        normal;
1387    Span        returned;
1388  };
1389
1390  // List of free spans of length >= kMaxPages
1391  SpanList large_;
1392
1393  // Array mapping from span length to a doubly linked list of free spans
1394  SpanList free_[kMaxPages];
1395
1396  // Number of pages kept in free lists
1397  uintptr_t free_pages_;
1398
1399  // Bytes allocated from system
1400  uint64_t system_bytes_;
1401
1402#if USE_BACKGROUND_THREAD_TO_SCAVENGE_MEMORY
1403  // Number of pages kept in free lists that are still committed.
1404  Length free_committed_pages_;
1405
1406  // Minimum number of free committed pages since last scavenge. (Can be 0 if
1407  // we've committed new pages since the last scavenge.)
1408  Length min_free_committed_pages_since_last_scavenge_;
1409#endif
1410
1411  bool GrowHeap(Length n);
1412
1413  // REQUIRES   span->length >= n
1414  // Remove span from its free list, and move any leftover part of
1415  // span into appropriate free lists.  Also update "span" to have
1416  // length exactly "n" and mark it as non-free so it can be returned
1417  // to the client.
1418  //
1419  // "released" is true iff "span" was found on a "returned" list.
1420  void Carve(Span* span, Length n, bool released);
1421
1422  void RecordSpan(Span* span) {
1423    pagemap_.set(span->start, span);
1424    if (span->length > 1) {
1425      pagemap_.set(span->start + span->length - 1, span);
1426    }
1427  }
1428  
1429    // Allocate a large span of length == n.  If successful, returns a
1430  // span of exactly the specified length.  Else, returns NULL.
1431  Span* AllocLarge(Length n);
1432
1433#if !USE_BACKGROUND_THREAD_TO_SCAVENGE_MEMORY
1434  // Incrementally release some memory to the system.
1435  // IncrementalScavenge(n) is called whenever n pages are freed.
1436  void IncrementalScavenge(Length n);
1437#endif
1438
1439  // Number of pages to deallocate before doing more scavenging
1440  int64_t scavenge_counter_;
1441
1442  // Index of last free list we scavenged
1443  size_t scavenge_index_;
1444  
1445#if defined(WTF_CHANGES) && OS(DARWIN)
1446  friend class FastMallocZone;
1447#endif
1448
1449#if USE_BACKGROUND_THREAD_TO_SCAVENGE_MEMORY
1450  void initializeScavenger();
1451  ALWAYS_INLINE void signalScavenger();
1452  void scavenge();
1453  ALWAYS_INLINE bool shouldScavenge() const;
1454
1455#if HAVE(DISPATCH_H) || OS(WINDOWS)
1456  void periodicScavenge();
1457  ALWAYS_INLINE bool isScavengerSuspended();
1458  ALWAYS_INLINE void scheduleScavenger();
1459  ALWAYS_INLINE void rescheduleScavenger();
1460  ALWAYS_INLINE void suspendScavenger();
1461#endif
1462
1463#if HAVE(DISPATCH_H)
1464  dispatch_queue_t m_scavengeQueue;
1465  dispatch_source_t m_scavengeTimer;
1466  bool m_scavengingSuspended;
1467#elif OS(WINDOWS)
1468  static void CALLBACK scavengerTimerFired(void*, BOOLEAN);
1469  HANDLE m_scavengeQueueTimer;
1470#else 
1471  static NO_RETURN_WITH_VALUE void* runScavengerThread(void*);
1472  NO_RETURN void scavengerThread();
1473
1474  // Keeps track of whether the background thread is actively scavenging memory every kScavengeDelayInSeconds, or
1475  // it's blocked waiting for more pages to be deleted.
1476  bool m_scavengeThreadActive;
1477
1478  pthread_mutex_t m_scavengeMutex;
1479  pthread_cond_t m_scavengeCondition;
1480#endif
1481
1482#endif  // USE_BACKGROUND_THREAD_TO_SCAVENGE_MEMORY
1483};
1484
1485void TCMalloc_PageHeap::init()
1486{
1487  pagemap_.init(MetaDataAlloc);
1488  pagemap_cache_ = PageMapCache(0);
1489  free_pages_ = 0;
1490  system_bytes_ = 0;
1491
1492#if USE_BACKGROUND_THREAD_TO_SCAVENGE_MEMORY
1493  free_committed_pages_ = 0;
1494  min_free_committed_pages_since_last_scavenge_ = 0;
1495#endif  // USE_BACKGROUND_THREAD_TO_SCAVENGE_MEMORY
1496
1497  scavenge_counter_ = 0;
1498  // Start scavenging at kMaxPages list
1499  scavenge_index_ = kMaxPages-1;
1500  COMPILE_ASSERT(kNumClasses <= (1 << PageMapCache::kValuebits), valuebits);
1501  DLL_Init(&large_.normal);
1502  DLL_Init(&large_.returned);
1503  for (size_t i = 0; i < kMaxPages; i++) {
1504    DLL_Init(&free_[i].normal);
1505    DLL_Init(&free_[i].returned);
1506  }
1507
1508#if USE_BACKGROUND_THREAD_TO_SCAVENGE_MEMORY
1509  initializeScavenger();
1510#endif  // USE_BACKGROUND_THREAD_TO_SCAVENGE_MEMORY
1511}
1512
1513#if USE_BACKGROUND_THREAD_TO_SCAVENGE_MEMORY
1514
1515#if HAVE(DISPATCH_H)
1516
1517void TCMalloc_PageHeap::initializeScavenger()
1518{
1519    m_scavengeQueue = dispatch_queue_create("com.apple.JavaScriptCore.FastMallocSavenger", NULL);
1520    m_scavengeTimer = dispatch_source_create(DISPATCH_SOURCE_TYPE_TIMER, 0, 0, m_scavengeQueue);
1521    dispatch_time_t startTime = dispatch_time(DISPATCH_TIME_NOW, kScavengeDelayInSeconds * NSEC_PER_SEC);
1522    dispatch_source_set_timer(m_scavengeTimer, startTime, kScavengeDelayInSeconds * NSEC_PER_SEC, 1000 * NSEC_PER_USEC);
1523    dispatch_source_set_event_handler(m_scavengeTimer, ^{ periodicScavenge(); });
1524    m_scavengingSuspended = true;
1525}
1526
1527ALWAYS_INLINE bool TCMalloc_PageHeap::isScavengerSuspended()
1528{
1529    ASSERT(IsHeld(pageheap_lock));
1530    return m_scavengingSuspended;
1531}
1532
1533ALWAYS_INLINE void TCMalloc_PageHeap::scheduleScavenger()
1534{
1535    ASSERT(IsHeld(pageheap_lock));
1536    m_scavengingSuspended = false;
1537    dispatch_resume(m_scavengeTimer);
1538}
1539
1540ALWAYS_INLINE void TCMalloc_PageHeap::rescheduleScavenger()
1541{
1542    // Nothing to do here for libdispatch.
1543}
1544
1545ALWAYS_INLINE void TCMalloc_PageHeap::suspendScavenger()
1546{
1547    ASSERT(IsHeld(pageheap_lock));
1548    m_scavengingSuspended = true;
1549    dispatch_suspend(m_scavengeTimer);
1550}
1551
1552#elif OS(WINDOWS)
1553
1554void TCMalloc_PageHeap::scavenger

Large files files are truncated, but you can click here to view the full file