PageRenderTime 53ms CodeModel.GetById 15ms RepoModel.GetById 0ms app.codeStats 0ms

/jemalloc-3.0.0/include/jemalloc/internal/arena.h

#
C Header | 975 lines | 550 code | 121 blank | 304 comment | 87 complexity | bd595edff0c5a2dacb493b7ec79f01d4 MD5 | raw file
Possible License(s): BSD-2-Clause
  1. /******************************************************************************/
  2. #ifdef JEMALLOC_H_TYPES
  3. /*
  4. * RUN_MAX_OVRHD indicates maximum desired run header overhead. Runs are sized
  5. * as small as possible such that this setting is still honored, without
  6. * violating other constraints. The goal is to make runs as small as possible
  7. * without exceeding a per run external fragmentation threshold.
  8. *
  9. * We use binary fixed point math for overhead computations, where the binary
  10. * point is implicitly RUN_BFP bits to the left.
  11. *
  12. * Note that it is possible to set RUN_MAX_OVRHD low enough that it cannot be
  13. * honored for some/all object sizes, since when heap profiling is enabled
  14. * there is one pointer of header overhead per object (plus a constant). This
  15. * constraint is relaxed (ignored) for runs that are so small that the
  16. * per-region overhead is greater than:
  17. *
  18. * (RUN_MAX_OVRHD / (reg_interval << (3+RUN_BFP))
  19. */
  20. #define RUN_BFP 12
  21. /* \/ Implicit binary fixed point. */
  22. #define RUN_MAX_OVRHD 0x0000003dU
  23. #define RUN_MAX_OVRHD_RELAX 0x00001800U
  24. /* Maximum number of regions in one run. */
  25. #define LG_RUN_MAXREGS 11
  26. #define RUN_MAXREGS (1U << LG_RUN_MAXREGS)
  27. /*
  28. * Minimum redzone size. Redzones may be larger than this if necessary to
  29. * preserve region alignment.
  30. */
  31. #define REDZONE_MINSIZE 16
  32. /*
  33. * The minimum ratio of active:dirty pages per arena is computed as:
  34. *
  35. * (nactive >> opt_lg_dirty_mult) >= ndirty
  36. *
  37. * So, supposing that opt_lg_dirty_mult is 5, there can be no less than 32
  38. * times as many active pages as dirty pages.
  39. */
  40. #define LG_DIRTY_MULT_DEFAULT 5
  41. typedef struct arena_chunk_map_s arena_chunk_map_t;
  42. typedef struct arena_chunk_s arena_chunk_t;
  43. typedef struct arena_run_s arena_run_t;
  44. typedef struct arena_bin_info_s arena_bin_info_t;
  45. typedef struct arena_bin_s arena_bin_t;
  46. typedef struct arena_s arena_t;
  47. #endif /* JEMALLOC_H_TYPES */
  48. /******************************************************************************/
  49. #ifdef JEMALLOC_H_STRUCTS
  50. /* Each element of the chunk map corresponds to one page within the chunk. */
  51. struct arena_chunk_map_s {
  52. #ifndef JEMALLOC_PROF
  53. /*
  54. * Overlay prof_ctx in order to allow it to be referenced by dead code.
  55. * Such antics aren't warranted for per arena data structures, but
  56. * chunk map overhead accounts for a percentage of memory, rather than
  57. * being just a fixed cost.
  58. */
  59. union {
  60. #endif
  61. union {
  62. /*
  63. * Linkage for run trees. There are two disjoint uses:
  64. *
  65. * 1) arena_t's runs_avail_{clean,dirty} trees.
  66. * 2) arena_run_t conceptually uses this linkage for in-use
  67. * non-full runs, rather than directly embedding linkage.
  68. */
  69. rb_node(arena_chunk_map_t) rb_link;
  70. /*
  71. * List of runs currently in purgatory. arena_chunk_purge()
  72. * temporarily allocates runs that contain dirty pages while
  73. * purging, so that other threads cannot use the runs while the
  74. * purging thread is operating without the arena lock held.
  75. */
  76. ql_elm(arena_chunk_map_t) ql_link;
  77. } u;
  78. /* Profile counters, used for large object runs. */
  79. prof_ctx_t *prof_ctx;
  80. #ifndef JEMALLOC_PROF
  81. }; /* union { ... }; */
  82. #endif
  83. /*
  84. * Run address (or size) and various flags are stored together. The bit
  85. * layout looks like (assuming 32-bit system):
  86. *
  87. * ???????? ???????? ????nnnn nnnndula
  88. *
  89. * ? : Unallocated: Run address for first/last pages, unset for internal
  90. * pages.
  91. * Small: Run page offset.
  92. * Large: Run size for first page, unset for trailing pages.
  93. * n : binind for small size class, BININD_INVALID for large size class.
  94. * d : dirty?
  95. * u : unzeroed?
  96. * l : large?
  97. * a : allocated?
  98. *
  99. * Following are example bit patterns for the three types of runs.
  100. *
  101. * p : run page offset
  102. * s : run size
  103. * n : binind for size class; large objects set these to BININD_INVALID
  104. * except for promoted allocations (see prof_promote)
  105. * x : don't care
  106. * - : 0
  107. * + : 1
  108. * [DULA] : bit set
  109. * [dula] : bit unset
  110. *
  111. * Unallocated (clean):
  112. * ssssssss ssssssss ssss++++ ++++du-a
  113. * xxxxxxxx xxxxxxxx xxxxxxxx xxxx-Uxx
  114. * ssssssss ssssssss ssss++++ ++++dU-a
  115. *
  116. * Unallocated (dirty):
  117. * ssssssss ssssssss ssss++++ ++++D--a
  118. * xxxxxxxx xxxxxxxx xxxxxxxx xxxxxxxx
  119. * ssssssss ssssssss ssss++++ ++++D--a
  120. *
  121. * Small:
  122. * pppppppp pppppppp ppppnnnn nnnnd--A
  123. * pppppppp pppppppp ppppnnnn nnnn---A
  124. * pppppppp pppppppp ppppnnnn nnnnd--A
  125. *
  126. * Large:
  127. * ssssssss ssssssss ssss++++ ++++D-LA
  128. * xxxxxxxx xxxxxxxx xxxxxxxx xxxxxxxx
  129. * -------- -------- ----++++ ++++D-LA
  130. *
  131. * Large (sampled, size <= PAGE):
  132. * ssssssss ssssssss ssssnnnn nnnnD-LA
  133. *
  134. * Large (not sampled, size == PAGE):
  135. * ssssssss ssssssss ssss++++ ++++D-LA
  136. */
  137. size_t bits;
  138. #define CHUNK_MAP_BININD_SHIFT 4
  139. #define BININD_INVALID ((size_t)0xffU)
  140. /* CHUNK_MAP_BININD_MASK == (BININD_INVALID << CHUNK_MAP_BININD_SHIFT) */
  141. #define CHUNK_MAP_BININD_MASK ((size_t)0xff0U)
  142. #define CHUNK_MAP_BININD_INVALID CHUNK_MAP_BININD_MASK
  143. #define CHUNK_MAP_FLAGS_MASK ((size_t)0xcU)
  144. #define CHUNK_MAP_DIRTY ((size_t)0x8U)
  145. #define CHUNK_MAP_UNZEROED ((size_t)0x4U)
  146. #define CHUNK_MAP_LARGE ((size_t)0x2U)
  147. #define CHUNK_MAP_ALLOCATED ((size_t)0x1U)
  148. #define CHUNK_MAP_KEY CHUNK_MAP_ALLOCATED
  149. };
  150. typedef rb_tree(arena_chunk_map_t) arena_avail_tree_t;
  151. typedef rb_tree(arena_chunk_map_t) arena_run_tree_t;
  152. /* Arena chunk header. */
  153. struct arena_chunk_s {
  154. /* Arena that owns the chunk. */
  155. arena_t *arena;
  156. /* Linkage for the arena's chunks_dirty list. */
  157. ql_elm(arena_chunk_t) link_dirty;
  158. /*
  159. * True if the chunk is currently in the chunks_dirty list, due to
  160. * having at some point contained one or more dirty pages. Removal
  161. * from chunks_dirty is lazy, so (dirtied && ndirty == 0) is possible.
  162. */
  163. bool dirtied;
  164. /* Number of dirty pages. */
  165. size_t ndirty;
  166. /*
  167. * Map of pages within chunk that keeps track of free/large/small. The
  168. * first map_bias entries are omitted, since the chunk header does not
  169. * need to be tracked in the map. This omission saves a header page
  170. * for common chunk sizes (e.g. 4 MiB).
  171. */
  172. arena_chunk_map_t map[1]; /* Dynamically sized. */
  173. };
  174. typedef rb_tree(arena_chunk_t) arena_chunk_tree_t;
  175. struct arena_run_s {
  176. /* Bin this run is associated with. */
  177. arena_bin_t *bin;
  178. /* Index of next region that has never been allocated, or nregs. */
  179. uint32_t nextind;
  180. /* Number of free regions in run. */
  181. unsigned nfree;
  182. };
  183. /*
  184. * Read-only information associated with each element of arena_t's bins array
  185. * is stored separately, partly to reduce memory usage (only one copy, rather
  186. * than one per arena), but mainly to avoid false cacheline sharing.
  187. *
  188. * Each run has the following layout:
  189. *
  190. * /--------------------\
  191. * | arena_run_t header |
  192. * | ... |
  193. * bitmap_offset | bitmap |
  194. * | ... |
  195. * ctx0_offset | ctx map |
  196. * | ... |
  197. * |--------------------|
  198. * | redzone |
  199. * reg0_offset | region 0 |
  200. * | redzone |
  201. * |--------------------| \
  202. * | redzone | |
  203. * | region 1 | > reg_interval
  204. * | redzone | /
  205. * |--------------------|
  206. * | ... |
  207. * | ... |
  208. * | ... |
  209. * |--------------------|
  210. * | redzone |
  211. * | region nregs-1 |
  212. * | redzone |
  213. * |--------------------|
  214. * | alignment pad? |
  215. * \--------------------/
  216. *
  217. * reg_interval has at least the same minimum alignment as reg_size; this
  218. * preserves the alignment constraint that sa2u() depends on. Alignment pad is
  219. * either 0 or redzone_size; it is present only if needed to align reg0_offset.
  220. */
  221. struct arena_bin_info_s {
  222. /* Size of regions in a run for this bin's size class. */
  223. size_t reg_size;
  224. /* Redzone size. */
  225. size_t redzone_size;
  226. /* Interval between regions (reg_size + (redzone_size << 1)). */
  227. size_t reg_interval;
  228. /* Total size of a run for this bin's size class. */
  229. size_t run_size;
  230. /* Total number of regions in a run for this bin's size class. */
  231. uint32_t nregs;
  232. /*
  233. * Offset of first bitmap_t element in a run header for this bin's size
  234. * class.
  235. */
  236. uint32_t bitmap_offset;
  237. /*
  238. * Metadata used to manipulate bitmaps for runs associated with this
  239. * bin.
  240. */
  241. bitmap_info_t bitmap_info;
  242. /*
  243. * Offset of first (prof_ctx_t *) in a run header for this bin's size
  244. * class, or 0 if (config_prof == false || opt_prof == false).
  245. */
  246. uint32_t ctx0_offset;
  247. /* Offset of first region in a run for this bin's size class. */
  248. uint32_t reg0_offset;
  249. };
  250. struct arena_bin_s {
  251. /*
  252. * All operations on runcur, runs, and stats require that lock be
  253. * locked. Run allocation/deallocation are protected by the arena lock,
  254. * which may be acquired while holding one or more bin locks, but not
  255. * vise versa.
  256. */
  257. malloc_mutex_t lock;
  258. /*
  259. * Current run being used to service allocations of this bin's size
  260. * class.
  261. */
  262. arena_run_t *runcur;
  263. /*
  264. * Tree of non-full runs. This tree is used when looking for an
  265. * existing run when runcur is no longer usable. We choose the
  266. * non-full run that is lowest in memory; this policy tends to keep
  267. * objects packed well, and it can also help reduce the number of
  268. * almost-empty chunks.
  269. */
  270. arena_run_tree_t runs;
  271. /* Bin statistics. */
  272. malloc_bin_stats_t stats;
  273. };
  274. struct arena_s {
  275. /* This arena's index within the arenas array. */
  276. unsigned ind;
  277. /*
  278. * Number of threads currently assigned to this arena. This field is
  279. * protected by arenas_lock.
  280. */
  281. unsigned nthreads;
  282. /*
  283. * There are three classes of arena operations from a locking
  284. * perspective:
  285. * 1) Thread asssignment (modifies nthreads) is protected by
  286. * arenas_lock.
  287. * 2) Bin-related operations are protected by bin locks.
  288. * 3) Chunk- and run-related operations are protected by this mutex.
  289. */
  290. malloc_mutex_t lock;
  291. arena_stats_t stats;
  292. /*
  293. * List of tcaches for extant threads associated with this arena.
  294. * Stats from these are merged incrementally, and at exit.
  295. */
  296. ql_head(tcache_t) tcache_ql;
  297. uint64_t prof_accumbytes;
  298. /* List of dirty-page-containing chunks this arena manages. */
  299. ql_head(arena_chunk_t) chunks_dirty;
  300. /*
  301. * In order to avoid rapid chunk allocation/deallocation when an arena
  302. * oscillates right on the cusp of needing a new chunk, cache the most
  303. * recently freed chunk. The spare is left in the arena's chunk trees
  304. * until it is deleted.
  305. *
  306. * There is one spare chunk per arena, rather than one spare total, in
  307. * order to avoid interactions between multiple threads that could make
  308. * a single spare inadequate.
  309. */
  310. arena_chunk_t *spare;
  311. /* Number of pages in active runs. */
  312. size_t nactive;
  313. /*
  314. * Current count of pages within unused runs that are potentially
  315. * dirty, and for which madvise(... MADV_DONTNEED) has not been called.
  316. * By tracking this, we can institute a limit on how much dirty unused
  317. * memory is mapped for each arena.
  318. */
  319. size_t ndirty;
  320. /*
  321. * Approximate number of pages being purged. It is possible for
  322. * multiple threads to purge dirty pages concurrently, and they use
  323. * npurgatory to indicate the total number of pages all threads are
  324. * attempting to purge.
  325. */
  326. size_t npurgatory;
  327. /*
  328. * Size/address-ordered trees of this arena's available runs. The trees
  329. * are used for first-best-fit run allocation. The dirty tree contains
  330. * runs with dirty pages (i.e. very likely to have been touched and
  331. * therefore have associated physical pages), whereas the clean tree
  332. * contains runs with pages that either have no associated physical
  333. * pages, or have pages that the kernel may recycle at any time due to
  334. * previous madvise(2) calls. The dirty tree is used in preference to
  335. * the clean tree for allocations, because using dirty pages reduces
  336. * the amount of dirty purging necessary to keep the active:dirty page
  337. * ratio below the purge threshold.
  338. */
  339. arena_avail_tree_t runs_avail_clean;
  340. arena_avail_tree_t runs_avail_dirty;
  341. /* bins is used to store trees of free regions. */
  342. arena_bin_t bins[NBINS];
  343. };
  344. #endif /* JEMALLOC_H_STRUCTS */
  345. /******************************************************************************/
  346. #ifdef JEMALLOC_H_EXTERNS
  347. extern ssize_t opt_lg_dirty_mult;
  348. /*
  349. * small_size2bin is a compact lookup table that rounds request sizes up to
  350. * size classes. In order to reduce cache footprint, the table is compressed,
  351. * and all accesses are via the SMALL_SIZE2BIN macro.
  352. */
  353. extern uint8_t const small_size2bin[];
  354. #define SMALL_SIZE2BIN(s) (small_size2bin[(s-1) >> LG_TINY_MIN])
  355. extern arena_bin_info_t arena_bin_info[NBINS];
  356. /* Number of large size classes. */
  357. #define nlclasses (chunk_npages - map_bias)
  358. void arena_purge_all(arena_t *arena);
  359. void arena_prof_accum(arena_t *arena, uint64_t accumbytes);
  360. void arena_tcache_fill_small(arena_t *arena, tcache_bin_t *tbin,
  361. size_t binind, uint64_t prof_accumbytes);
  362. void arena_alloc_junk_small(void *ptr, arena_bin_info_t *bin_info,
  363. bool zero);
  364. void arena_dalloc_junk_small(void *ptr, arena_bin_info_t *bin_info);
  365. void *arena_malloc_small(arena_t *arena, size_t size, bool zero);
  366. void *arena_malloc_large(arena_t *arena, size_t size, bool zero);
  367. void *arena_palloc(arena_t *arena, size_t size, size_t alignment, bool zero);
  368. void arena_prof_promoted(const void *ptr, size_t size);
  369. void arena_dalloc_bin_locked(arena_t *arena, arena_chunk_t *chunk, void *ptr,
  370. arena_chunk_map_t *mapelm);
  371. void arena_dalloc_bin(arena_t *arena, arena_chunk_t *chunk, void *ptr,
  372. size_t pageind, arena_chunk_map_t *mapelm);
  373. void arena_dalloc_small(arena_t *arena, arena_chunk_t *chunk, void *ptr,
  374. size_t pageind);
  375. void arena_dalloc_large_locked(arena_t *arena, arena_chunk_t *chunk,
  376. void *ptr);
  377. void arena_dalloc_large(arena_t *arena, arena_chunk_t *chunk, void *ptr);
  378. void arena_stats_merge(arena_t *arena, size_t *nactive, size_t *ndirty,
  379. arena_stats_t *astats, malloc_bin_stats_t *bstats,
  380. malloc_large_stats_t *lstats);
  381. void *arena_ralloc_no_move(void *ptr, size_t oldsize, size_t size,
  382. size_t extra, bool zero);
  383. void *arena_ralloc(void *ptr, size_t oldsize, size_t size, size_t extra,
  384. size_t alignment, bool zero, bool try_tcache);
  385. bool arena_new(arena_t *arena, unsigned ind);
  386. void arena_boot(void);
  387. void arena_prefork(arena_t *arena);
  388. void arena_postfork_parent(arena_t *arena);
  389. void arena_postfork_child(arena_t *arena);
  390. #endif /* JEMALLOC_H_EXTERNS */
  391. /******************************************************************************/
  392. #ifdef JEMALLOC_H_INLINES
  393. #ifndef JEMALLOC_ENABLE_INLINE
  394. arena_chunk_map_t *arena_mapp_get(arena_chunk_t *chunk, size_t pageind);
  395. size_t *arena_mapbitsp_get(arena_chunk_t *chunk, size_t pageind);
  396. size_t arena_mapbits_get(arena_chunk_t *chunk, size_t pageind);
  397. size_t arena_mapbits_unallocated_size_get(arena_chunk_t *chunk,
  398. size_t pageind);
  399. size_t arena_mapbits_large_size_get(arena_chunk_t *chunk, size_t pageind);
  400. size_t arena_mapbits_small_runind_get(arena_chunk_t *chunk, size_t pageind);
  401. size_t arena_mapbits_binind_get(arena_chunk_t *chunk, size_t pageind);
  402. size_t arena_mapbits_dirty_get(arena_chunk_t *chunk, size_t pageind);
  403. size_t arena_mapbits_unzeroed_get(arena_chunk_t *chunk, size_t pageind);
  404. size_t arena_mapbits_large_get(arena_chunk_t *chunk, size_t pageind);
  405. size_t arena_mapbits_allocated_get(arena_chunk_t *chunk, size_t pageind);
  406. void arena_mapbits_unallocated_set(arena_chunk_t *chunk, size_t pageind,
  407. size_t size, size_t flags);
  408. void arena_mapbits_unallocated_size_set(arena_chunk_t *chunk, size_t pageind,
  409. size_t size);
  410. void arena_mapbits_large_set(arena_chunk_t *chunk, size_t pageind,
  411. size_t size, size_t flags);
  412. void arena_mapbits_large_binind_set(arena_chunk_t *chunk, size_t pageind,
  413. size_t binind);
  414. void arena_mapbits_small_set(arena_chunk_t *chunk, size_t pageind,
  415. size_t runind, size_t binind, size_t flags);
  416. void arena_mapbits_unzeroed_set(arena_chunk_t *chunk, size_t pageind,
  417. size_t unzeroed);
  418. size_t arena_ptr_small_binind_get(const void *ptr, size_t mapbits);
  419. size_t arena_bin_index(arena_t *arena, arena_bin_t *bin);
  420. unsigned arena_run_regind(arena_run_t *run, arena_bin_info_t *bin_info,
  421. const void *ptr);
  422. prof_ctx_t *arena_prof_ctx_get(const void *ptr);
  423. void arena_prof_ctx_set(const void *ptr, prof_ctx_t *ctx);
  424. void *arena_malloc(arena_t *arena, size_t size, bool zero, bool try_tcache);
  425. size_t arena_salloc(const void *ptr, bool demote);
  426. void arena_dalloc(arena_t *arena, arena_chunk_t *chunk, void *ptr,
  427. bool try_tcache);
  428. #endif
  429. #if (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_ARENA_C_))
  430. # ifdef JEMALLOC_ARENA_INLINE_A
  431. JEMALLOC_INLINE arena_chunk_map_t *
  432. arena_mapp_get(arena_chunk_t *chunk, size_t pageind)
  433. {
  434. assert(pageind >= map_bias);
  435. assert(pageind < chunk_npages);
  436. return (&chunk->map[pageind-map_bias]);
  437. }
  438. JEMALLOC_INLINE size_t *
  439. arena_mapbitsp_get(arena_chunk_t *chunk, size_t pageind)
  440. {
  441. return (&arena_mapp_get(chunk, pageind)->bits);
  442. }
  443. JEMALLOC_INLINE size_t
  444. arena_mapbits_get(arena_chunk_t *chunk, size_t pageind)
  445. {
  446. return (*arena_mapbitsp_get(chunk, pageind));
  447. }
  448. JEMALLOC_INLINE size_t
  449. arena_mapbits_unallocated_size_get(arena_chunk_t *chunk, size_t pageind)
  450. {
  451. size_t mapbits;
  452. mapbits = arena_mapbits_get(chunk, pageind);
  453. assert((mapbits & (CHUNK_MAP_LARGE|CHUNK_MAP_ALLOCATED)) == 0);
  454. return (mapbits & ~PAGE_MASK);
  455. }
  456. JEMALLOC_INLINE size_t
  457. arena_mapbits_large_size_get(arena_chunk_t *chunk, size_t pageind)
  458. {
  459. size_t mapbits;
  460. mapbits = arena_mapbits_get(chunk, pageind);
  461. assert((mapbits & (CHUNK_MAP_LARGE|CHUNK_MAP_ALLOCATED)) ==
  462. (CHUNK_MAP_LARGE|CHUNK_MAP_ALLOCATED));
  463. return (mapbits & ~PAGE_MASK);
  464. }
  465. JEMALLOC_INLINE size_t
  466. arena_mapbits_small_runind_get(arena_chunk_t *chunk, size_t pageind)
  467. {
  468. size_t mapbits;
  469. mapbits = arena_mapbits_get(chunk, pageind);
  470. assert((mapbits & (CHUNK_MAP_LARGE|CHUNK_MAP_ALLOCATED)) ==
  471. CHUNK_MAP_ALLOCATED);
  472. return (mapbits >> LG_PAGE);
  473. }
  474. JEMALLOC_INLINE size_t
  475. arena_mapbits_binind_get(arena_chunk_t *chunk, size_t pageind)
  476. {
  477. size_t mapbits;
  478. size_t binind;
  479. mapbits = arena_mapbits_get(chunk, pageind);
  480. binind = (mapbits & CHUNK_MAP_BININD_MASK) >> CHUNK_MAP_BININD_SHIFT;
  481. assert(binind < NBINS || binind == BININD_INVALID);
  482. return (binind);
  483. }
  484. JEMALLOC_INLINE size_t
  485. arena_mapbits_dirty_get(arena_chunk_t *chunk, size_t pageind)
  486. {
  487. size_t mapbits;
  488. mapbits = arena_mapbits_get(chunk, pageind);
  489. return (mapbits & CHUNK_MAP_DIRTY);
  490. }
  491. JEMALLOC_INLINE size_t
  492. arena_mapbits_unzeroed_get(arena_chunk_t *chunk, size_t pageind)
  493. {
  494. size_t mapbits;
  495. mapbits = arena_mapbits_get(chunk, pageind);
  496. return (mapbits & CHUNK_MAP_UNZEROED);
  497. }
  498. JEMALLOC_INLINE size_t
  499. arena_mapbits_large_get(arena_chunk_t *chunk, size_t pageind)
  500. {
  501. size_t mapbits;
  502. mapbits = arena_mapbits_get(chunk, pageind);
  503. return (mapbits & CHUNK_MAP_LARGE);
  504. }
  505. JEMALLOC_INLINE size_t
  506. arena_mapbits_allocated_get(arena_chunk_t *chunk, size_t pageind)
  507. {
  508. size_t mapbits;
  509. mapbits = arena_mapbits_get(chunk, pageind);
  510. return (mapbits & CHUNK_MAP_ALLOCATED);
  511. }
  512. JEMALLOC_INLINE void
  513. arena_mapbits_unallocated_set(arena_chunk_t *chunk, size_t pageind, size_t size,
  514. size_t flags)
  515. {
  516. size_t *mapbitsp;
  517. mapbitsp = arena_mapbitsp_get(chunk, pageind);
  518. assert((size & PAGE_MASK) == 0);
  519. assert((flags & ~CHUNK_MAP_FLAGS_MASK) == 0);
  520. assert((flags & (CHUNK_MAP_DIRTY|CHUNK_MAP_UNZEROED)) == flags);
  521. *mapbitsp = size | CHUNK_MAP_BININD_INVALID | flags;
  522. }
  523. JEMALLOC_INLINE void
  524. arena_mapbits_unallocated_size_set(arena_chunk_t *chunk, size_t pageind,
  525. size_t size)
  526. {
  527. size_t *mapbitsp;
  528. mapbitsp = arena_mapbitsp_get(chunk, pageind);
  529. assert((size & PAGE_MASK) == 0);
  530. assert((*mapbitsp & (CHUNK_MAP_LARGE|CHUNK_MAP_ALLOCATED)) == 0);
  531. *mapbitsp = size | (*mapbitsp & PAGE_MASK);
  532. }
  533. JEMALLOC_INLINE void
  534. arena_mapbits_large_set(arena_chunk_t *chunk, size_t pageind, size_t size,
  535. size_t flags)
  536. {
  537. size_t *mapbitsp;
  538. size_t unzeroed;
  539. mapbitsp = arena_mapbitsp_get(chunk, pageind);
  540. assert((size & PAGE_MASK) == 0);
  541. assert((flags & CHUNK_MAP_DIRTY) == flags);
  542. unzeroed = *mapbitsp & CHUNK_MAP_UNZEROED; /* Preserve unzeroed. */
  543. *mapbitsp = size | CHUNK_MAP_BININD_INVALID | flags | unzeroed |
  544. CHUNK_MAP_LARGE | CHUNK_MAP_ALLOCATED;
  545. }
  546. JEMALLOC_INLINE void
  547. arena_mapbits_large_binind_set(arena_chunk_t *chunk, size_t pageind,
  548. size_t binind)
  549. {
  550. size_t *mapbitsp;
  551. assert(binind <= BININD_INVALID);
  552. mapbitsp = arena_mapbitsp_get(chunk, pageind);
  553. assert(arena_mapbits_large_size_get(chunk, pageind) == PAGE);
  554. *mapbitsp = (*mapbitsp & ~CHUNK_MAP_BININD_MASK) | (binind <<
  555. CHUNK_MAP_BININD_SHIFT);
  556. }
  557. JEMALLOC_INLINE void
  558. arena_mapbits_small_set(arena_chunk_t *chunk, size_t pageind, size_t runind,
  559. size_t binind, size_t flags)
  560. {
  561. size_t *mapbitsp;
  562. size_t unzeroed;
  563. assert(binind < BININD_INVALID);
  564. mapbitsp = arena_mapbitsp_get(chunk, pageind);
  565. assert(pageind - runind >= map_bias);
  566. assert((flags & CHUNK_MAP_DIRTY) == flags);
  567. unzeroed = *mapbitsp & CHUNK_MAP_UNZEROED; /* Preserve unzeroed. */
  568. *mapbitsp = (runind << LG_PAGE) | (binind << CHUNK_MAP_BININD_SHIFT) |
  569. flags | unzeroed | CHUNK_MAP_ALLOCATED;
  570. }
  571. JEMALLOC_INLINE void
  572. arena_mapbits_unzeroed_set(arena_chunk_t *chunk, size_t pageind,
  573. size_t unzeroed)
  574. {
  575. size_t *mapbitsp;
  576. mapbitsp = arena_mapbitsp_get(chunk, pageind);
  577. *mapbitsp = (*mapbitsp & ~CHUNK_MAP_UNZEROED) | unzeroed;
  578. }
  579. JEMALLOC_INLINE size_t
  580. arena_ptr_small_binind_get(const void *ptr, size_t mapbits)
  581. {
  582. size_t binind;
  583. binind = (mapbits & CHUNK_MAP_BININD_MASK) >> CHUNK_MAP_BININD_SHIFT;
  584. if (config_debug) {
  585. arena_chunk_t *chunk;
  586. arena_t *arena;
  587. size_t pageind;
  588. size_t actual_mapbits;
  589. arena_run_t *run;
  590. arena_bin_t *bin;
  591. size_t actual_binind;
  592. arena_bin_info_t *bin_info;
  593. assert(binind != BININD_INVALID);
  594. assert(binind < NBINS);
  595. chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr);
  596. arena = chunk->arena;
  597. pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> LG_PAGE;
  598. actual_mapbits = arena_mapbits_get(chunk, pageind);
  599. assert(mapbits == actual_mapbits);
  600. assert(arena_mapbits_large_get(chunk, pageind) == 0);
  601. assert(arena_mapbits_allocated_get(chunk, pageind) != 0);
  602. run = (arena_run_t *)((uintptr_t)chunk + (uintptr_t)((pageind -
  603. (actual_mapbits >> LG_PAGE)) << LG_PAGE));
  604. bin = run->bin;
  605. actual_binind = bin - arena->bins;
  606. assert(binind == actual_binind);
  607. bin_info = &arena_bin_info[actual_binind];
  608. assert(((uintptr_t)ptr - ((uintptr_t)run +
  609. (uintptr_t)bin_info->reg0_offset)) % bin_info->reg_interval
  610. == 0);
  611. }
  612. return (binind);
  613. }
  614. # endif /* JEMALLOC_ARENA_INLINE_A */
  615. # ifdef JEMALLOC_ARENA_INLINE_B
  616. JEMALLOC_INLINE size_t
  617. arena_bin_index(arena_t *arena, arena_bin_t *bin)
  618. {
  619. size_t binind = bin - arena->bins;
  620. assert(binind < NBINS);
  621. return (binind);
  622. }
  623. JEMALLOC_INLINE unsigned
  624. arena_run_regind(arena_run_t *run, arena_bin_info_t *bin_info, const void *ptr)
  625. {
  626. unsigned shift, diff, regind;
  627. size_t interval;
  628. /*
  629. * Freeing a pointer lower than region zero can cause assertion
  630. * failure.
  631. */
  632. assert((uintptr_t)ptr >= (uintptr_t)run +
  633. (uintptr_t)bin_info->reg0_offset);
  634. /*
  635. * Avoid doing division with a variable divisor if possible. Using
  636. * actual division here can reduce allocator throughput by over 20%!
  637. */
  638. diff = (unsigned)((uintptr_t)ptr - (uintptr_t)run -
  639. bin_info->reg0_offset);
  640. /* Rescale (factor powers of 2 out of the numerator and denominator). */
  641. interval = bin_info->reg_interval;
  642. shift = ffs(interval) - 1;
  643. diff >>= shift;
  644. interval >>= shift;
  645. if (interval == 1) {
  646. /* The divisor was a power of 2. */
  647. regind = diff;
  648. } else {
  649. /*
  650. * To divide by a number D that is not a power of two we
  651. * multiply by (2^21 / D) and then right shift by 21 positions.
  652. *
  653. * X / D
  654. *
  655. * becomes
  656. *
  657. * (X * interval_invs[D - 3]) >> SIZE_INV_SHIFT
  658. *
  659. * We can omit the first three elements, because we never
  660. * divide by 0, and 1 and 2 are both powers of two, which are
  661. * handled above.
  662. */
  663. #define SIZE_INV_SHIFT ((sizeof(unsigned) << 3) - LG_RUN_MAXREGS)
  664. #define SIZE_INV(s) (((1U << SIZE_INV_SHIFT) / (s)) + 1)
  665. static const unsigned interval_invs[] = {
  666. SIZE_INV(3),
  667. SIZE_INV(4), SIZE_INV(5), SIZE_INV(6), SIZE_INV(7),
  668. SIZE_INV(8), SIZE_INV(9), SIZE_INV(10), SIZE_INV(11),
  669. SIZE_INV(12), SIZE_INV(13), SIZE_INV(14), SIZE_INV(15),
  670. SIZE_INV(16), SIZE_INV(17), SIZE_INV(18), SIZE_INV(19),
  671. SIZE_INV(20), SIZE_INV(21), SIZE_INV(22), SIZE_INV(23),
  672. SIZE_INV(24), SIZE_INV(25), SIZE_INV(26), SIZE_INV(27),
  673. SIZE_INV(28), SIZE_INV(29), SIZE_INV(30), SIZE_INV(31)
  674. };
  675. if (interval <= ((sizeof(interval_invs) / sizeof(unsigned)) +
  676. 2)) {
  677. regind = (diff * interval_invs[interval - 3]) >>
  678. SIZE_INV_SHIFT;
  679. } else
  680. regind = diff / interval;
  681. #undef SIZE_INV
  682. #undef SIZE_INV_SHIFT
  683. }
  684. assert(diff == regind * interval);
  685. assert(regind < bin_info->nregs);
  686. return (regind);
  687. }
  688. JEMALLOC_INLINE prof_ctx_t *
  689. arena_prof_ctx_get(const void *ptr)
  690. {
  691. prof_ctx_t *ret;
  692. arena_chunk_t *chunk;
  693. size_t pageind, mapbits;
  694. cassert(config_prof);
  695. assert(ptr != NULL);
  696. assert(CHUNK_ADDR2BASE(ptr) != ptr);
  697. chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr);
  698. pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> LG_PAGE;
  699. mapbits = arena_mapbits_get(chunk, pageind);
  700. assert((mapbits & CHUNK_MAP_ALLOCATED) != 0);
  701. if ((mapbits & CHUNK_MAP_LARGE) == 0) {
  702. if (prof_promote)
  703. ret = (prof_ctx_t *)(uintptr_t)1U;
  704. else {
  705. arena_run_t *run = (arena_run_t *)((uintptr_t)chunk +
  706. (uintptr_t)((pageind - (mapbits >> LG_PAGE)) <<
  707. LG_PAGE));
  708. size_t binind = arena_ptr_small_binind_get(ptr,
  709. mapbits);
  710. arena_bin_info_t *bin_info = &arena_bin_info[binind];
  711. unsigned regind;
  712. regind = arena_run_regind(run, bin_info, ptr);
  713. ret = *(prof_ctx_t **)((uintptr_t)run +
  714. bin_info->ctx0_offset + (regind *
  715. sizeof(prof_ctx_t *)));
  716. }
  717. } else
  718. ret = arena_mapp_get(chunk, pageind)->prof_ctx;
  719. return (ret);
  720. }
  721. JEMALLOC_INLINE void
  722. arena_prof_ctx_set(const void *ptr, prof_ctx_t *ctx)
  723. {
  724. arena_chunk_t *chunk;
  725. size_t pageind, mapbits;
  726. cassert(config_prof);
  727. assert(ptr != NULL);
  728. assert(CHUNK_ADDR2BASE(ptr) != ptr);
  729. chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr);
  730. pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> LG_PAGE;
  731. mapbits = arena_mapbits_get(chunk, pageind);
  732. assert((mapbits & CHUNK_MAP_ALLOCATED) != 0);
  733. if ((mapbits & CHUNK_MAP_LARGE) == 0) {
  734. if (prof_promote == false) {
  735. arena_run_t *run = (arena_run_t *)((uintptr_t)chunk +
  736. (uintptr_t)((pageind - (mapbits >> LG_PAGE)) <<
  737. LG_PAGE));
  738. size_t binind;
  739. arena_bin_info_t *bin_info;
  740. unsigned regind;
  741. binind = arena_ptr_small_binind_get(ptr, mapbits);
  742. bin_info = &arena_bin_info[binind];
  743. regind = arena_run_regind(run, bin_info, ptr);
  744. *((prof_ctx_t **)((uintptr_t)run + bin_info->ctx0_offset
  745. + (regind * sizeof(prof_ctx_t *)))) = ctx;
  746. } else
  747. assert((uintptr_t)ctx == (uintptr_t)1U);
  748. } else
  749. arena_mapp_get(chunk, pageind)->prof_ctx = ctx;
  750. }
  751. JEMALLOC_INLINE void *
  752. arena_malloc(arena_t *arena, size_t size, bool zero, bool try_tcache)
  753. {
  754. tcache_t *tcache;
  755. assert(size != 0);
  756. assert(size <= arena_maxclass);
  757. if (size <= SMALL_MAXCLASS) {
  758. if (try_tcache && (tcache = tcache_get(true)) != NULL)
  759. return (tcache_alloc_small(tcache, size, zero));
  760. else {
  761. return (arena_malloc_small(choose_arena(arena), size,
  762. zero));
  763. }
  764. } else {
  765. /*
  766. * Initialize tcache after checking size in order to avoid
  767. * infinite recursion during tcache initialization.
  768. */
  769. if (try_tcache && size <= tcache_maxclass && (tcache =
  770. tcache_get(true)) != NULL)
  771. return (tcache_alloc_large(tcache, size, zero));
  772. else {
  773. return (arena_malloc_large(choose_arena(arena), size,
  774. zero));
  775. }
  776. }
  777. }
  778. /* Return the size of the allocation pointed to by ptr. */
  779. JEMALLOC_INLINE size_t
  780. arena_salloc(const void *ptr, bool demote)
  781. {
  782. size_t ret;
  783. arena_chunk_t *chunk;
  784. size_t pageind, binind;
  785. assert(ptr != NULL);
  786. assert(CHUNK_ADDR2BASE(ptr) != ptr);
  787. chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr);
  788. pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> LG_PAGE;
  789. assert(arena_mapbits_allocated_get(chunk, pageind) != 0);
  790. binind = arena_mapbits_binind_get(chunk, pageind);
  791. if (binind == BININD_INVALID || (config_prof && demote == false &&
  792. prof_promote && arena_mapbits_large_get(chunk, pageind) != 0)) {
  793. /*
  794. * Large allocation. In the common case (demote == true), and
  795. * as this is an inline function, most callers will only end up
  796. * looking at binind to determine that ptr is a small
  797. * allocation.
  798. */
  799. assert(((uintptr_t)ptr & PAGE_MASK) == 0);
  800. ret = arena_mapbits_large_size_get(chunk, pageind);
  801. assert(ret != 0);
  802. assert(pageind + (ret>>LG_PAGE) <= chunk_npages);
  803. assert(ret == PAGE || arena_mapbits_large_size_get(chunk,
  804. pageind+(ret>>LG_PAGE)-1) == 0);
  805. assert(binind == arena_mapbits_binind_get(chunk,
  806. pageind+(ret>>LG_PAGE)-1));
  807. assert(arena_mapbits_dirty_get(chunk, pageind) ==
  808. arena_mapbits_dirty_get(chunk, pageind+(ret>>LG_PAGE)-1));
  809. } else {
  810. /*
  811. * Small allocation (possibly promoted to a large object due to
  812. * prof_promote).
  813. */
  814. assert(arena_mapbits_large_get(chunk, pageind) != 0 ||
  815. arena_ptr_small_binind_get(ptr, arena_mapbits_get(chunk,
  816. pageind)) == binind);
  817. ret = arena_bin_info[binind].reg_size;
  818. }
  819. return (ret);
  820. }
  821. JEMALLOC_INLINE void
  822. arena_dalloc(arena_t *arena, arena_chunk_t *chunk, void *ptr, bool try_tcache)
  823. {
  824. size_t pageind, mapbits;
  825. tcache_t *tcache;
  826. assert(arena != NULL);
  827. assert(chunk->arena == arena);
  828. assert(ptr != NULL);
  829. assert(CHUNK_ADDR2BASE(ptr) != ptr);
  830. pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> LG_PAGE;
  831. mapbits = arena_mapbits_get(chunk, pageind);
  832. assert(arena_mapbits_allocated_get(chunk, pageind) != 0);
  833. if ((mapbits & CHUNK_MAP_LARGE) == 0) {
  834. /* Small allocation. */
  835. if (try_tcache && (tcache = tcache_get(false)) != NULL) {
  836. size_t binind;
  837. binind = arena_ptr_small_binind_get(ptr, mapbits);
  838. tcache_dalloc_small(tcache, ptr, binind);
  839. } else
  840. arena_dalloc_small(arena, chunk, ptr, pageind);
  841. } else {
  842. size_t size = arena_mapbits_large_size_get(chunk, pageind);
  843. assert(((uintptr_t)ptr & PAGE_MASK) == 0);
  844. if (try_tcache && size <= tcache_maxclass && (tcache =
  845. tcache_get(false)) != NULL) {
  846. tcache_dalloc_large(tcache, ptr, size);
  847. } else
  848. arena_dalloc_large(arena, chunk, ptr);
  849. }
  850. }
  851. # endif /* JEMALLOC_ARENA_INLINE_B */
  852. #endif
  853. #endif /* JEMALLOC_H_INLINES */
  854. /******************************************************************************/