/erts/emulator/beam/erl_alloc_util.c
C | 8251 lines | 6391 code | 1444 blank | 416 comment | 1244 complexity | b9fbce4ac17882801f6d4ee963dfc2ea MD5 | raw file
Possible License(s): BSD-3-Clause, LGPL-2.1, MPL-2.0-no-copyleft-exception, Apache-2.0
Large files files are truncated, but you can click here to view the full file
- /*
- * %CopyrightBegin%
- *
- * Copyright Ericsson AB 2002-2018. All Rights Reserved.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- * %CopyrightEnd%
- */
- /*
- * Description: A memory allocator utility. This utility provides
- * management of (multiple) memory segments, coalescing
- * of free blocks, etc. Allocators are implemented by
- * implementing a callback-interface which is called by
- * this utility. The only task the callback-module has to
- * perform is to supervise the free blocks.
- *
- * Author: Rickard Green
- */
- /*
- * Alloc util will enforce 8 byte alignment if sys_alloc and mseg_alloc at
- * least enforces 8 byte alignment. If sys_alloc only enforces 4 byte
- * alignment then alloc util will do so too.
- */
- #ifdef HAVE_CONFIG_H
- # include "config.h"
- #endif
- #include "global.h"
- #include "big.h"
- #include "erl_mmap.h"
- #include "erl_mtrace.h"
- #define GET_ERL_ALLOC_UTIL_IMPL
- #include "erl_alloc_util.h"
- #include "erl_mseg.h"
- #include "erl_threads.h"
- #include "erl_thr_progress.h"
- #include "erl_bif_unique.h"
- #include "erl_nif.h"
- #ifdef ERTS_ENABLE_LOCK_COUNT
- #include "erl_lock_count.h"
- #endif
- #include "lttng-wrapper.h"
- #if defined(ERTS_ALLOC_UTIL_HARD_DEBUG) && defined(__GNUC__)
- #warning "* * * * * * * * * *"
- #warning "* * * * * * * * * *"
- #warning "* * NOTE: * *"
- #warning "* * Hard debug * *"
- #warning "* * is enabled! * *"
- #warning "* * * * * * * * * *"
- #warning "* * * * * * * * * *"
- #endif
- #define ERTS_ALCU_DD_OPS_LIM_HIGH 20
- #define ERTS_ALCU_DD_OPS_LIM_LOW 2
- /* Fix alloc limit */
- #define ERTS_ALCU_FIX_MAX_LIST_SZ 1000
- #define ERTS_ALC_FIX_MAX_SHRINK_OPS 30
- #define ALLOC_ZERO_EQ_NULL 0
- #ifndef ERTS_MSEG_FLG_2POW
- # define ERTS_MSEG_FLG_2POW 0
- #endif
- #ifndef ERTS_MSEG_FLG_NONE
- # define ERTS_MSEG_FLG_NONE 0
- #endif
- static int atoms_initialized = 0;
- static int initialized = 0;
- #define INV_SYS_ALLOC_CARRIER_MASK ((UWord) (sys_alloc_carrier_size - 1))
- #define SYS_ALLOC_CARRIER_MASK (~INV_SYS_ALLOC_CARRIER_MASK)
- #define SYS_ALLOC_CARRIER_FLOOR(X) ((X) & SYS_ALLOC_CARRIER_MASK)
- #define SYS_ALLOC_CARRIER_CEILING(X) \
- SYS_ALLOC_CARRIER_FLOOR((X) + INV_SYS_ALLOC_CARRIER_MASK)
- #define SYS_PAGE_SIZE (sys_page_size)
- #define SYS_PAGE_SZ_MASK ((UWord)(SYS_PAGE_SIZE - 1))
- #if 0
- /* Can be useful for debugging */
- #define MBC_REALLOC_ALWAYS_MOVES
- #endif
- /* alloc_util global parameters */
- static Uint sys_alloc_carrier_size;
- static Uint sys_page_size;
- #if HAVE_ERTS_MSEG
- static Uint max_mseg_carriers;
- #endif
- static int allow_sys_alloc_carriers;
- #define ONE_GIGA (1000000000)
- #define ERTS_ALC_CC_GIGA_VAL(CC) ((CC) / ONE_GIGA)
- #define ERTS_ALC_CC_VAL(CC) ((CC) % ONE_GIGA)
- #define INC_CC(CC) ((CC)++)
- #define DEC_CC(CC) ((CC)--)
- /* Multi block carrier (MBC) memory layout in OTP 22:
- Empty MBC:
- [Carrier_t|pad|Block_t L0T0|fhdr| free... ]
- MBC after allocating first block:
- [Carrier_t|pad|Block_t 0000| udata |pad|Block_t L0T0|fhdr| free... ]
- MBC after allocating second block:
- [Carrier_t|pad|Block_t 0000| udata |pad|Block_t 0000| udata |pad|Block_t L0T0|fhdr| free... ]
- MBC after deallocating first block:
- [Carrier_t|pad|Block_t 00T0|fhdr| free |FreeBlkFtr_t|Block_t 0P00| udata |pad|Block_t L0T0|fhdr| free... ]
- MBC after allocating first block, with allocation tagging enabled:
- [Carrier_t|pad|Block_t 000A| udata |atag|pad|Block_t L0T0|fhdr| free... ]
- udata = Allocated user data
- atag = A tag with basic metadata about this allocation
- pad = Padding to ensure correct alignment for user data
- fhdr = Allocator specific header to keep track of free block
- free = Unused free memory
- T = This block is free (THIS_FREE_BLK_HDR_FLG)
- P = Previous block is free (PREV_FREE_BLK_HDR_FLG)
- L = Last block in carrier (LAST_BLK_HDR_FLG)
- A = Block has an allocation tag footer, only valid for allocated blocks
- (ATAG_BLK_HDR_FLG)
- */
- /* Single block carrier (SBC):
- [Carrier_t|pad|Block_t 1110| udata... ]
- [Carrier_t|pad|Block_t 111A| udata | atag]
- */
- /* Allocation tags ...
- *
- * These are added to the footer of every block when enabled. Currently they
- * consist of the allocation type and an atom identifying the allocating
- * driver/nif (or 'system' if that can't be determined), but the format is not
- * supposed to be set in stone.
- *
- * The packing scheme requires that the atom values are small enough to fit
- * into a word with ERTS_ALC_N_BITS to spare. Users must check for overflow
- * before MAKE_ATAG(). */
- typedef UWord alcu_atag_t;
- #define MAKE_ATAG(IdAtom, TypeNum) \
- (ASSERT((TypeNum) >= ERTS_ALC_N_MIN && (TypeNum) <= ERTS_ALC_N_MAX), \
- ASSERT(atom_val(IdAtom) <= MAX_ATAG_ATOM_ID), \
- (atom_val(IdAtom) << ERTS_ALC_N_BITS) | (TypeNum))
- #define ATAG_ID(AT) (make_atom((AT) >> ERTS_ALC_N_BITS))
- #define ATAG_TYPE(AT) ((AT) & ERTS_ALC_N_MASK)
- #define MAX_ATAG_ATOM_ID (ERTS_UWORD_MAX >> ERTS_ALC_N_BITS)
- #define DBG_IS_VALID_ATAG(AT) \
- (ATAG_TYPE(AT) >= ERTS_ALC_N_MIN && \
- ATAG_TYPE(AT) <= ERTS_ALC_N_MAX && \
- ATAG_ID(AT) <= MAX_ATAG_ATOM_ID)
- /* Blocks ... */
- #define UNUSED0_BLK_FTR_FLG (((UWord) 1) << 0)
- #define UNUSED1_BLK_FTR_FLG (((UWord) 1) << 1)
- #define UNUSED2_BLK_FTR_FLG (((UWord) 1) << 2)
- #if MBC_ABLK_OFFSET_BITS
- # define ABLK_HDR_SZ (offsetof(Block_t,u))
- #else
- # define ABLK_HDR_SZ (sizeof(Block_t))
- #endif
- #define FBLK_FTR_SZ (sizeof(FreeBlkFtr_t))
- #define BLK_HAS_ATAG(B) \
- (!!((B)->bhdr & ATAG_BLK_HDR_FLG))
- #define GET_BLK_ATAG(B) \
- (ASSERT(BLK_HAS_ATAG(B)), \
- ((alcu_atag_t *) (((char *) (B)) + (BLK_SZ(B))))[-1])
- #define SET_BLK_ATAG(B, T) \
- ((B)->bhdr |= ATAG_BLK_HDR_FLG, \
- ((alcu_atag_t *) (((char *) (B)) + (BLK_SZ(B))))[-1] = (T))
- #define BLK_ATAG_SZ(AP) ((AP)->atags ? sizeof(alcu_atag_t) : 0)
- #define UMEMSZ2BLKSZ(AP, SZ) \
- (ABLK_HDR_SZ + BLK_ATAG_SZ(AP) + (SZ) <= (AP)->min_block_size \
- ? (AP)->min_block_size \
- : UNIT_CEILING(ABLK_HDR_SZ + BLK_ATAG_SZ(AP) + (SZ)))
- #define UMEM2BLK(P) ((Block_t *) (((char *) (P)) - ABLK_HDR_SZ))
- #define BLK2UMEM(P) ((void *) (((char *) (P)) + ABLK_HDR_SZ))
- #define PREV_BLK_SZ(B) ((UWord) (((FreeBlkFtr_t *)(B))[-1]))
- #define SET_BLK_SZ_FTR(B, SZ) \
- (((FreeBlkFtr_t *) (((char *) (B)) + (SZ)))[-1] = (SZ))
- #define SET_MBC_ABLK_SZ(B, SZ) \
- (ASSERT(((SZ) & BLK_FLG_MASK) == 0), \
- (B)->bhdr = (((B)->bhdr) & ~MBC_ABLK_SZ_MASK) | (SZ))
- #define SET_MBC_FBLK_SZ(B, SZ) \
- (ASSERT(((SZ) & BLK_FLG_MASK) == 0), \
- (B)->bhdr = (((B)->bhdr) & ~MBC_FBLK_SZ_MASK) | (SZ))
- #define SET_SBC_BLK_SZ(B, SZ) \
- (ASSERT(((SZ) & BLK_FLG_MASK) == 0), \
- (B)->bhdr = (((B)->bhdr) & ~SBC_BLK_SZ_MASK) | (SZ))
- #define SET_PREV_BLK_FREE(AP,B) \
- (ASSERT(!IS_MBC_FIRST_BLK(AP,B)), \
- ASSERT(!IS_FREE_BLK(B)), \
- (B)->bhdr |= PREV_FREE_BLK_HDR_FLG)
- #define SET_PREV_BLK_ALLOCED(B) \
- ((B)->bhdr &= ~PREV_FREE_BLK_HDR_FLG)
- #define SET_LAST_BLK(B) \
- ((B)->bhdr |= LAST_BLK_HDR_FLG)
- #define SET_NOT_LAST_BLK(B) \
- ((B)->bhdr &= ~LAST_BLK_HDR_FLG)
- #define SBH_THIS_FREE THIS_FREE_BLK_HDR_FLG
- #define SBH_PREV_FREE PREV_FREE_BLK_HDR_FLG
- #define SBH_LAST_BLK LAST_BLK_HDR_FLG
- #if MBC_ABLK_OFFSET_BITS
- # define MBC_SZ_MAX_LIMIT ((((UWord)1 << MBC_ABLK_OFFSET_BITS) - 1) << ERTS_SUPER_ALIGN_BITS)
- # define BLK_CARRIER_OFFSET(B, C) (((char*)(B) - (char*)(C)) >> ERTS_SACRR_UNIT_SHIFT)
- # define SET_MBC_ABLK_HDR(B, Sz, F, C) \
- (ASSERT(((Sz) & ~MBC_ABLK_SZ_MASK) == 0), \
- ASSERT(!((UWord)(F) & (~BLK_FLG_MASK|THIS_FREE_BLK_HDR_FLG))), \
- (B)->bhdr = ((Sz) | (F) | (BLK_CARRIER_OFFSET(B,C) << MBC_ABLK_OFFSET_SHIFT)))
- # define SET_MBC_FBLK_HDR(B, Sz, F, C) \
- (ASSERT(((Sz) & ~MBC_FBLK_SZ_MASK) == 0), \
- ASSERT(((UWord)(F) & (~BLK_FLG_MASK|THIS_FREE_BLK_HDR_FLG|PREV_FREE_BLK_HDR_FLG)) == THIS_FREE_BLK_HDR_FLG), \
- (B)->bhdr = ((Sz) | (F)), \
- (B)->u.carrier = (C))
- # define IS_MBC_FIRST_ABLK(AP,B) \
- ((((UWord)(B) & ~ERTS_SACRR_UNIT_MASK) == MBC_HEADER_SIZE(AP)) \
- && ((B)->bhdr & MBC_ABLK_OFFSET_MASK) == 0)
- # define IS_MBC_FIRST_FBLK(AP,B) \
- ((char*)(B) == (char*)((B)->u.carrier) + MBC_HEADER_SIZE(AP))
- # define IS_MBC_FIRST_BLK(AP,B) \
- (IS_FREE_BLK(B) ? IS_MBC_FIRST_FBLK(AP,B) : IS_MBC_FIRST_ABLK(AP,B))
- # define SET_BLK_FREE(B) \
- (ASSERT(!IS_PREV_BLK_FREE(B)), \
- (B)->u.carrier = ABLK_TO_MBC(B), \
- (B)->bhdr &= (MBC_ABLK_SZ_MASK|LAST_BLK_HDR_FLG), \
- (B)->bhdr |= THIS_FREE_BLK_HDR_FLG)
- # define SET_BLK_ALLOCED(B) \
- (ASSERT(((B)->bhdr & (MBC_ABLK_OFFSET_MASK|THIS_FREE_BLK_HDR_FLG)) == THIS_FREE_BLK_HDR_FLG), \
- (B)->bhdr &= ~THIS_FREE_BLK_HDR_FLG, \
- (B)->bhdr |= (BLK_CARRIER_OFFSET(B,(B)->u.carrier) << MBC_ABLK_OFFSET_SHIFT))
- #else /* !MBC_ABLK_OFFSET_BITS */
- # define MBC_SZ_MAX_LIMIT ((UWord)~0)
- # define SET_MBC_ABLK_HDR(B, Sz, F, C) \
- (ASSERT(((Sz) & BLK_FLG_MASK) == 0), \
- ASSERT(((F) & ~BLK_FLG_MASK) == 0), \
- ASSERT(!((UWord)(F) & (~BLK_FLG_MASK|THIS_FREE_BLK_HDR_FLG))), \
- (B)->bhdr = ((Sz) | (F)), \
- (B)->carrier = (C))
- # define SET_MBC_FBLK_HDR(B, Sz, F, C) \
- (ASSERT(((Sz) & BLK_FLG_MASK) == 0), \
- ASSERT(((F) & ~BLK_FLG_MASK) == 0), \
- ASSERT(((UWord)(F) & (~BLK_FLG_MASK|THIS_FREE_BLK_HDR_FLG|PREV_FREE_BLK_HDR_FLG)) == THIS_FREE_BLK_HDR_FLG), \
- (B)->bhdr = ((Sz) | (F)), \
- (B)->carrier = (C))
- # define IS_MBC_FIRST_BLK(AP,B) \
- ((char*)(B) == (char*)((B)->carrier) + MBC_HEADER_SIZE(AP))
- # define IS_MBC_FIRST_ABLK(AP,B) IS_MBC_FIRST_BLK(AP,B)
- # define IS_MBC_FIRST_FBLK(AP,B) IS_MBC_FIRST_BLK(AP,B)
- # define SET_BLK_FREE(B) \
- (ASSERT(!IS_PREV_BLK_FREE(B)), \
- (B)->bhdr |= THIS_FREE_BLK_HDR_FLG)
- # define SET_BLK_ALLOCED(B) \
- ((B)->bhdr &= ~THIS_FREE_BLK_HDR_FLG)
- #endif /* !MBC_ABLK_OFFSET_BITS */
- #define SET_SBC_BLK_HDR(B, Sz) \
- (ASSERT(((Sz) & BLK_FLG_MASK) == 0), (B)->bhdr = ((Sz) | (SBC_BLK_HDR_FLG)))
- #define BLK_UMEM_SZ(B) \
- (BLK_SZ(B) - (ABLK_HDR_SZ))
- #define IS_PREV_BLK_FREE(B) \
- ((B)->bhdr & PREV_FREE_BLK_HDR_FLG)
- #define IS_PREV_BLK_ALLOCED(B) \
- (!IS_PREV_BLK_FREE((B)))
- #define IS_ALLOCED_BLK(B) \
- (!IS_FREE_BLK((B)))
- #define IS_LAST_BLK(B) \
- ((B)->bhdr & LAST_BLK_HDR_FLG)
- #define IS_NOT_LAST_BLK(B) \
- (!IS_LAST_BLK((B)))
- #define GET_LAST_BLK_HDR_FLG(B) \
- ((B)->bhdr & LAST_BLK_HDR_FLG)
- #define GET_THIS_FREE_BLK_HDR_FLG(B) \
- ((B)->bhdr & THIS_FREE_BLK_HDR_FLG)
- #define GET_PREV_FREE_BLK_HDR_FLG(B) \
- ((B)->bhdr & PREV_FREE_BLK_HDR_FLG)
- #define GET_BLK_HDR_FLGS(B) \
- ((B)->bhdr & BLK_FLG_MASK)
- #define NXT_BLK(B) \
- (ASSERT(IS_MBC_BLK(B)), \
- (Block_t *) (((char *) (B)) + MBC_BLK_SZ((B))))
- #define PREV_BLK(B) \
- ((Block_t *) (((char *) (B)) - PREV_BLK_SZ((B))))
- #define BLK_AFTER(B,Sz) \
- ((Block_t *) (((char *) (B)) + (Sz)))
- #define BLK_SZ(B) ((B)->bhdr & (((B)->bhdr & THIS_FREE_BLK_HDR_FLG) ? MBC_FBLK_SZ_MASK : MBC_ABLK_SZ_MASK))
- /* Carriers ... */
- /* #define ERTS_ALC_CPOOL_DEBUG */
- #if defined(DEBUG) && !defined(ERTS_ALC_CPOOL_DEBUG)
- # define ERTS_ALC_CPOOL_DEBUG
- #endif
- #ifdef ERTS_ALC_CPOOL_DEBUG
- # define ERTS_ALC_CPOOL_ASSERT(A) \
- ((void) ((A) \
- ? 1 \
- : (erts_alcu_assert_failed(#A, \
- (char *) __FILE__, \
- __LINE__, \
- (char *) __func__), \
- 0)))
- #else
- # define ERTS_ALC_CPOOL_ASSERT(A) ((void) 1)
- #endif
- #define ERTS_ALC_IS_CPOOL_ENABLED(A) ((A)->cpool.util_limit)
- #define ERTS_ALC_CPOOL_MAX_DISABLE_ABANDON 1000
- #define ERTS_ALC_CPOOL_ALLOC_OP_INC 8
- #define ERTS_ALC_CPOOL_FREE_OP_DEC 10
- #define ERTS_ALC_CPOOL_ALLOC_OP(A) \
- do { \
- if ((A)->cpool.disable_abandon < ERTS_ALC_CPOOL_MAX_DISABLE_ABANDON) { \
- (A)->cpool.disable_abandon += ERTS_ALC_CPOOL_ALLOC_OP_INC; \
- if ((A)->cpool.disable_abandon > ERTS_ALC_CPOOL_MAX_DISABLE_ABANDON) \
- (A)->cpool.disable_abandon = ERTS_ALC_CPOOL_MAX_DISABLE_ABANDON; \
- } \
- } while (0)
- #if ERTS_ALC_CPOOL_ALLOC_OP_INC >= ERTS_ALC_CPOOL_FREE_OP_DEC
- # error "Implementation assume ERTS_ALC_CPOOL_ALLOC_OP_INC < ERTS_ALC_CPOOL_FREE_OP_DEC"
- #endif
- #define ERTS_ALC_CPOOL_REALLOC_OP(A) \
- do { \
- if ((A)->cpool.disable_abandon) { \
- (A)->cpool.disable_abandon -= (ERTS_ALC_CPOOL_FREE_OP_DEC \
- - ERTS_ALC_CPOOL_ALLOC_OP_INC); \
- if ((A)->cpool.disable_abandon < 0) \
- (A)->cpool.disable_abandon = 0; \
- } \
- } while (0)
- #define ERTS_ALC_CPOOL_FREE_OP(A) \
- do { \
- if ((A)->cpool.disable_abandon) { \
- (A)->cpool.disable_abandon -= ERTS_ALC_CPOOL_FREE_OP_DEC; \
- if ((A)->cpool.disable_abandon < 0) \
- (A)->cpool.disable_abandon = 0; \
- } \
- } while (0)
- #define ERTS_CRR_ALCTR_FLG_IN_POOL (((erts_aint_t) 1) << 0)
- #define ERTS_CRR_ALCTR_FLG_BUSY (((erts_aint_t) 1) << 1)
- #define ERTS_CRR_ALCTR_FLG_HOMECOMING (((erts_aint_t) 1) << 2)
- #define ERTS_CRR_ALCTR_FLG_MASK (ERTS_CRR_ALCTR_FLG_IN_POOL | \
- ERTS_CRR_ALCTR_FLG_BUSY | \
- ERTS_CRR_ALCTR_FLG_HOMECOMING)
- #define SBC_HEADER_SIZE \
- (UNIT_CEILING(offsetof(Carrier_t, cpool) \
- + ABLK_HDR_SZ) \
- - ABLK_HDR_SZ)
- #define MBC_HEADER_SIZE(AP) ((AP)->mbc_header_size)
- #define MSEG_CARRIER_HDR_FLAG (((UWord) 1) << 0)
- #define SBC_CARRIER_HDR_FLAG (((UWord) 1) << 1)
- #define SCH_SYS_ALLOC 0
- #define SCH_MSEG MSEG_CARRIER_HDR_FLAG
- #define SCH_MBC 0
- #define SCH_SBC SBC_CARRIER_HDR_FLAG
- #define SET_CARRIER_HDR(C, Sz, F, AP) \
- (ASSERT(((Sz) & CRR_FLG_MASK) == 0), (C)->chdr = ((Sz) | (F)), \
- erts_atomic_init_nob(&(C)->allctr, (erts_aint_t) (AP)))
- #define BLK_TO_SBC(B) \
- ((Carrier_t *) (((char *) (B)) - SBC_HEADER_SIZE))
- #define FIRST_BLK_TO_MBC(AP, B) \
- ((Carrier_t *) (((char *) (B)) - MBC_HEADER_SIZE(AP)))
- #define MBC_TO_FIRST_BLK(AP, P) \
- ((Block_t *) (((char *) (P)) + MBC_HEADER_SIZE(AP)))
- #define SBC2BLK(AP, P) \
- ((Block_t *) (((char *) (P)) + SBC_HEADER_SIZE))
- #define SBC2UMEM(AP, P) \
- ((void *) (((char *) (P)) + (SBC_HEADER_SIZE + ABLK_HDR_SZ)))
- #define IS_MSEG_CARRIER(C) \
- ((C)->chdr & MSEG_CARRIER_HDR_FLAG)
- #define IS_SYS_ALLOC_CARRIER(C) \
- (!IS_MSEG_CARRIER((C)))
- #define IS_SB_CARRIER(C) \
- ((C)->chdr & SBC_CARRIER_HDR_FLAG)
- #define IS_MB_CARRIER(C) \
- (!IS_SB_CARRIER((C)))
- #define SET_CARRIER_SZ(C, SZ) \
- (ASSERT(((SZ) & CRR_FLG_MASK) == 0), \
- ((C)->chdr = ((C)->chdr & CRR_FLG_MASK) | (SZ)))
- #define CFLG_SBC (1 << 0)
- #define CFLG_MBC (1 << 1)
- #define CFLG_FORCE_MSEG (1 << 2)
- #define CFLG_FORCE_SYS_ALLOC (1 << 3)
- #define CFLG_FORCE_SIZE (1 << 4)
- #define CFLG_MAIN_CARRIER (1 << 5)
- #define CFLG_NO_CPOOL (1 << 6)
- #ifdef ERTS_ALLOC_UTIL_HARD_DEBUG
- static void check_blk_carrier(Allctr_t *, Block_t *);
- #define HARD_CHECK_BLK_CARRIER(A, B) check_blk_carrier((A), (B))
- #else
- #define HARD_CHECK_BLK_CARRIER(A, B)
- #endif
- /* Statistics updating ... */
- #ifdef DEBUG
- #define DEBUG_CHECK_CARRIER_NO_SZ(AP) \
- ASSERT(((AP)->sbcs.curr.norm.mseg.no \
- && (AP)->sbcs.curr.norm.mseg.size) \
- || (!(AP)->sbcs.curr.norm.mseg.no \
- && !(AP)->sbcs.curr.norm.mseg.size)); \
- ASSERT(((AP)->sbcs.curr.norm.sys_alloc.no \
- && (AP)->sbcs.curr.norm.sys_alloc.size) \
- || (!(AP)->sbcs.curr.norm.sys_alloc.no \
- && !(AP)->sbcs.curr.norm.sys_alloc.size)); \
- ASSERT(((AP)->mbcs.curr.norm.mseg.no \
- && (AP)->mbcs.curr.norm.mseg.size) \
- || (!(AP)->mbcs.curr.norm.mseg.no \
- && !(AP)->mbcs.curr.norm.mseg.size)); \
- ASSERT(((AP)->mbcs.curr.norm.sys_alloc.no \
- && (AP)->mbcs.curr.norm.sys_alloc.size) \
- || (!(AP)->mbcs.curr.norm.sys_alloc.no \
- && !(AP)->mbcs.curr.norm.sys_alloc.size));
- #else
- #define DEBUG_CHECK_CARRIER_NO_SZ(AP)
- #endif
- #define STAT_SBC_ALLOC(AP, BSZ) \
- (AP)->sbcs.blocks.curr.size += (BSZ); \
- if ((AP)->sbcs.blocks.max.size < (AP)->sbcs.blocks.curr.size) \
- (AP)->sbcs.blocks.max.size = (AP)->sbcs.blocks.curr.size; \
- if ((AP)->sbcs.max.no < ((AP)->sbcs.curr.norm.mseg.no \
- + (AP)->sbcs.curr.norm.sys_alloc.no)) \
- (AP)->sbcs.max.no = ((AP)->sbcs.curr.norm.mseg.no \
- + (AP)->sbcs.curr.norm.sys_alloc.no); \
- if ((AP)->sbcs.max.size < ((AP)->sbcs.curr.norm.mseg.size \
- + (AP)->sbcs.curr.norm.sys_alloc.size)) \
- (AP)->sbcs.max.size = ((AP)->sbcs.curr.norm.mseg.size \
- + (AP)->sbcs.curr.norm.sys_alloc.size)
- #define STAT_MSEG_SBC_ALLOC(AP, CSZ, BSZ) \
- do { \
- (AP)->sbcs.curr.norm.mseg.no++; \
- (AP)->sbcs.curr.norm.mseg.size += (CSZ); \
- STAT_SBC_ALLOC((AP), (BSZ)); \
- DEBUG_CHECK_CARRIER_NO_SZ((AP)); \
- } while (0)
- #define STAT_SYS_ALLOC_SBC_ALLOC(AP, CSZ, BSZ) \
- do { \
- (AP)->sbcs.curr.norm.sys_alloc.no++; \
- (AP)->sbcs.curr.norm.sys_alloc.size += (CSZ); \
- STAT_SBC_ALLOC((AP), (BSZ)); \
- DEBUG_CHECK_CARRIER_NO_SZ((AP)); \
- } while (0)
- #define STAT_SBC_FREE(AP, BSZ) \
- ASSERT((AP)->sbcs.blocks.curr.size >= (BSZ)); \
- (AP)->sbcs.blocks.curr.size -= (BSZ)
- #define STAT_MSEG_SBC_FREE(AP, CSZ, BSZ) \
- do { \
- ASSERT((AP)->sbcs.curr.norm.mseg.no > 0); \
- (AP)->sbcs.curr.norm.mseg.no--; \
- ASSERT((AP)->sbcs.curr.norm.mseg.size >= (CSZ)); \
- (AP)->sbcs.curr.norm.mseg.size -= (CSZ); \
- STAT_SBC_FREE((AP), (BSZ)); \
- DEBUG_CHECK_CARRIER_NO_SZ((AP)); \
- } while (0)
- #define STAT_SYS_ALLOC_SBC_FREE(AP, CSZ, BSZ) \
- do { \
- ASSERT((AP)->sbcs.curr.norm.sys_alloc.no > 0); \
- (AP)->sbcs.curr.norm.sys_alloc.no--; \
- ASSERT((AP)->sbcs.curr.norm.sys_alloc.size >= (CSZ)); \
- (AP)->sbcs.curr.norm.sys_alloc.size -= (CSZ); \
- STAT_SBC_FREE((AP), (BSZ)); \
- DEBUG_CHECK_CARRIER_NO_SZ((AP)); \
- } while (0)
- #define STAT_MBC_ALLOC(AP) \
- if ((AP)->mbcs.max.no < ((AP)->mbcs.curr.norm.mseg.no \
- + (AP)->mbcs.curr.norm.sys_alloc.no)) \
- (AP)->mbcs.max.no = ((AP)->mbcs.curr.norm.mseg.no \
- + (AP)->mbcs.curr.norm.sys_alloc.no); \
- if ((AP)->mbcs.max.size < ((AP)->mbcs.curr.norm.mseg.size \
- + (AP)->mbcs.curr.norm.sys_alloc.size)) \
- (AP)->mbcs.max.size = ((AP)->mbcs.curr.norm.mseg.size \
- + (AP)->mbcs.curr.norm.sys_alloc.size)
- #define STAT_MSEG_MBC_ALLOC(AP, CSZ) \
- do { \
- (AP)->mbcs.curr.norm.mseg.no++; \
- (AP)->mbcs.curr.norm.mseg.size += (CSZ); \
- STAT_MBC_ALLOC((AP)); \
- DEBUG_CHECK_CARRIER_NO_SZ((AP)); \
- } while (0)
- #define STAT_SYS_ALLOC_MBC_ALLOC(AP, CSZ) \
- do { \
- (AP)->mbcs.curr.norm.sys_alloc.no++; \
- (AP)->mbcs.curr.norm.sys_alloc.size += (CSZ); \
- STAT_MBC_ALLOC((AP)); \
- DEBUG_CHECK_CARRIER_NO_SZ((AP)); \
- } while (0)
- #define STAT_MBC_CPOOL_FETCH(AP, CRR) \
- do { \
- UWord csz__ = CARRIER_SZ((CRR)); \
- if (IS_MSEG_CARRIER((CRR))) \
- STAT_MSEG_MBC_ALLOC((AP), csz__); \
- else \
- STAT_SYS_ALLOC_MBC_ALLOC((AP), csz__); \
- set_new_allctr_abandon_limit(AP); \
- (AP)->mbcs.blocks.curr.no += (CRR)->cpool.blocks[(AP)->alloc_no]; \
- if ((AP)->mbcs.blocks.max.no < (AP)->mbcs.blocks.curr.no) \
- (AP)->mbcs.blocks.max.no = (AP)->mbcs.blocks.curr.no; \
- (AP)->mbcs.blocks.curr.size += \
- (CRR)->cpool.blocks_size[(AP)->alloc_no]; \
- if ((AP)->mbcs.blocks.max.size < (AP)->mbcs.blocks.curr.size) \
- (AP)->mbcs.blocks.max.size = (AP)->mbcs.blocks.curr.size; \
- } while (0)
- #define STAT_MSEG_MBC_FREE(AP, CSZ) \
- do { \
- ASSERT((AP)->mbcs.curr.norm.mseg.no > 0); \
- (AP)->mbcs.curr.norm.mseg.no--; \
- ASSERT((AP)->mbcs.curr.norm.mseg.size >= (CSZ)); \
- (AP)->mbcs.curr.norm.mseg.size -= (CSZ); \
- DEBUG_CHECK_CARRIER_NO_SZ((AP)); \
- } while (0)
- #define STAT_SYS_ALLOC_MBC_FREE(AP, CSZ) \
- do { \
- ASSERT((AP)->mbcs.curr.norm.sys_alloc.no > 0); \
- (AP)->mbcs.curr.norm.sys_alloc.no--; \
- ASSERT((AP)->mbcs.curr.norm.sys_alloc.size >= (CSZ)); \
- (AP)->mbcs.curr.norm.sys_alloc.size -= (CSZ); \
- DEBUG_CHECK_CARRIER_NO_SZ((AP)); \
- } while (0)
- #define STAT_MBC_FREE(AP, CRR) \
- do { \
- UWord csz__ = CARRIER_SZ((CRR)); \
- if (IS_MSEG_CARRIER((CRR))) { \
- STAT_MSEG_MBC_FREE((AP), csz__); \
- } else { \
- STAT_SYS_ALLOC_MBC_FREE((AP), csz__); \
- } \
- set_new_allctr_abandon_limit(AP); \
- } while (0)
- #define STAT_MBC_ABANDON(AP, CRR) \
- do { \
- STAT_MBC_FREE(AP, CRR); \
- ERTS_ALC_CPOOL_ASSERT((AP)->mbcs.blocks.curr.no \
- >= (CRR)->cpool.blocks[(AP)->alloc_no]); \
- (AP)->mbcs.blocks.curr.no -= (CRR)->cpool.blocks[(AP)->alloc_no]; \
- ERTS_ALC_CPOOL_ASSERT((AP)->mbcs.blocks.curr.size \
- >= (CRR)->cpool.blocks_size[(AP)->alloc_no]); \
- (AP)->mbcs.blocks.curr.size -= (CRR)->cpool.blocks_size[(AP)->alloc_no]; \
- } while (0)
- #define STAT_MBC_BLK_ALLOC_CRR(AP, CRR, BSZ) \
- do { \
- (CRR)->cpool.blocks[(AP)->alloc_no]++; \
- (CRR)->cpool.blocks_size[(AP)->alloc_no] += (BSZ); \
- (CRR)->cpool.total_blocks_size += (BSZ); \
- } while (0)
- #define STAT_MBC_BLK_ALLOC(AP, CRR, BSZ, FLGS) \
- do { \
- CarriersStats_t *cstats__ = &(AP)->mbcs; \
- cstats__->blocks.curr.no++; \
- if (cstats__->blocks.max.no < cstats__->blocks.curr.no) \
- cstats__->blocks.max.no = cstats__->blocks.curr.no; \
- cstats__->blocks.curr.size += (BSZ); \
- if (cstats__->blocks.max.size < cstats__->blocks.curr.size) \
- cstats__->blocks.max.size = cstats__->blocks.curr.size; \
- STAT_MBC_BLK_ALLOC_CRR((AP), (CRR), (BSZ)); \
- } while (0)
- static ERTS_INLINE int
- stat_cpool_mbc_blk_free(Allctr_t *allctr,
- ErtsAlcType_t type,
- Carrier_t *crr,
- Carrier_t **busy_pcrr_pp,
- UWord blksz)
- {
- Allctr_t *orig_allctr;
- int alloc_no;
- alloc_no = ERTS_ALC_T2A(type);
- ERTS_ALC_CPOOL_ASSERT(crr->cpool.blocks[alloc_no] > 0);
- crr->cpool.blocks[alloc_no]--;
- ERTS_ALC_CPOOL_ASSERT(crr->cpool.blocks_size[alloc_no] >= blksz);
- crr->cpool.blocks_size[alloc_no] -= blksz;
- ERTS_ALC_CPOOL_ASSERT(crr->cpool.total_blocks_size >= blksz);
- crr->cpool.total_blocks_size -= blksz;
- if (allctr->alloc_no == alloc_no && (!busy_pcrr_pp || !*busy_pcrr_pp)) {
- /* This is a local block, so we should not update the pool
- * statistics. */
- return 0;
- }
- /* This is either a foreign block that's been fetched from the pool, or any
- * block that's in the pool. The carrier's owner keeps the statistics for
- * both pooled and foreign blocks. */
- orig_allctr = crr->cpool.orig_allctr;
- ERTS_ALC_CPOOL_ASSERT(alloc_no != allctr->alloc_no ||
- (crr == *busy_pcrr_pp && allctr == orig_allctr));
- #ifdef ERTS_ALC_CPOOL_DEBUG
- ERTS_ALC_CPOOL_ASSERT(
- erts_atomic_dec_read_nob(&orig_allctr->cpool.stat.no_blocks[alloc_no]) >= 0);
- ERTS_ALC_CPOOL_ASSERT(
- erts_atomic_add_read_nob(&orig_allctr->cpool.stat.blocks_size[alloc_no],
- -((erts_aint_t) blksz)) >= 0);
- #else
- erts_atomic_dec_nob(&orig_allctr->cpool.stat.no_blocks[alloc_no]);
- erts_atomic_add_nob(&orig_allctr->cpool.stat.blocks_size[alloc_no],
- -((erts_aint_t) blksz));
- #endif
- return 1;
- }
- #define STAT_MBC_BLK_FREE(AP, TYPE, CRR, BPCRRPP, BSZ, FLGS) \
- do { \
- if (!stat_cpool_mbc_blk_free((AP), (TYPE), (CRR), (BPCRRPP), (BSZ))) { \
- CarriersStats_t *cstats__ = &(AP)->mbcs; \
- ASSERT(cstats__->blocks.curr.no > 0); \
- cstats__->blocks.curr.no--; \
- ASSERT(cstats__->blocks.curr.size >= (BSZ)); \
- cstats__->blocks.curr.size -= (BSZ); \
- } \
- } while (0)
- /* Debug stuff... */
- #ifdef DEBUG
- static UWord carrier_alignment;
- #define DEBUG_SAVE_ALIGNMENT(C) \
- do { \
- UWord algnmnt__ = sizeof(Unit_t) - (((UWord) (C)) % sizeof(Unit_t));\
- carrier_alignment = MIN(carrier_alignment, algnmnt__); \
- ASSERT(((UWord) (C)) % sizeof(UWord) == 0); \
- } while (0)
- #define DEBUG_CHECK_ALIGNMENT(P) \
- do { \
- ASSERT(sizeof(Unit_t) - (((UWord) (P)) % sizeof(Unit_t)) \
- >= carrier_alignment); \
- ASSERT(((UWord) (P)) % sizeof(UWord) == 0); \
- } while (0)
- #else
- #define DEBUG_SAVE_ALIGNMENT(C)
- #define DEBUG_CHECK_ALIGNMENT(P)
- #endif
- #ifdef DEBUG
- # define IS_ACTUALLY_BLOCKING (erts_thr_progress_is_blocking())
- #define ERTS_ALCU_DBG_CHK_THR_ACCESS(A) \
- do { \
- if (!(A)->thread_safe && !IS_ACTUALLY_BLOCKING) { \
- if (!(A)->debug.saved_tid) { \
- (A)->debug.tid = erts_thr_self(); \
- (A)->debug.saved_tid = 1; \
- } \
- else { \
- ERTS_LC_ASSERT( \
- ethr_equal_tids((A)->debug.tid, erts_thr_self())); \
- } \
- } \
- } while (0)
- #else
- #define ERTS_ALCU_DBG_CHK_THR_ACCESS(A)
- #endif
- static void make_name_atoms(Allctr_t *allctr);
- static Block_t *create_carrier(Allctr_t *, Uint, UWord);
- static void destroy_carrier(Allctr_t *, Block_t *, Carrier_t **);
- static void mbc_free(Allctr_t *allctr, ErtsAlcType_t type, void *p, Carrier_t **busy_pcrr_pp);
- static void dealloc_block(Allctr_t *, ErtsAlcType_t, Uint32, void *, ErtsAlcFixList_t *);
- static alcu_atag_t determine_alloc_tag(Allctr_t *allocator, ErtsAlcType_t type)
- {
- ErtsSchedulerData *esdp;
- Eterm id;
- ERTS_CT_ASSERT(_unchecked_atom_val(am_system) <= MAX_ATAG_ATOM_ID);
- ASSERT(allocator->atags);
- esdp = erts_get_scheduler_data();
- id = am_system;
- if (esdp) {
- if (esdp->current_nif) {
- Module *mod = erts_nif_get_module((esdp->current_nif)->mod_nif);
- /* Mod can be NULL if a resource destructor allocates memory after
- * the module has been unloaded. */
- if (mod) {
- id = make_atom(mod->module);
- }
- } else if (esdp->current_port) {
- Port *p = esdp->current_port;
- id = (p->drv_ptr)->name_atom;
- }
- /* We fall back to 'system' if we can't pack the driver/NIF name into
- * the tag. This may be a bit misleading but we've made no promises
- * that the information is complete.
- *
- * This can only happen on 32-bit emulators when a new driver/NIF has
- * been loaded *after* 16 million atoms have been used, and supporting
- * that fringe case is not worth an extra word. 64-bit emulators are
- * unaffected since the atom cache limits atom indexes to 32 bits. */
- if(MAX_ATOM_TABLE_SIZE > MAX_ATAG_ATOM_ID) {
- if (atom_val(id) > MAX_ATAG_ATOM_ID) {
- id = am_system;
- }
- }
- }
- return MAKE_ATAG(id, ERTS_ALC_T2N(type));
- }
- static void set_alloc_tag(Allctr_t *allocator, void *p, alcu_atag_t tag)
- {
- Block_t *block;
- ASSERT(DBG_IS_VALID_ATAG(tag));
- ASSERT(allocator->atags && p);
- (void)allocator;
- block = UMEM2BLK(p);
- SET_BLK_ATAG(block, tag);
- }
- /* internal data... */
- #if 0
- static ERTS_INLINE void *
- internal_alloc(UWord size)
- {
- void *res = erts_sys_alloc(0, NULL, size);
- if (!res)
- erts_alloc_enomem(ERTS_ALC_T_UNDEF, size);
- return res;
- }
- static ERTS_INLINE void *
- internal_realloc(void *ptr, UWord size)
- {
- void *res = erts_sys_realloc(0, NULL, ptr, size);
- if (!res)
- erts_alloc_enomem(ERTS_ALC_T_UNDEF, size);
- return res;
- }
- static ERTS_INLINE void
- internal_free(void *ptr)
- {
- erts_sys_free(0, NULL, ptr);
- }
- #endif
- #ifdef ARCH_32
- /*
- * Bit vector for the entire 32-bit virtual address space
- * with one bit for each super aligned memory segment.
- */
- #define VSPACE_MAP_BITS (1 << (32 - ERTS_MMAP_SUPERALIGNED_BITS))
- #define VSPACE_MAP_SZ (VSPACE_MAP_BITS / ERTS_VSPACE_WORD_BITS)
- static ERTS_INLINE void set_bit(UWord* map, Uint ix)
- {
- ASSERT(ix / ERTS_VSPACE_WORD_BITS < VSPACE_MAP_SZ);
- map[ix / ERTS_VSPACE_WORD_BITS]
- |= ((UWord)1 << (ix % ERTS_VSPACE_WORD_BITS));
- }
- static ERTS_INLINE void clr_bit(UWord* map, Uint ix)
- {
- ASSERT(ix / ERTS_VSPACE_WORD_BITS < VSPACE_MAP_SZ);
- map[ix / ERTS_VSPACE_WORD_BITS]
- &= ~((UWord)1 << (ix % ERTS_VSPACE_WORD_BITS));
- }
- #ifdef DEBUG
- static ERTS_INLINE int is_bit_set(UWord* map, Uint ix)
- {
- ASSERT(ix / ERTS_VSPACE_WORD_BITS < VSPACE_MAP_SZ);
- return map[ix / ERTS_VSPACE_WORD_BITS]
- & ((UWord)1 << (ix % ERTS_VSPACE_WORD_BITS));
- }
- #endif
- UWord erts_literal_vspace_map[VSPACE_MAP_SZ];
- static void set_literal_range(void* start, Uint size)
- {
- Uint ix = (UWord)start >> ERTS_MMAP_SUPERALIGNED_BITS;
- Uint n = size >> ERTS_MMAP_SUPERALIGNED_BITS;
- ASSERT(!((UWord)start & ERTS_INV_SUPERALIGNED_MASK));
- ASSERT(!((UWord)size & ERTS_INV_SUPERALIGNED_MASK));
- ASSERT(n);
- while (n--) {
- ASSERT(!is_bit_set(erts_literal_vspace_map, ix));
- set_bit(erts_literal_vspace_map, ix);
- ix++;
- }
- }
- static void clear_literal_range(void* start, Uint size)
- {
- Uint ix = (UWord)start >> ERTS_MMAP_SUPERALIGNED_BITS;
- Uint n = size >> ERTS_MMAP_SUPERALIGNED_BITS;
- ASSERT(!((UWord)start & ERTS_INV_SUPERALIGNED_MASK));
- ASSERT(!((UWord)size & ERTS_INV_SUPERALIGNED_MASK));
- ASSERT(n);
- while (n--) {
- ASSERT(is_bit_set(erts_literal_vspace_map, ix));
- clr_bit(erts_literal_vspace_map, ix);
- ix++;
- }
- }
- #endif /* ARCH_32 */
- /* mseg ... */
- #if HAVE_ERTS_MSEG
- static void*
- erts_alcu_mseg_alloc(Allctr_t *allctr, Uint *size_p, Uint flags)
- {
- void *res;
- UWord size = (UWord) *size_p;
- res = erts_mseg_alloc_opt(allctr->alloc_no, &size, flags, &allctr->mseg_opt);
- *size_p = (Uint) size;
- INC_CC(allctr->calls.mseg_alloc);
- return res;
- }
- static void*
- erts_alcu_mseg_realloc(Allctr_t *allctr, void *seg,
- Uint old_size, Uint *new_size_p)
- {
- void *res;
- UWord new_size = (UWord) *new_size_p;
- res = erts_mseg_realloc_opt(allctr->alloc_no, seg, (UWord) old_size, &new_size,
- ERTS_MSEG_FLG_NONE, &allctr->mseg_opt);
- *new_size_p = (Uint) new_size;
- INC_CC(allctr->calls.mseg_realloc);
- return res;
- }
- static void
- erts_alcu_mseg_dealloc(Allctr_t *allctr, void *seg, Uint size, Uint flags)
- {
- erts_mseg_dealloc_opt(allctr->alloc_no, seg, (UWord) size, flags, &allctr->mseg_opt);
- INC_CC(allctr->calls.mseg_dealloc);
- }
- #if defined(ARCH_32)
- void*
- erts_alcu_literal_32_mseg_alloc(Allctr_t *allctr, Uint *size_p, Uint flags)
- {
- void* res;
- Uint sz = ERTS_SUPERALIGNED_CEILING(*size_p);
- ERTS_LC_ASSERT(allctr->alloc_no == ERTS_ALC_A_LITERAL &&
- allctr->t == 0);
- ERTS_LC_ASSERT(allctr->thread_safe);
- res = erts_alcu_mseg_alloc(allctr, &sz, flags);
- if (res) {
- set_literal_range(res, sz);
- *size_p = sz;
- }
- return res;
- }
- void*
- erts_alcu_literal_32_mseg_realloc(Allctr_t *allctr, void *seg,
- Uint old_size, Uint *new_size_p)
- {
- void* res;
- Uint new_sz = ERTS_SUPERALIGNED_CEILING(*new_size_p);
- ERTS_LC_ASSERT(allctr->alloc_no == ERTS_ALC_A_LITERAL &&
- allctr->t == 0);
- ERTS_LC_ASSERT(allctr->thread_safe);
- if (seg && old_size)
- clear_literal_range(seg, old_size);
- res = erts_alcu_mseg_realloc(allctr, seg, old_size, &new_sz);
- if (res) {
- set_literal_range(res, new_sz);
- *new_size_p = new_sz;
- }
- return res;
- }
- void
- erts_alcu_literal_32_mseg_dealloc(Allctr_t *allctr, void *seg, Uint size,
- Uint flags)
- {
- ERTS_LC_ASSERT(allctr->alloc_no == ERTS_ALC_A_LITERAL &&
- allctr->t == 0);
- ERTS_LC_ASSERT(allctr->thread_safe);
- erts_alcu_mseg_dealloc(allctr, seg, size, flags);
- clear_literal_range(seg, size);
- }
- #elif defined(ARCH_64) && defined(ERTS_HAVE_OS_PHYSICAL_MEMORY_RESERVATION)
- /* For allocators that have their own mmapper (super carrier),
- * like literal_alloc.
- */
- void*
- erts_alcu_mmapper_mseg_alloc(Allctr_t *allctr, Uint *size_p, Uint flags)
- {
- void* res;
- UWord size = (UWord) *size_p;
- Uint32 mmap_flags = ERTS_MMAPFLG_SUPERCARRIER_ONLY;
- if (flags & ERTS_MSEG_FLG_2POW)
- mmap_flags |= ERTS_MMAPFLG_SUPERALIGNED;
- res = erts_mmap(allctr->mseg_mmapper, mmap_flags, &size);
- *size_p = (Uint)size;
- INC_CC(allctr->calls.mseg_alloc);
- return res;
- }
- void*
- erts_alcu_mmapper_mseg_realloc(Allctr_t *allctr, void *seg,
- Uint old_size, Uint *new_size_p)
- {
- void *res;
- UWord new_size = (UWord) *new_size_p;
- res = erts_mremap(allctr->mseg_mmapper, ERTS_MSEG_FLG_NONE, seg, old_size, &new_size);
- *new_size_p = (Uint) new_size;
- INC_CC(allctr->calls.mseg_realloc);
- return res;
- }
- void
- erts_alcu_mmapper_mseg_dealloc(Allctr_t *allctr, void *seg, Uint size,
- Uint flags)
- {
- Uint32 mmap_flags = ERTS_MMAPFLG_SUPERCARRIER_ONLY;
- if (flags & ERTS_MSEG_FLG_2POW)
- mmap_flags |= ERTS_MMAPFLG_SUPERALIGNED;
- erts_munmap(allctr->mseg_mmapper, mmap_flags, seg, (UWord)size);
- INC_CC(allctr->calls.mseg_dealloc);
- }
- #endif /* ARCH_64 && ERTS_HAVE_OS_PHYSICAL_MEMORY_RESERVATION */
- #if defined(ERTS_ALC_A_EXEC)
- /*
- * For exec_alloc that need memory with PROT_EXEC
- */
- void*
- erts_alcu_exec_mseg_alloc(Allctr_t *allctr, Uint *size_p, Uint flags)
- {
- void* res = erts_alcu_mseg_alloc(allctr, size_p, flags);
- if (res) {
- int r = mprotect(res, *size_p, PROT_EXEC | PROT_READ | PROT_WRITE);
- ASSERT(r == 0); (void)r;
- }
- return res;
- }
- void*
- erts_alcu_exec_mseg_realloc(Allctr_t *allctr, void *seg,
- Uint old_size, Uint *new_size_p)
- {
- void *res;
- if (seg && old_size) {
- int r = mprotect(seg, old_size, PROT_READ | PROT_WRITE);
- ASSERT(r == 0); (void)r;
- }
- res = erts_alcu_mseg_realloc(allctr, seg, old_size, new_size_p);
- if (res) {
- int r = mprotect(res, *new_size_p, PROT_EXEC | PROT_READ | PROT_WRITE);
- ASSERT(r == 0); (void)r;
- }
- return res;
- }
- void
- erts_alcu_exec_mseg_dealloc(Allctr_t *allctr, void *seg, Uint size, Uint flags)
- {
- int r = mprotect(seg, size, PROT_READ | PROT_WRITE);
- ASSERT(r == 0); (void)r;
- erts_alcu_mseg_dealloc(allctr, seg, size, flags);
- }
- #endif /* ERTS_ALC_A_EXEC */
- #endif /* HAVE_ERTS_MSEG */
- static void*
- erts_alcu_sys_alloc(Allctr_t *allctr, Uint* size_p, int superalign)
- {
- void *res;
- const Uint size = *size_p;
- #if ERTS_SA_MB_CARRIERS && ERTS_HAVE_ERTS_SYS_ALIGNED_ALLOC
- if (superalign)
- res = erts_sys_aligned_alloc(ERTS_SACRR_UNIT_SZ, size);
- else
- #endif
- res = erts_sys_alloc(0, NULL, size);
- INC_CC(allctr->calls.sys_alloc);
- if (erts_mtrace_enabled)
- erts_mtrace_crr_alloc(res, allctr->alloc_no, ERTS_ALC_A_SYSTEM, size);
- return res;
- }
- static void*
- erts_alcu_sys_realloc(Allctr_t *allctr, void *ptr, Uint *size_p, Uint old_size, int superalign)
- {
- void *res;
- const Uint size = *size_p;
- #if ERTS_SA_MB_CARRIERS && ERTS_HAVE_ERTS_SYS_ALIGNED_ALLOC
- if (superalign)
- res = erts_sys_aligned_realloc(ERTS_SACRR_UNIT_SZ, ptr, size, old_size);
- else
- #endif
- res = erts_sys_realloc(0, NULL, ptr, size);
- INC_CC(allctr->calls.sys_realloc);
- if (erts_mtrace_enabled)
- erts_mtrace_crr_realloc(res,
- allctr->alloc_no,
- ERTS_ALC_A_SYSTEM,
- ptr,
- size);
- return res;
- }
- static void
- erts_alcu_sys_dealloc(Allctr_t *allctr, void *ptr, Uint size, int superalign)
- {
- #if ERTS_SA_MB_CARRIERS && ERTS_HAVE_ERTS_SYS_ALIGNED_ALLOC
- if (superalign)
- erts_sys_aligned_free(ERTS_SACRR_UNIT_SZ, ptr);
- else
- #endif
- erts_sys_free(0, NULL, ptr);
- INC_CC(allctr->calls.sys_free);
- if (erts_mtrace_enabled)
- erts_mtrace_crr_free(allctr->alloc_no, ERTS_ALC_A_SYSTEM, ptr);
- }
- #ifdef ARCH_32
- void*
- erts_alcu_literal_32_sys_alloc(Allctr_t *allctr, Uint* size_p, int superalign)
- {
- void* res;
- Uint size = ERTS_SUPERALIGNED_CEILING(*size_p);
- ERTS_LC_ASSERT(allctr->alloc_no == ERTS_ALC_A_LITERAL &&
- allctr->t == 0);
- ERTS_LC_ASSERT(allctr->thread_safe);
- res = erts_alcu_sys_alloc(allctr, &size, 1);
- if (res) {
- set_literal_range(res, size);
- *size_p = size;
- }
- return res;
- }
- void*
- erts_alcu_literal_32_sys_realloc(Allctr_t *allctr, void *ptr, Uint* size_p, Uint old_size, int superalign)
- {
- void* res;
- Uint size = ERTS_SUPERALIGNED_CEILING(*size_p);
- ERTS_LC_ASSERT(allctr->alloc_no == ERTS_ALC_A_LITERAL &&
- allctr->t == 0);
- ERTS_LC_ASSERT(allctr->thread_safe);
- if (ptr && old_size)
- clear_literal_range(ptr, old_size);
- res = erts_alcu_sys_realloc(allctr, ptr, &size, old_size, 1);
- if (res) {
- set_literal_range(res, size);
- *size_p = size;
- }
- return res;
- }
- void
- erts_alcu_literal_32_sys_dealloc(Allctr_t *allctr, void *ptr, Uint size, int superalign)
- {
- ERTS_LC_ASSERT(allctr->alloc_no == ERTS_ALC_A_LITERAL &&
- allctr->t == 0);
- ERTS_LC_ASSERT(allctr->thread_safe);
- erts_alcu_sys_dealloc(allctr, ptr, size, 1);
- clear_literal_range(ptr, size);
- }
- #endif /* ARCH_32 */
- static Uint
- get_next_mbc_size(Allctr_t *allctr)
- {
- Uint size;
- int cs = (allctr->mbcs.curr.norm.mseg.no
- + allctr->mbcs.curr.norm.sys_alloc.no
- - (allctr->main_carrier ? 1 : 0));
- ASSERT(cs >= 0);
- ASSERT(allctr->largest_mbc_size >= allctr->smallest_mbc_size);
- if (cs >= allctr->mbc_growth_stages)
- size = allctr->largest_mbc_size;
- else
- size = ((cs*(allctr->largest_mbc_size - allctr->smallest_mbc_size)
- / allctr->mbc_growth_stages)
- + allctr->smallest_mbc_size);
- if (size < allctr->min_mbc_size)
- size = allctr->min_mbc_size;
- return size;
- }
- static ERTS_INLINE void
- link_carrier(CarrierList_t *cl, Carrier_t *crr)
- {
- crr->next = NULL;
- if (!cl->last) {
- ASSERT(!cl->first);
- cl->first = cl->last = crr;
- crr->prev = NULL;
- }
- else {
- ASSERT(cl->first);
- ASSERT(!cl->first->prev);
- ASSERT(cl->last);
- ASSERT(!cl->last->next);
- crr->prev = cl->last;
- cl->last->next = crr;
- cl->last = crr;
- }
- ASSERT(crr->next != crr);
- ASSERT(crr->prev != crr);
- }
- static ERTS_INLINE void
- relink_carrier(CarrierList_t *cl, Carrier_t *crr)
- {
- if (crr->next) {
- if (crr->next->prev != crr)
- crr->next->prev = crr;
- }
- else if (cl->last != crr)
- cl->last = crr;
- if (crr->prev) {
- if (crr->prev->next != crr)
- crr->prev->next = crr;
- }
- else if (cl->first != crr)
- cl->first = crr;
- }
- static ERTS_INLINE void
- unlink_carrier(CarrierList_t *cl, Carrier_t *crr)
- {
- ASSERT(crr->next != crr);
- ASSERT(crr->prev != crr);
- if (cl->first == crr) {
- ASSERT(!crr->prev);
- cl->first = crr->next;
- }
- else {
- ASSERT(crr->prev);
- crr->prev->next = crr->next;
- }
- if (cl->last == crr) {
- ASSERT(!crr->next);
- cl->last = crr->prev;
- }
- else {
- ASSERT(crr->next);
- crr->next->prev = crr->prev;
- }
- #ifdef DEBUG
- crr->next = crr;
- crr->prev = crr;
- #endif
- }
- static ERTS_INLINE int is_abandoned(Carrier_t *crr)
- {
- return crr->cpool.state != ERTS_MBC_IS_HOME;
- }
- static ERTS_INLINE void
- unlink_abandoned_carrier(Carrier_t *crr)
- {
- if (crr->cpool.state == ERTS_MBC_WAS_POOLED) {
- aoff_remove_pooled_mbc(crr->cpool.orig_allctr, crr);
- }
- }
- static ERTS_INLINE void
- clear_busy_pool_carrier(Allctr_t *allctr, Carrier_t *crr)
- {
- if (crr) {
- erts_aint_t max_size;
- erts_aint_t iallctr;
- max_size = (erts_aint_t) allctr->largest_fblk_in_mbc(allctr, crr);
- erts_atomic_set_nob(&crr->cpool.max_size, max_size);
- iallctr = erts_atomic_read_nob(&crr->allctr);
- ERTS_ALC_CPOOL_ASSERT((iallctr & ~ERTS_CRR_ALCTR_FLG_HOMECOMING)
- == ((erts_aint_t)allctr |
- ERTS_CRR_ALCTR_FLG_IN_POOL |
- ERTS_CRR_ALCTR_FLG_BUSY));
- iallctr &= ~ERTS_CRR_ALCTR_FLG_BUSY;
- erts_atomic_set_relb(&crr->allctr, iallctr);
- }
- }
- #if 0
- #define ERTS_DBG_CHK_FIX_LIST(A, FIX, IX, B) \
- do { if ((FIX)) chk_fix_list((A), (FIX), (IX), (B)); } while (0)
- static void
- chk_fix_list(Allctr_t *allctr, ErtsAlcFixList_t *fix, int ix, int before)
- {
- void *p;
- int n;
- for (n = 0, p = fix[ix].list; p; p = *((void **) p))
- n++;
- if (n != fix[ix].list_size) {
- erts_fprintf(stderr, "FOUND IT ts=%d, sched=%d, ix=%d, n=%d, ls=%d %s!\n",
- allctr->thread_safe, allctr->ix, ix, n, fix[ix].list_size, before ? "before" : "after");
- abort();
- }
- }
- #else
- #define ERTS_DBG_CHK_FIX_LIST(A, FIX, IX, B)
- #endif
- static ERTS_INLINE Allctr_t *get_pref_allctr(void *extra);
- static void *mbc_alloc(Allctr_t *allctr, Uint size);
- static ERTS_INLINE void
- sched_fix_shrink(Allctr_t *allctr, int on)
- {
- if (on && !allctr->fix_shrink_scheduled) {
- allctr->fix_shrink_scheduled = 1;
- erts_set_aux_work_timeout(allctr->ix,
- (ERTS_SSI_AUX_WORK_FIX_ALLOC_LOWER_LIM
- | ERTS_SSI_AUX_WORK_FIX_ALLOC_DEALLOC),
- 1);
- }
- else if (!on && allctr->fix_shrink_scheduled) {
- allctr->fix_shrink_scheduled = 0;
- erts_set_aux_work_timeout(allctr->ix,
- (ERTS_SSI_AUX_WORK_FIX_ALLOC_LOWER_LIM
- | ERTS_SSI_AUX_WORK_FIX_ALLOC_DEALLOC),
- 0);
- }
- }
- static ERTS_INLINE void
- fix_cpool_check_shrink(Allctr_t *allctr,
- ErtsAlcType_t type,
- ErtsAlcFixList_t *fix,
- Carrier_t **busy_pcrr_pp)
- {
- if (fix->u.cpool.shrink_list > 0) {
- if (fix->list_size == 0)
- fix->u.cpool.shrink_list = 0;
- else {
- void *p;
- if (busy_pcrr_pp) {
- clear_busy_pool_carrier(allctr, *busy_pcrr_pp);
- *busy_pcrr_pp = NULL;
- }
- fix->u.cpool.shrink_list--;
- p = fix->list;
- fix->list = *((void **) p);
- fix->list_size--;
- if (fix->u.cpool.min_list_size > fix->list_size)
- fix->u.cpool.min_list_size = fix->list_size;
- dealloc_block(allctr, type, DEALLOC_FLG_FIX_SHRINK, p, fix);
- }
- }
- }
- static ERTS_INLINE void *
- fix_cpool_alloc(Allctr_t *allctr, ErtsAlcType_t type, Uint size)
- {
- void *res;
- ErtsAlcFixList_t *fix;
- fix = &allctr->fix[ERTS_ALC_FIX_TYPE_IX(type)];
- ASSERT(type == fix->type && size == fix->type_size);
- ASSERT(size >= sizeof(ErtsAllctrDDBlock_t));
- res = fix->list;
- if (res) {
- fix->list = *((void **) res);
- fix->list_size--;
- if (fix->u.cpool.min_list_size > fix->list_size)
- fix->u.cpool.min_list_size = fix->list_size;
- fix->u.cpool.used++;
- fix_cpool_check_shrink(allctr, type, fix, NULL);
- return res;
- }
- if (size >= allctr->sbc_threshold) {
- Block_t *blk;
- blk = create_carrier(allctr, size, CFLG_SBC);
- res = blk ? BLK2UMEM(blk) : NULL;
- }
- else
- res = mbc_alloc(allctr, size);
- if (res) {
- fix->u.cpool.used++;
- fix->u.cpool.allocated++;
- }
- return res;
- }
- static ERTS_INLINE void
- fix_cpool_free(Allctr_t *allctr,
- ErtsAlcType_t type,
- Uint32 flags,
- void *p,
- Carrier_t **busy_pcrr_pp)
- {
- ErtsAlcFixList_t *fix;
- Allctr_t *fix_allctr;
- /* If this isn't a fix allocator we need to update the fix list of our
- * neighboring fix_alloc to keep the statistics consistent. */
- if (!allctr->fix) {
- ErtsAllocatorThrSpec_t *tspec = &erts_allctr_thr_spec[ERTS_ALC_A_FIXED_SIZE];
- fix_allctr = get_pref_allctr(tspec);
- ASSERT(!fix_allctr->thread_safe);
- ASSERT(allctr != fix_allctr);
- }
- else {
- fix_allctr = allctr;
- }
- ASSERT(ERTS_ALC_IS_CPOOL_ENABLED(fix_allctr));
- ASSERT(ERTS_ALC_IS_CPOOL_ENABLED(allctr));
- fix = &fix_allctr->fix[ERTS_ALC_FIX_TYPE_IX(type)];
- ASSERT(type == fix->type);
- if (!(flags & DEALLOC_FLG_FIX_SHRINK)) {
- fix->u.cpool.used--;
- }
- /* We don't want foreign blocks to be long-lived, so we skip recycling if
- * allctr != fix_allctr. */
- if (allctr == fix_allctr
- && (!busy_pcrr_pp || !*busy_pcrr_pp)
- && !fix->u.cpool.shrink_list
- && fix->list_size < ERTS_ALCU_FIX_MAX_LIST_SZ) {
- *((void **) p) = fix->list;
- fix->list = p;
- fix->list_size++;
- sched_fix_shrink(allctr, 1);
- }
- else {
- Block_t *blk = UMEM2BLK(p);
- if (IS_SBC_BLK(blk))
- destroy_carrier(allctr, blk, NULL);
- else
- mbc_free(allctr, type, p, busy_pcrr_pp);
- fix->u.cpool.allocated--;
- fix_cpool_check_shrink(allctr, type, fix, busy_pcrr_pp);
- }
- }
- static ERTS_INLINE erts_aint32_t
- fix_cpool_alloc_shrink(Allctr_t *allctr, erts_aint32_t flgs)
- {
- int all_empty = 1;
- erts_aint32_t res = 0;
- int ix, o;
- int flush = flgs == 0;
- if (allctr->thread_safe)
- erts_mtx_lock(&allctr->mutex);
- for (ix = 0; ix < ERTS_ALC_NO_FIXED_SIZES; ix++) {
- ErtsAlcFixList_t *fix = &allctr->fix[ix];
- ErtsAlcType_t type;
- ERTS_DBG_CHK_FIX_LIST(allctr, fix, ix, 1);
- if (flush)
- fix->u.cpool.shrink_list = fix->list_size;
- else if (flgs & ERTS_SSI_AUX_WORK_FIX_ALLOC_LOWER_LIM) {
- fix->u.cpool.shrink_list = fix->u.cpool.min_list_size;
- fix->u.cpool.min_list_size = fix->list_size;
- }
- type = ERTS_ALC_N2T((ErtsAlcType_t) (ix + ERTS_ALC_N_MIN_A_FIXED_SIZE));
- for (o = 0; o < ERTS_ALC_FIX_MAX_SHRINK_OPS || flush; o++) {
- void *ptr;
- if (fix->u.cpool.shrink_list == 0)
- break;
- if (fix->list_size == 0) {
- fix->u.cpool.shrink_list = 0;
- break;
- }
- ptr = fix->list;
- fix->list = *((void **) ptr);
- fix->list_size--;
- fix->u.cpool.shrink_list--;
- dealloc_block(allctr, type, DEALLOC_FLG_FIX_SHRINK, ptr, fix);
- }
- if (fix->u.cpool.min_list_size > fix->list_size)
- fix->u.cpool.min_list_size = fix->list_size;
- if (fix->list_size != 0) {
- if (fix->u.cpool.shrink_list > 0)
- res |= ERTS_SSI_AUX_WORK_FIX_ALLOC_DEALLOC;
- all_empty = 0;
- }
- }
- if (all_empty)
- sched_fix_shrink(allctr, 0);
- if (allctr->thread_safe)
- erts_mtx_unlock(&allctr->mutex);
- return res;
- }
- static ERTS_INLINE void *
- fix_nocpool_alloc(Allctr_t *allctr, ErtsAlcType_t type, Uint size)
- {
- ErtsAlcFixList_t *fix;
- void *res;
- fix = &allctr->fix[ERTS_ALC_FIX_TYPE_IX(type)];
- ASSERT(type == fix->type && size == fix->type_size);
- ASSERT(size >= sizeof(ErtsAllctrDDBlock_t));
- ERTS_DBG_CHK_FIX_LIST(allctr, fix, ix, 1);
- fix->u.nocpool.used++;
- res = fix->list;
- if (res) {
- fix->list_size--;
- fix->list = *((void **) res);
- if (fix->list && fix->u.nocpool.allocated > fix->u.nocpool.limit) {
- Block_t *blk;
- void *p = fix->list;
- fix->list = *((void **) p);
- fix->list_size--;
- blk = UMEM2BLK(p);
- if (IS_SBC_BLK(blk))
- destroy_carrier(allctr, blk, NULL);
- else
- mbc_free(allctr, type, p, NULL);
- fix->u.nocpool.allocated--;
- }
- ERTS_DBG_CHK_FIX_LIST(allctr, fix, ix, 0);
- return res;
- }
- if (fix->u.nocpool.limit < fix->u.nocpool.used)
- fix->u.nocpool.limit = fix->u.nocpool.used;
- if (fix->u.nocpool.max_used < fix->u.nocpool.used)
- fix->u.nocpool.max_used = fix->u.nocpool.used;
- fix->u.nocpool.allocated++;
- if (size >= allctr->sbc_threshold) {
- Block_t *blk;
- blk = create_carrier(allctr, size, CFLG_SBC);
- res = blk ? BLK2UMEM(blk) : NULL;
- }
- else
- res = mbc_alloc(allctr, size);
- if (!res) {
- fix->u.nocpool.allocated--;
- fix->u.nocpool.used--;
- }
- return res;
- }
- static ERTS_INLINE void
- fix_nocpool_free(Allctr_t *allctr,
- ErtsAlcType_t type,
- void *p)
- {
- Block_t *blk;
- ErtsAlcFixList_t *fix;
- fix = &allctr->fix[ERTS_ALC_T2N(type) - ERTS_ALC_N_MIN_A_FIXED_SIZE];
- ASSERT(fix->type == type);
- ERTS_DBG_CHK_FIX_LIST(allctr, fix, ix, 1);
- fix->u.nocpool.used--;
- if (fix->u.nocpool.allocated < fix->u.nocpool.limit
- && fix->list_size < ERTS_ALCU_FIX_MAX_LIST_SZ) {
- *((void **) p) = fix->list;
- fix->list = p;
- fix->list_size++;
- sched_fix_shrink(allctr, 1);
- ERTS_DBG_CHK_FIX_LIST(allctr, fix, ix, 0);
- return;
- }
- fix->u.nocpool.allocated--;
- if (fix->list && fix->u.nocpool.allocated > fix->u.nocpool.limit) {
- blk = UMEM2BLK(p);
- if (IS_SBC_BLK(blk))
- destroy_carrier(allctr, blk, NULL);
- else
- mbc_free(allctr, type, p, NULL);
- p = fix->list;
- fix->list = *((void **) p);
- fix->list_size--;
- fix->u.nocpool.allocated--;
- }
- blk = UMEM2BLK(p);
- if (IS_SBC_BLK(blk))
- destroy_carrier(allctr, blk, NULL);
- else
- mbc_free(allctr, type, p, NULL);
- ERTS_DBG_CHK_FIX_LIST(allctr, fix, ix, 0);
- }
- static ERTS_INLINE erts_aint32_t
- fix_nocpool_alloc_shrink(Allctr_t *allctr, erts_aint32_t flgs)
- {
- int all_empty = 1;
- erts_aint32_t res = 0;
- int ix, o;
- int flush = flgs == 0;
- if (allctr->thread_safe)
- erts_mtx_lock(&allctr->mutex);
- for (i…
Large files files are truncated, but you can click here to view the full file