1// Copyright 2014 The Go Authors. All rights reserved.2// Use of this source code is governed by a BSD-style3// license that can be found in the LICENSE file.45// Memory allocator.6//7// This was originally based on tcmalloc, but has diverged quite a bit.8// http://goog-perftools.sourceforge.net/doc/tcmalloc.html910// The main allocator works in runs of pages.11// Small allocation sizes (up to and including 32 kB) are12// rounded to one of about 70 size classes, each of which13// has its own free set of objects of exactly that size.14// Any free page of memory can be split into a set of objects15// of one size class, which are then managed using a free bitmap.16//17// The allocator's data structures are:18//19// fixalloc: a free-list allocator for fixed-size off-heap objects,20// used to manage storage used by the allocator.21// mheap: the malloc heap, managed at page (8192-byte) granularity.22// mspan: a run of in-use pages managed by the mheap.23// mcentral: collects all spans of a given size class.24// mcache: a per-P cache of mspans with free space.25// mstats: allocation statistics.26//27// Allocating a small object proceeds up a hierarchy of caches:28//29// 1. Round the size up to one of the small size classes30// and look in the corresponding mspan in this P's mcache.31// Scan the mspan's free bitmap to find a free slot.32// If there is a free slot, allocate it.33// This can all be done without acquiring a lock.34//35// 2. If the mspan has no free slots, obtain a new mspan36// from the mcentral's list of mspans of the required size37// class that have free space.38// Obtaining a whole span amortizes the cost of locking39// the mcentral.40//41// 3. If the mcentral's mspan list is empty, obtain a run42// of pages from the mheap to use for the mspan.43//44// 4. If the mheap is empty or has no page runs large enough,45// allocate a new group of pages (at least 1MB) from the46// operating system. Allocating a large run of pages47// amortizes the cost of talking to the operating system.48//49// Sweeping an mspan and freeing objects on it proceeds up a similar50// hierarchy:51//52// 1. If the mspan is being swept in response to allocation, it53// is returned to the mcache to satisfy the allocation.54//55// 2. Otherwise, if the mspan still has allocated objects in it,56// it is placed on the mcentral free list for the mspan's size57// class.58//59// 3. Otherwise, if all objects in the mspan are free, the mspan's60// pages are returned to the mheap and the mspan is now dead.61//62// Allocating and freeing a large object uses the mheap63// directly, bypassing the mcache and mcentral.64//65// If mspan.needzero is false, then free object slots in the mspan are66// already zeroed. Otherwise if needzero is true, objects are zeroed as67// they are allocated. There are various benefits to delaying zeroing68// this way:69//70// 1. Stack frame allocation can avoid zeroing altogether.71//72// 2. It exhibits better temporal locality, since the program is73// probably about to write to the memory.74//75// 3. We don't zero pages that never get reused.7677// Virtual memory layout78//79// The heap consists of a set of arenas, which are 64MB on 64-bit and80// 4MB on 32-bit (heapArenaBytes). Each arena's start address is also81// aligned to the arena size.82//83// Each arena has an associated heapArena object that stores the84// metadata for that arena: the heap bitmap for all words in the arena85// and the span map for all pages in the arena. heapArena objects are86// themselves allocated off-heap.87//88// Since arenas are aligned, the address space can be viewed as a89// series of arena frames. The arena map (mheap_.arenas) maps from90// arena frame number to *heapArena, or nil for parts of the address91// space not backed by the Go heap. The arena map is structured as a92// two-level array consisting of a "L1" arena map and many "L2" arena93// maps; however, since arenas are large, on many architectures, the94// arena map consists of a single, large L2 map.95//96// The arena map covers the entire possible address space, allowing97// the Go heap to use any part of the address space. The allocator98// attempts to keep arenas contiguous so that large spans (and hence99// large objects) can cross arenas.100101package runtime102103import (104 "internal/goarch"105 "internal/goexperiment"106 "internal/goos"107 "internal/runtime/atomic"108 "internal/runtime/gc"109 "internal/runtime/math"110 "internal/runtime/sys"111 "unsafe"112)113114const (115 maxTinySize = _TinySize116 tinySizeClass = _TinySizeClass117 maxSmallSize = gc.MaxSmallSize118 pageSize = 1 << gc.PageShift119 pageMask = pageSize - 1120121 // Unused. Left for viewcore.122 _PageSize = pageSize123 minSizeForMallocHeader = gc.MinSizeForMallocHeader124 mallocHeaderSize = gc.MallocHeaderSize125126 // _64bit = 1 on 64-bit systems, 0 on 32-bit systems127 _64bit = 1 << (^uintptr(0) >> 63) / 2128129 // Tiny allocator parameters, see "Tiny allocator" comment in malloc.go.130 _TinySize = gc.TinySize131 _TinySizeClass = int8(gc.TinySizeClass)132133 _FixAllocChunk = 16 << 10 // Chunk size for FixAlloc134135 // Per-P, per order stack segment cache size.136 _StackCacheSize = 32 * 1024137138 // Number of orders that get caching. Order 0 is FixedStack139 // and each successive order is twice as large.140 // We want to cache 2KB, 4KB, 8KB, and 16KB stacks. Larger stacks141 // will be allocated directly.142 // Since FixedStack is different on different systems, we143 // must vary NumStackOrders to keep the same maximum cached size.144 // OS | FixedStack | NumStackOrders145 // -----------------+------------+---------------146 // linux/darwin/bsd | 2KB | 4147 // windows/32 | 4KB | 3148 // windows/64 | 8KB | 2149 // plan9 | 4KB | 3150 _NumStackOrders = 4 - goarch.PtrSize/4*goos.IsWindows - 1*goos.IsPlan9151152 // heapAddrBits is the number of bits in a heap address. On153 // amd64, addresses are sign-extended beyond heapAddrBits. On154 // other arches, they are zero-extended.155 //156 // On most 64-bit platforms, we limit this to 48 bits based on a157 // combination of hardware and OS limitations.158 //159 // amd64 hardware limits addresses to 48 bits, sign-extended160 // to 64 bits. Addresses where the top 16 bits are not either161 // all 0 or all 1 are "non-canonical" and invalid. Because of162 // these "negative" addresses, we offset addresses by 1<<47163 // (arenaBaseOffset) on amd64 before computing indexes into164 // the heap arenas index. In 2017, amd64 hardware added165 // support for 57 bit addresses; however, currently only Linux166 // supports this extension and the kernel will never choose an167 // address above 1<<47 unless mmap is called with a hint168 // address above 1<<47 (which we never do).169 //170 // arm64 hardware (as of ARMv8) limits user addresses to 48171 // bits, in the range [0, 1<<48).172 //173 // ppc64, mips64, and s390x support arbitrary 64 bit addresses174 // in hardware. On Linux, Go leans on stricter OS limits. Based175 // on Linux's processor.h, the user address space is limited as176 // follows on 64-bit architectures:177 //178 // Architecture Name Maximum Value (exclusive)179 // ---------------------------------------------------------------------180 // amd64 TASK_SIZE_MAX 0x007ffffffff000 (47 bit addresses)181 // arm64 TASK_SIZE_64 0x01000000000000 (48 bit addresses)182 // ppc64{,le} TASK_SIZE_USER64 0x00400000000000 (46 bit addresses)183 // mips64{,le} TASK_SIZE64 0x00010000000000 (40 bit addresses)184 // s390x TASK_SIZE 1<<64 (64 bit addresses)185 //186 // These limits may increase over time, but are currently at187 // most 48 bits except on s390x. On all architectures, Linux188 // starts placing mmap'd regions at addresses that are189 // significantly below 48 bits, so even if it's possible to190 // exceed Go's 48 bit limit, it's extremely unlikely in191 // practice.192 //193 // On 32-bit platforms, we accept the full 32-bit address194 // space because doing so is cheap.195 // mips32 only has access to the low 2GB of virtual memory, so196 // we further limit it to 31 bits.197 //198 // On ios/arm64, although 64-bit pointers are presumably199 // available, pointers are truncated to 33 bits in iOS <14.200 // Furthermore, only the top 4 GiB of the address space are201 // actually available to the application. In iOS >=14, more202 // of the address space is available, and the OS can now203 // provide addresses outside of those 33 bits. Pick 40 bits204 // as a reasonable balance between address space usage by the205 // page allocator, and flexibility for what mmap'd regions206 // we'll accept for the heap. We can't just move to the full207 // 48 bits because this uses too much address space for older208 // iOS versions.209 // TODO(mknyszek): Once iOS <14 is deprecated, promote ios/arm64210 // to a 48-bit address space like every other arm64 platform.211 //212 // WebAssembly currently has a limit of 4GB linear memory.213 heapAddrBits = (_64bit*(1-goarch.IsWasm)*(1-goos.IsIos*goarch.IsArm64))*48 + (1-_64bit+goarch.IsWasm)*(32-(goarch.IsMips+goarch.IsMipsle)) + 40*goos.IsIos*goarch.IsArm64214215 // maxAlloc is the maximum size of an allocation. On 64-bit,216 // it's theoretically possible to allocate 1<<heapAddrBits bytes. On217 // 32-bit, however, this is one less than 1<<32 because the218 // number of bytes in the address space doesn't actually fit219 // in a uintptr.220 maxAlloc = (1 << heapAddrBits) - (1-_64bit)*1221222 // The number of bits in a heap address, the size of heap223 // arenas, and the L1 and L2 arena map sizes are related by224 //225 // (1 << addr bits) = arena size * L1 entries * L2 entries226 //227 // Currently, we balance these as follows:228 //229 // Platform Addr bits Arena size L1 entries L2 entries230 // -------------- --------- ---------- ---------- -----------231 // */64-bit 48 64MB 1 4M (32MB)232 // windows/64-bit 48 4MB 64 1M (8MB)233 // ios/arm64 40 4MB 1 256K (2MB)234 // */32-bit 32 4MB 1 1024 (4KB)235 // */mips(le) 31 4MB 1 512 (2KB)236 // wasm 32 512KB 1 8192 (64KB)237238 // heapArenaBytes is the size of a heap arena. The heap239 // consists of mappings of size heapArenaBytes, aligned to240 // heapArenaBytes. The initial heap mapping is one arena.241 //242 // This is currently 64MB on 64-bit non-Windows, 4MB on243 // 32-bit and on Windows, and 512KB on Wasm. We use smaller244 // arenas on Windows because all committed memory is charged245 // to the process, even if it's not touched. Hence, for246 // processes with small heaps, the mapped arena space needs247 // to be commensurate. This is particularly important with248 // the race detector, since it significantly amplifies the249 // cost of committed memory. We use smaller arenas on Wasm250 // because some Wasm programs have very small heap, and251 // everything in the Wasm linear memory is charged.252 heapArenaBytes = 1 << logHeapArenaBytes253254 heapArenaWords = heapArenaBytes / goarch.PtrSize255256 // logHeapArenaBytes is log_2 of heapArenaBytes. For clarity,257 // prefer using heapArenaBytes where possible (we need the258 // constant to compute some other constants).259 logHeapArenaBytes = (6+20)*(_64bit*(1-goos.IsWindows)*(1-goarch.IsWasm)*(1-goos.IsIos*goarch.IsArm64)) + (2+20)*(_64bit*goos.IsWindows) + (2+20)*(1-_64bit) + (9+10)*goarch.IsWasm + (2+20)*goos.IsIos*goarch.IsArm64260261 // heapArenaBitmapWords is the size of each heap arena's bitmap in uintptrs.262 heapArenaBitmapWords = heapArenaWords / (8 * goarch.PtrSize)263264 pagesPerArena = heapArenaBytes / pageSize265266 // arenaL1Bits is the number of bits of the arena number267 // covered by the first level arena map.268 //269 // This number should be small, since the first level arena270 // map requires PtrSize*(1<<arenaL1Bits) of space in the271 // binary's BSS. It can be zero, in which case the first level272 // index is effectively unused. There is a performance benefit273 // to this, since the generated code can be more efficient,274 // but comes at the cost of having a large L2 mapping.275 //276 // We use the L1 map on 64-bit Windows because the arena size277 // is small, but the address space is still 48 bits, and278 // there's a high cost to having a large L2.279 arenaL1Bits = 6 * (_64bit * goos.IsWindows)280281 // arenaL2Bits is the number of bits of the arena number282 // covered by the second level arena index.283 //284 // The size of each arena map allocation is proportional to285 // 1<<arenaL2Bits, so it's important that this not be too286 // large. 48 bits leads to 32MB arena index allocations, which287 // is about the practical threshold.288 arenaL2Bits = heapAddrBits - logHeapArenaBytes - arenaL1Bits289290 // arenaL1Shift is the number of bits to shift an arena frame291 // number by to compute an index into the first level arena map.292 arenaL1Shift = arenaL2Bits293294 // arenaBits is the total bits in a combined arena map index.295 // This is split between the index into the L1 arena map and296 // the L2 arena map.297 arenaBits = arenaL1Bits + arenaL2Bits298299 // arenaBaseOffset is the pointer value that corresponds to300 // index 0 in the heap arena map.301 //302 // On amd64, the address space is 48 bits, sign extended to 64303 // bits. This offset lets us handle "negative" addresses (or304 // high addresses if viewed as unsigned).305 //306 // On aix/ppc64, this offset allows to keep the heapAddrBits to307 // 48. Otherwise, it would be 60 in order to handle mmap addresses308 // (in range 0x0a00000000000000 - 0x0afffffffffffff). But in this309 // case, the memory reserved in (s *pageAlloc).init for chunks310 // is causing important slowdowns.311 //312 // On other platforms, the user address space is contiguous313 // and starts at 0, so no offset is necessary.314 arenaBaseOffset = 0xffff800000000000*goarch.IsAmd64 + 0x0a00000000000000*goos.IsAix315 // A typed version of this constant that will make it into DWARF (for viewcore).316 arenaBaseOffsetUintptr = uintptr(arenaBaseOffset)317318 // Max number of threads to run garbage collection.319 // 2, 3, and 4 are all plausible maximums depending320 // on the hardware details of the machine. The garbage321 // collector scales well to 32 cpus.322 _MaxGcproc = 32323324 // minLegalPointer is the smallest possible legal pointer.325 // This is the smallest possible architectural page size,326 // since we assume that the first page is never mapped.327 //328 // This should agree with minZeroPage in the compiler.329 minLegalPointer uintptr = 4096330331 // minHeapForMetadataHugePages sets a threshold on when certain kinds of332 // heap metadata, currently the arenas map L2 entries and page alloc bitmap333 // mappings, are allowed to be backed by huge pages. If the heap goal ever334 // exceeds this threshold, then huge pages are enabled.335 //336 // These numbers are chosen with the assumption that huge pages are on the337 // order of a few MiB in size.338 //339 // The kind of metadata this applies to has a very low overhead when compared340 // to address space used, but their constant overheads for small heaps would341 // be very high if they were to be backed by huge pages (e.g. a few MiB makes342 // a huge difference for an 8 MiB heap, but barely any difference for a 1 GiB343 // heap). The benefit of huge pages is also not worth it for small heaps,344 // because only a very, very small part of the metadata is used for small heaps.345 //346 // N.B. If the heap goal exceeds the threshold then shrinks to a very small size347 // again, then huge pages will still be enabled for this mapping. The reason is that348 // there's no point unless we're also returning the physical memory for these349 // metadata mappings back to the OS. That would be quite complex to do in general350 // as the heap is likely fragmented after a reduction in heap size.351 minHeapForMetadataHugePages = 1 << 30352353 // randomizeHeapBase indicates if the heap base address should be randomized.354 // See comment in mallocinit for how the randomization is performed.355 randomizeHeapBase = goexperiment.RandomizedHeapBase64 && goarch.PtrSize == 8 && !isSbrkPlatform && !raceenabled && !msanenabled && !asanenabled356357 // randHeapBasePrefixMask is used to extract the top byte of the randomized358 // heap base address.359 randHeapBasePrefixMask = ^uintptr(0xff << (heapAddrBits - 8))360)361362// physPageSize is the size in bytes of the OS's physical pages.363// Mapping and unmapping operations must be done at multiples of364// physPageSize.365//366// This must be set by the OS init code (typically in osinit) before367// mallocinit.368var physPageSize uintptr369370// physHugePageSize is the size in bytes of the OS's default physical huge371// page size whose allocation is opaque to the application. It is assumed372// and verified to be a power of two.373//374// If set, this must be set by the OS init code (typically in osinit) before375// mallocinit. However, setting it at all is optional, and leaving the default376// value is always safe (though potentially less efficient).377//378// Since physHugePageSize is always assumed to be a power of two,379// physHugePageShift is defined as physHugePageSize == 1 << physHugePageShift.380// The purpose of physHugePageShift is to avoid doing divisions in381// performance critical functions.382var (383 physHugePageSize uintptr384 physHugePageShift uint385)386387var (388 // heapRandSeed is a random value that is populated in mallocinit if389 // randomizeHeapBase is set. It is used in mallocinit, and mheap.grow, to390 // randomize the base heap address.391 heapRandSeed uintptr392 heapRandSeedBitsRemaining int393)394395func nextHeapRandBits(bits int) uintptr {396 if bits > heapRandSeedBitsRemaining {397 throw("not enough heapRandSeed bits remaining")398 }399 r := heapRandSeed >> (64 - bits)400 heapRandSeed <<= bits401 heapRandSeedBitsRemaining -= bits402 return r403}404405func mallocinit() {406 if gc.SizeClassToSize[tinySizeClass] != maxTinySize {407 throw("bad TinySizeClass")408 }409410 if heapArenaBitmapWords&(heapArenaBitmapWords-1) != 0 {411 // heapBits expects modular arithmetic on bitmap412 // addresses to work.413 throw("heapArenaBitmapWords not a power of 2")414 }415416 // Check physPageSize.417 if physPageSize == 0 {418 // The OS init code failed to fetch the physical page size.419 throw("failed to get system page size")420 }421 if physPageSize > maxPhysPageSize {422 print("system page size (", physPageSize, ") is larger than maximum page size (", maxPhysPageSize, ")\n")423 throw("bad system page size")424 }425 if physPageSize < minPhysPageSize {426 print("system page size (", physPageSize, ") is smaller than minimum page size (", minPhysPageSize, ")\n")427 throw("bad system page size")428 }429 if physPageSize&(physPageSize-1) != 0 {430 print("system page size (", physPageSize, ") must be a power of 2\n")431 throw("bad system page size")432 }433 if physHugePageSize&(physHugePageSize-1) != 0 {434 print("system huge page size (", physHugePageSize, ") must be a power of 2\n")435 throw("bad system huge page size")436 }437 if physHugePageSize > maxPhysHugePageSize {438 // physHugePageSize is greater than the maximum supported huge page size.439 // Don't throw here, like in the other cases, since a system configured440 // in this way isn't wrong, we just don't have the code to support them.441 // Instead, silently set the huge page size to zero.442 physHugePageSize = 0443 }444 if physHugePageSize != 0 {445 // Since physHugePageSize is a power of 2, it suffices to increase446 // physHugePageShift until 1<<physHugePageShift == physHugePageSize.447 for 1<<physHugePageShift != physHugePageSize {448 physHugePageShift++449 }450 }451 if pagesPerArena%pagesPerSpanRoot != 0 {452 print("pagesPerArena (", pagesPerArena, ") is not divisible by pagesPerSpanRoot (", pagesPerSpanRoot, ")\n")453 throw("bad pagesPerSpanRoot")454 }455 if pagesPerArena%pagesPerReclaimerChunk != 0 {456 print("pagesPerArena (", pagesPerArena, ") is not divisible by pagesPerReclaimerChunk (", pagesPerReclaimerChunk, ")\n")457 throw("bad pagesPerReclaimerChunk")458 }459 // Check that the minimum size (exclusive) for a malloc header is also460 // a size class boundary. This is important to making sure checks align461 // across different parts of the runtime.462 //463 // While we're here, also check to make sure all these size classes'464 // span sizes are one page. Some code relies on this.465 minSizeForMallocHeaderIsSizeClass := false466 sizeClassesUpToMinSizeForMallocHeaderAreOnePage := true467 for i := 0; i < len(gc.SizeClassToSize); i++ {468 if gc.SizeClassToNPages[i] > 1 {469 sizeClassesUpToMinSizeForMallocHeaderAreOnePage = false470 }471 if gc.MinSizeForMallocHeader == uintptr(gc.SizeClassToSize[i]) {472 minSizeForMallocHeaderIsSizeClass = true473 break474 }475 }476 if !minSizeForMallocHeaderIsSizeClass {477 throw("min size of malloc header is not a size class boundary")478 }479 if !sizeClassesUpToMinSizeForMallocHeaderAreOnePage {480 throw("expected all size classes up to min size for malloc header to fit in one-page spans")481 }482 // Check that the pointer bitmap for all small sizes without a malloc header483 // fits in a word.484 if gc.MinSizeForMallocHeader/goarch.PtrSize > 8*goarch.PtrSize {485 throw("max pointer/scan bitmap size for headerless objects is too large")486 }487488 if minTagBits > tagBits {489 throw("tagBits too small")490 }491492 // Initialize the heap.493 mheap_.init()494 mcache0 = allocmcache()495 lockInit(&gcBitsArenas.lock, lockRankGcBitsArenas)496 lockInit(&profInsertLock, lockRankProfInsert)497 lockInit(&profBlockLock, lockRankProfBlock)498 lockInit(&profMemActiveLock, lockRankProfMemActive)499 for i := range profMemFutureLock {500 lockInit(&profMemFutureLock[i], lockRankProfMemFuture)501 }502 lockInit(&globalAlloc.mutex, lockRankGlobalAlloc)503504 // Create initial arena growth hints.505 if isSbrkPlatform {506 // Don't generate hints on sbrk platforms. We can507 // only grow the break sequentially.508 } else if goarch.PtrSize == 8 {509 // On a 64-bit machine, we pick the following hints510 // because:511 //512 // 1. Starting from the middle of the address space513 // makes it easier to grow out a contiguous range514 // without running in to some other mapping.515 //516 // 2. This makes Go heap addresses more easily517 // recognizable when debugging.518 //519 // 3. Stack scanning in gccgo is still conservative,520 // so it's important that addresses be distinguishable521 // from other data.522 //523 // Starting at 0x00c0 means that the valid memory addresses524 // will begin 0x00c0, 0x00c1, ...525 // In little-endian, that's c0 00, c1 00, ... None of those are valid526 // UTF-8 sequences, and they are otherwise as far away from527 // ff (likely a common byte) as possible. If that fails, we try other 0xXXc0528 // addresses. An earlier attempt to use 0x11f8 caused out of memory errors529 // on OS X during thread allocations. 0x00c0 causes conflicts with530 // AddressSanitizer which reserves all memory up to 0x0100.531 // These choices reduce the odds of a conservative garbage collector532 // not collecting memory because some non-pointer block of memory533 // had a bit pattern that matched a memory address.534 //535 // However, on arm64, we ignore all this advice above and slam the536 // allocation at 0x40 << 32 because when using 4k pages with 3-level537 // translation buffers, the user address space is limited to 39 bits538 // On ios/arm64, the address space is even smaller.539 //540 // On AIX, mmaps starts at 0x0A00000000000000 for 64-bit.541 // processes.542 //543 // Space mapped for user arenas comes immediately after the range544 // originally reserved for the regular heap when race mode is not545 // enabled because user arena chunks can never be used for regular heap546 // allocations and we want to avoid fragmenting the address space.547 //548 // In race mode we have no choice but to just use the same hints because549 // the race detector requires that the heap be mapped contiguously.550 //551 // If randomizeHeapBase is set, we attempt to randomize the base address552 // as much as possible. We do this by generating a random uint64 via553 // bootstrapRand and using it's bits to randomize portions of the base554 // address as follows:555 // * We first generate a random heapArenaBytes aligned address that we use for556 // generating the hints.557 // * On the first call to mheap.grow, we then generate a random PallocChunkBytes558 // aligned offset into the mmap'd heap region, which we use as the base for559 // the heap region.560 // * We then select a page offset in that PallocChunkBytes region to start the561 // heap at, and mark all the pages up to that offset as allocated.562 //563 // Our final randomized "heap base address" becomes the first byte of564 // the first available page returned by the page allocator. This results565 // in an address with at least heapAddrBits-gc.PageShift-2-(1*goarch.IsAmd64)566 // bits of entropy.567568 var randHeapBase uintptr569 var randHeapBasePrefix byte570 // heapAddrBits is 48 on most platforms, but we only use 47 of those571 // bits in order to provide a good amount of room for the heap to grow572 // contiguously. On amd64, there are 48 bits, but the top bit is sign573 // extended, so we throw away another bit, just to be safe.574 randHeapAddrBits := heapAddrBits - 1 - (goarch.IsAmd64 * 1)575 if randomizeHeapBase {576 // Generate a random value, and take the bottom heapAddrBits-logHeapArenaBytes577 // bits, using them as the top bits for randHeapBase.578 heapRandSeed, heapRandSeedBitsRemaining = uintptr(bootstrapRand()), 64579580 topBits := (randHeapAddrBits - logHeapArenaBytes)581 randHeapBase = nextHeapRandBits(topBits) << (randHeapAddrBits - topBits)582 randHeapBase = alignUp(randHeapBase, heapArenaBytes)583 randHeapBasePrefix = byte(randHeapBase >> (randHeapAddrBits - 8))584 }585586 var vmaSize int587 if GOARCH == "riscv64" {588 // Identify which memory layout is in use based on the system589 // stack address, knowing that the bottom half of virtual memory590 // is user space. This should result in 39, 48 or 57. It may be591 // possible to use RISCV_HWPROBE_KEY_HIGHEST_VIRT_ADDRESS at some592 // point in the future - for now use the system stack address.593 vmaSize = sys.Len64(uint64(getg().m.g0.stack.hi)) + 1594 if raceenabled && vmaSize != 39 && vmaSize != 48 {595 println("vma size = ", vmaSize)596 throw("riscv64 vma size is unknown and race mode is enabled")597 }598 }599600 for i := 0x7f; i >= 0; i-- {601 var p uintptr602 switch {603 case raceenabled && GOARCH == "riscv64" && vmaSize == 39:604 p = uintptr(i)<<28 | uintptrMask&(0x0013<<28)605 if p >= uintptrMask&0x000f00000000 {606 continue607 }608 case raceenabled:609 // The TSAN runtime requires the heap610 // to be in the range [0x00c000000000,611 // 0x00e000000000).612 p = uintptr(i)<<32 | uintptrMask&(0x00c0<<32)613 if p >= uintptrMask&0x00e000000000 {614 continue615 }616 case randomizeHeapBase:617 prefix := uintptr(randHeapBasePrefix+byte(i)) << (randHeapAddrBits - 8)618 p = prefix | (randHeapBase & randHeapBasePrefixMask)619 case GOARCH == "arm64" && GOOS == "ios":620 p = uintptr(i)<<40 | uintptrMask&(0x0013<<28)621 case GOARCH == "arm64":622 p = uintptr(i)<<40 | uintptrMask&(0x0040<<32)623 case GOARCH == "riscv64" && vmaSize == 39:624 p = uintptr(i)<<32 | uintptrMask&(0x0013<<28)625 case GOOS == "aix":626 if i == 0 {627 // We don't use addresses directly after 0x0A00000000000000628 // to avoid collisions with others mmaps done by non-go programs.629 continue630 }631 p = uintptr(i)<<40 | uintptrMask&(0xa0<<52)632 default:633 p = uintptr(i)<<40 | uintptrMask&(0x00c0<<32)634 }635 // Switch to generating hints for user arenas if we've gone636 // through about half the hints. In race mode, take only about637 // a quarter; we don't have very much space to work with.638 hintList := &mheap_.arenaHints639 if (!raceenabled && i > 0x3f) || (raceenabled && i > 0x5f) {640 hintList = &mheap_.userArena.arenaHints641 }642 hint := (*arenaHint)(mheap_.arenaHintAlloc.alloc())643 hint.addr = p644 hint.next, *hintList = *hintList, hint645 }646 } else {647 // On a 32-bit machine, we're much more concerned648 // about keeping the usable heap contiguous.649 // Hence:650 //651 // 1. We reserve space for all heapArenas up front so652 // they don't get interleaved with the heap. They're653 // ~258MB, so this isn't too bad. (We could reserve a654 // smaller amount of space up front if this is a655 // problem.)656 //657 // 2. We hint the heap to start right above the end of658 // the binary so we have the best chance of keeping it659 // contiguous.660 //661 // 3. We try to stake out a reasonably large initial662 // heap reservation.663664 const arenaMetaSize = (1 << arenaBits) * unsafe.Sizeof(heapArena{})665 meta := uintptr(sysReserve(nil, arenaMetaSize, "heap reservation"))666 if meta != 0 {667 mheap_.heapArenaAlloc.init(meta, arenaMetaSize, true)668 }669670 // We want to start the arena low, but if we're linked671 // against C code, it's possible global constructors672 // have called malloc and adjusted the process' brk.673 // Query the brk so we can avoid trying to map the674 // region over it (which will cause the kernel to put675 // the region somewhere else, likely at a high676 // address).677 procBrk := sbrk0()678679 // If we ask for the end of the data segment but the680 // operating system requires a little more space681 // before we can start allocating, it will give out a682 // slightly higher pointer. Except QEMU, which is683 // buggy, as usual: it won't adjust the pointer684 // upward. So adjust it upward a little bit ourselves:685 // 1/4 MB to get away from the running binary image.686 p := firstmoduledata.end687 if p < procBrk {688 p = procBrk689 }690 if mheap_.heapArenaAlloc.next <= p && p < mheap_.heapArenaAlloc.end {691 p = mheap_.heapArenaAlloc.end692 }693 p = alignUp(p+(256<<10), heapArenaBytes)694 // Because we're worried about fragmentation on695 // 32-bit, we try to make a large initial reservation.696 arenaSizes := []uintptr{697 512 << 20,698 256 << 20,699 128 << 20,700 }701 for _, arenaSize := range arenaSizes {702 a, size := sysReserveAligned(unsafe.Pointer(p), arenaSize, heapArenaBytes, "heap reservation")703 if a != nil {704 mheap_.arena.init(uintptr(a), size, false)705 p = mheap_.arena.end // For hint below706 break707 }708 }709 hint := (*arenaHint)(mheap_.arenaHintAlloc.alloc())710 hint.addr = p711 hint.next, mheap_.arenaHints = mheap_.arenaHints, hint712713 // Place the hint for user arenas just after the large reservation.714 //715 // While this potentially competes with the hint above, in practice we probably716 // aren't going to be getting this far anyway on 32-bit platforms.717 userArenaHint := (*arenaHint)(mheap_.arenaHintAlloc.alloc())718 userArenaHint.addr = p719 userArenaHint.next, mheap_.userArena.arenaHints = mheap_.userArena.arenaHints, userArenaHint720 }721 // Initialize the memory limit here because the allocator is going to look at it722 // but we haven't called gcinit yet and we're definitely going to allocate memory before then.723 gcController.memoryLimit.Store(math.MaxInt64)724}725726// sysAlloc allocates heap arena space for at least n bytes. The727// returned pointer is always heapArenaBytes-aligned and backed by728// h.arenas metadata. The returned size is always a multiple of729// heapArenaBytes. sysAlloc returns nil on failure.730// There is no corresponding free function.731//732// hintList is a list of hint addresses for where to allocate new733// heap arenas. It must be non-nil.734//735// sysAlloc returns a memory region in the Reserved state. This region must736// be transitioned to Prepared and then Ready before use.737//738// arenaList is the list the arena should be added to.739//740// h must be locked.741func (h *mheap) sysAlloc(n uintptr, hintList **arenaHint, arenaList *[]arenaIdx) (v unsafe.Pointer, size uintptr) {742 assertLockHeld(&h.lock)743744 n = alignUp(n, heapArenaBytes)745746 if hintList == &h.arenaHints {747 // First, try the arena pre-reservation.748 // Newly-used mappings are considered released.749 //750 // Only do this if we're using the regular heap arena hints.751 // This behavior is only for the heap.752 v = h.arena.alloc(n, heapArenaBytes, &gcController.heapReleased, "heap")753 if v != nil {754 size = n755 goto mapped756 }757 }758759 // Try to grow the heap at a hint address.760 for *hintList != nil {761 hint := *hintList762 p := hint.addr763 if hint.down {764 p -= n765 }766 if p+n < p {767 // We can't use this, so don't ask.768 v = nil769 } else if arenaIndex(p+n-1) >= 1<<arenaBits {770 // Outside addressable heap. Can't use.771 v = nil772 } else {773 v = sysReserve(unsafe.Pointer(p), n, "heap reservation")774 }775 if p == uintptr(v) {776 // Success. Update the hint.777 if !hint.down {778 p += n779 }780 hint.addr = p781 size = n782 break783 }784 // Failed. Discard this hint and try the next.785 //786 // TODO: This would be cleaner if sysReserve could be787 // told to only return the requested address. In788 // particular, this is already how Windows behaves, so789 // it would simplify things there.790 if v != nil {791 sysUnreserve(v, n)792 }793 *hintList = hint.next794 h.arenaHintAlloc.free(unsafe.Pointer(hint))795 }796797 if size == 0 {798 if raceenabled {799 // The race detector assumes the heap lives in800 // [0x00c000000000, 0x00e000000000), but we801 // just ran out of hints in this region. Give802 // a nice failure.803 throw("too many address space collisions for -race mode")804 }805806 // All of the hints failed, so we'll take any807 // (sufficiently aligned) address the kernel will give808 // us.809 v, size = sysReserveAligned(nil, n, heapArenaBytes, "heap")810 if v == nil {811 return nil, 0812 }813814 // Create new hints for extending this region.815 hint := (*arenaHint)(h.arenaHintAlloc.alloc())816 hint.addr, hint.down = uintptr(v), true817 hint.next, mheap_.arenaHints = mheap_.arenaHints, hint818 hint = (*arenaHint)(h.arenaHintAlloc.alloc())819 hint.addr = uintptr(v) + size820 hint.next, mheap_.arenaHints = mheap_.arenaHints, hint821 }822823 // Check for bad pointers or pointers we can't use.824 {825 var bad string826 p := uintptr(v)827 if p+size < p {828 bad = "region exceeds uintptr range"829 } else if arenaIndex(p) >= 1<<arenaBits {830 bad = "base outside usable address space"831 } else if arenaIndex(p+size-1) >= 1<<arenaBits {832 bad = "end outside usable address space"833 }834 if bad != "" {835 // This should be impossible on most architectures,836 // but it would be really confusing to debug.837 print("runtime: memory allocated by OS [", hex(p), ", ", hex(p+size), ") not in usable address space: ", bad, "\n")838 throw("memory reservation exceeds address space limit")839 }840 }841842 if uintptr(v)&(heapArenaBytes-1) != 0 {843 throw("misrounded allocation in sysAlloc")844 }845846mapped:847 if valgrindenabled {848 valgrindCreateMempool(v)849 valgrindMakeMemNoAccess(v, size)850 }851852 // Create arena metadata.853 for ri := arenaIndex(uintptr(v)); ri <= arenaIndex(uintptr(v)+size-1); ri++ {854 l2 := h.arenas[ri.l1()]855 if l2 == nil {856 // Allocate an L2 arena map.857 //858 // Use sysAllocOS instead of sysAlloc or persistentalloc because there's no859 // statistic we can comfortably account for this space in. With this structure,860 // we rely on demand paging to avoid large overheads, but tracking which memory861 // is paged in is too expensive. Trying to account for the whole region means862 // that it will appear like an enormous memory overhead in statistics, even though863 // it is not.864 l2 = (*[1 << arenaL2Bits]*heapArena)(sysAllocOS(unsafe.Sizeof(*l2), "heap index"))865 if l2 == nil {866 throw("out of memory allocating heap arena map")867 }868 if h.arenasHugePages {869 sysHugePage(unsafe.Pointer(l2), unsafe.Sizeof(*l2))870 } else {871 sysNoHugePage(unsafe.Pointer(l2), unsafe.Sizeof(*l2))872 }873 atomic.StorepNoWB(unsafe.Pointer(&h.arenas[ri.l1()]), unsafe.Pointer(l2))874 }875876 if l2[ri.l2()] != nil {877 throw("arena already initialized")878 }879 var r *heapArena880 r = (*heapArena)(h.heapArenaAlloc.alloc(unsafe.Sizeof(*r), goarch.PtrSize, &memstats.gcMiscSys, "heap metadata"))881 if r == nil {882 r = (*heapArena)(persistentalloc(unsafe.Sizeof(*r), goarch.PtrSize, &memstats.gcMiscSys))883 if r == nil {884 throw("out of memory allocating heap arena metadata")885 }886 }887888 // Register the arena in allArenas if requested.889 if len((*arenaList)) == cap((*arenaList)) {890 size := 2 * uintptr(cap((*arenaList))) * goarch.PtrSize891 if size == 0 {892 size = physPageSize893 }894 newArray := (*notInHeap)(persistentalloc(size, goarch.PtrSize, &memstats.gcMiscSys))895 if newArray == nil {896 throw("out of memory allocating allArenas")897 }898 oldSlice := (*arenaList)899 *(*notInHeapSlice)(unsafe.Pointer(&(*arenaList))) = notInHeapSlice{newArray, len((*arenaList)), int(size / goarch.PtrSize)}900 copy((*arenaList), oldSlice)901 // Do not free the old backing array because902 // there may be concurrent readers. Since we903 // double the array each time, this can lead904 // to at most 2x waste.905 }906 (*arenaList) = (*arenaList)[:len((*arenaList))+1]907 (*arenaList)[len((*arenaList))-1] = ri908909 // Store atomically just in case an object from the910 // new heap arena becomes visible before the heap lock911 // is released (which shouldn't happen, but there's912 // little downside to this).913 atomic.StorepNoWB(unsafe.Pointer(&l2[ri.l2()]), unsafe.Pointer(r))914 }915916 // Tell the race detector about the new heap memory.917 if raceenabled {918 racemapshadow(v, size)919 }920921 return922}923924// enableMetadataHugePages enables huge pages for various sources of heap metadata.925//926// A note on latency: for sufficiently small heaps (<10s of GiB) this function will take constant927// time, but may take time proportional to the size of the mapped heap beyond that.928//929// This function is idempotent.930//931// The heap lock must not be held over this operation, since it will briefly acquire932// the heap lock.933//934// Must be called on the system stack because it acquires the heap lock.935//936//go:systemstack937func (h *mheap) enableMetadataHugePages() {938 // Enable huge pages for page structure.939 h.pages.enableChunkHugePages()940941 // Grab the lock and set arenasHugePages if it's not.942 //943 // Once arenasHugePages is set, all new L2 entries will be eligible for944 // huge pages. We'll set all the old entries after we release the lock.945 lock(&h.lock)946 if h.arenasHugePages {947 unlock(&h.lock)948 return949 }950 h.arenasHugePages = true951 unlock(&h.lock)952953 // N.B. The arenas L1 map is quite small on all platforms, so it's fine to954 // just iterate over the whole thing.955 for i := range h.arenas {956 l2 := (*[1 << arenaL2Bits]*heapArena)(atomic.Loadp(unsafe.Pointer(&h.arenas[i])))957 if l2 == nil {958 continue959 }960 sysHugePage(unsafe.Pointer(l2), unsafe.Sizeof(*l2))961 }962}963964// base address for all 0-byte allocations965var zerobase uintptr966967// nextFreeFast returns the next free object if one is quickly available.968// Otherwise it returns 0.969func nextFreeFast(s *mspan) gclinkptr {970 theBit := sys.TrailingZeros64(s.allocCache) // Is there a free object in the allocCache?971 if theBit < 64 {972 result := s.freeindex + uint16(theBit)973 if result < s.nelems {974 freeidx := result + 1975 if freeidx%64 == 0 && freeidx != s.nelems {976 return 0977 }978 s.allocCache >>= uint(theBit + 1)979 s.freeindex = freeidx980 s.allocCount++981 return gclinkptr(uintptr(result)*s.elemsize + s.base())982 }983 }984 return 0985}986987// nextFree returns the next free object from the cached span if one is available.988// Otherwise it refills the cache with a span with an available object and989// returns that object along with a flag indicating that this was a heavy990// weight allocation. If it is a heavy weight allocation the caller must991// determine whether a new GC cycle needs to be started or if the GC is active992// whether this goroutine needs to assist the GC.993//994// Must run in a non-preemptible context since otherwise the owner of995// c could change.996func (c *mcache) nextFree(spc spanClass) (v gclinkptr, s *mspan, checkGCTrigger bool) {997 s = c.alloc[spc]998 checkGCTrigger = false999 freeIndex := s.nextFreeIndex()1000 if freeIndex == s.nelems {1001 // The span is full.1002 if s.allocCount != s.nelems {1003 println("runtime: s.allocCount=", s.allocCount, "s.nelems=", s.nelems)1004 throw("s.allocCount != s.nelems && freeIndex == s.nelems")1005 }1006 c.refill(spc)1007 checkGCTrigger = true1008 s = c.alloc[spc]10091010 freeIndex = s.nextFreeIndex()1011 }10121013 if freeIndex >= s.nelems {1014 throw("freeIndex is not valid")1015 }10161017 v = gclinkptr(uintptr(freeIndex)*s.elemsize + s.base())1018 s.allocCount++1019 if s.allocCount > s.nelems {1020 println("s.allocCount=", s.allocCount, "s.nelems=", s.nelems)1021 throw("s.allocCount > s.nelems")1022 }1023 return1024}10251026// doubleCheckMalloc enables a bunch of extra checks to malloc to double-check1027// that various invariants are upheld.1028//1029// We might consider turning these on by default; many of them previously were.1030// They account for a few % of mallocgc's cost though, which does matter somewhat1031// at scale. (When testing changes to malloc, consider enabling this, and also1032// some function-local 'doubleCheck' consts such as in mbitmap.go currently.)1033const doubleCheckMalloc = false10341035// sizeSpecializedMallocEnabled is the set of conditions where we enable the size-specialized1036// mallocgc implementation: none of the sanitizers should be enabled. The tables used to select1037// the size-specialized malloc function do not compile properly on plan9, so1038// size-specialized malloc is also disabled on plan9.1039const sizeSpecializedMallocEnabled = GOOS != "plan9" && !asanenabled && !raceenabled && !msanenabled && !valgrindenabled10401041// runtimeFreegcEnabled is the set of conditions where we enable the runtime.freegc1042// implementation and the corresponding allocation-related changes: the experiment must be1043// enabled, and none of the memory sanitizers should be enabled. We allow the race detector,1044// in contrast to sizeSpecializedMallocEnabled.1045// TODO(thepudds): it would be nice to check Valgrind integration, though there are some hints1046// there might not be any canned tests in tree for Go's integration with Valgrind.1047const runtimeFreegcEnabled = goexperiment.RuntimeFreegc && !asanenabled && !msanenabled && !valgrindenabled10481049// Allocate an object of size bytes.1050// Small objects are allocated from the per-P cache's free lists.1051// Large objects (> 32 kB) are allocated straight from the heap.1052//1053// mallocgc should be an internal detail,1054// but widely used packages access it using linkname.1055// Notable members of the hall of shame include:1056// - github.com/bytedance/gopkg1057// - github.com/bytedance/sonic1058// - github.com/cloudwego/frugal1059// - github.com/cockroachdb/cockroach1060// - github.com/cockroachdb/pebble1061// - github.com/ugorji/go/codec1062//1063// Do not remove or change the type signature.1064// See go.dev/issue/67401.1065//1066//go:linkname mallocgc1067func mallocgc(size uintptr, typ *_type, needzero bool) unsafe.Pointer {1068 if doubleCheckMalloc {1069 if gcphase == _GCmarktermination {1070 throw("mallocgc called with gcphase == _GCmarktermination")1071 }1072 }10731074 // Short-circuit zero-sized allocation requests.1075 if size == 0 {1076 return unsafe.Pointer(&zerobase)1077 }10781079 if sizeSpecializedMallocEnabled && size < uintptr(len(mallocNoScanTable)) {1080 if typ == nil || !typ.Pointers() {1081 if size >= maxTinySize {1082 return mallocNoScanTable[size](size, typ, needzero)1083 }1084 return mallocgcTinySC2(size, typ, needzero)1085 } else {1086 if !needzero {1087 throw("objects with pointers must be zeroed")1088 }1089 return mallocScanTable[size](size, typ, needzero)1090 }1091 }10921093 // It's possible for any malloc to trigger sweeping, which may in1094 // turn queue finalizers. Record this dynamic lock edge.1095 // N.B. Compiled away if lockrank experiment is not enabled.1096 lockRankMayQueueFinalizer()10971098 // Pre-malloc debug hooks.1099 if debug.malloc {1100 if x := preMallocgcDebug(size, typ); x != nil {1101 return x1102 }1103 }11041105 // For ASAN, we allocate extra memory around each allocation called the "redzone."1106 // These "redzones" are marked as unaddressable.1107 var asanRZ uintptr1108 if asanenabled {1109 asanRZ = redZoneSize(size)1110 size += asanRZ1111 }11121113 // Assist the GC if needed. (On the reuse path, we currently compensate for this;1114 // changes here might require changes there.)1115 if gcBlackenEnabled != 0 {1116 deductAssistCredit(size)1117 }11181119 // Actually do the allocation.1120 var x unsafe.Pointer1121 var elemsize uintptr1122 if sizeSpecializedMallocEnabled {1123 if size <= maxSmallSize-gc.MallocHeaderSize {1124 if typ == nil || !typ.Pointers() {1125 x, elemsize = mallocgcSmallNoscan(size, typ, needzero)1126 } else {1127 if !needzero {1128 throw("objects with pointers must be zeroed")1129 }1130 if heapBitsInSpan(size) {1131 x, elemsize = mallocgcSmallScanNoHeader(size, typ)1132 } else {1133 x, elemsize = mallocgcSmallScanHeader(size, typ)1134 }1135 }1136 } else {1137 x, elemsize = mallocgcLarge(size, typ, needzero)1138 }1139 } else {1140 if size <= maxSmallSize-gc.MallocHeaderSize {1141 if typ == nil || !typ.Pointers() {1142 // tiny allocations might be kept alive by other co-located values.1143 // Make sure secret allocations get zeroed by avoiding the tiny allocator1144 // See go.dev/issue/763561145 gp := getg()1146 if size < maxTinySize && gp.secret == 0 {1147 x, elemsize = mallocgcTiny(size, typ)1148 } else {1149 x, elemsize = mallocgcSmallNoscan(size, typ, needzero)1150 }1151 } else {1152 if !needzero {1153 throw("objects with pointers must be zeroed")1154 }1155 if heapBitsInSpan(size) {1156 x, elemsize = mallocgcSmallScanNoHeader(size, typ)1157 } else {1158 x, elemsize = mallocgcSmallScanHeader(size, typ)1159 }1160 }1161 } else {1162 x, elemsize = mallocgcLarge(size, typ, needzero)1163 }1164 }11651166 gp := getg()1167 if goexperiment.RuntimeSecret && gp.secret > 0 {1168 // Mark any object allocated while in secret mode as secret.1169 // This ensures we zero it immediately when freeing it.1170 addSecret(x, size)1171 }11721173 // Notify sanitizers, if enabled.1174 if raceenabled {1175 racemalloc(x, size-asanRZ)1176 }1177 if msanenabled {1178 msanmalloc(x, size-asanRZ)1179 }1180 if asanenabled {1181 // Poison the space between the end of the requested size of x1182 // and the end of the slot. Unpoison the requested allocation.1183 asanpoison(unsafe.Add(x, size-asanRZ), asanRZ)1184 asanunpoison(x, size-asanRZ)1185 }1186 if valgrindenabled {1187 valgrindMalloc(x, size-asanRZ)1188 }11891190 // Adjust our GC assist debt to account for internal fragmentation.1191 if gcBlackenEnabled != 0 && elemsize != 0 {1192 if assistG := getg().m.curg; assistG != nil {1193 assistG.gcAssistBytes -= int64(elemsize - size)1194 }1195 }11961197 // Post-malloc debug hooks.1198 if debug.malloc {1199 postMallocgcDebug(x, elemsize, typ)1200 }1201 return x1202}12031204func mallocgcTiny(size uintptr, typ *_type) (unsafe.Pointer, uintptr) {1205 // Set mp.mallocing to keep from being preempted by GC.1206 mp := acquirem()1207 if doubleCheckMalloc {1208 if mp.mallocing != 0 {1209 throw("malloc deadlock")1210 }1211 if mp.gsignal == getg() {1212 throw("malloc during signal")1213 }1214 if typ != nil && typ.Pointers() {1215 throw("expected noscan for tiny alloc")1216 }1217 }1218 mp.mallocing = 112191220 // Tiny allocator.1221 //1222 // Tiny allocator combines several tiny allocation requests1223 // into a single memory block. The resulting memory block1224 // is freed when all subobjects are unreachable. The subobjects1225 // must be noscan (don't have pointers), this ensures that1226 // the amount of potentially wasted memory is bounded.1227 //1228 // Size of the memory block used for combining (maxTinySize) is tunable.1229 // Current setting is 16 bytes, which relates to 2x worst case memory1230 // wastage (when all but one subobjects are unreachable).1231 // 8 bytes would result in no wastage at all, but provides less1232 // opportunities for combining.1233 // 32 bytes provides more opportunities for combining,1234 // but can lead to 4x worst case wastage.1235 // The best case winning is 8x regardless of block size.1236 //1237 // Objects obtained from tiny allocator must not be freed explicitly.1238 // So when an object will be freed explicitly, we ensure that1239 // its size >= maxTinySize.1240 //1241 // SetFinalizer has a special case for objects potentially coming1242 // from tiny allocator, it such case it allows to set finalizers1243 // for an inner byte of a memory block.1244 //1245 // The main targets of tiny allocator are small strings and1246 // standalone escaping variables. On a json benchmark1247 // the allocator reduces number of allocations by ~12% and1248 // reduces heap size by ~20%.1249 c := getMCache(mp)1250 off := c.tinyoffset1251 // Align tiny pointer for required (conservative) alignment.1252 if size&7 == 0 {1253 off = alignUp(off, 8)1254 } else if goarch.PtrSize == 4 && size == 12 {1255 // Conservatively align 12-byte objects to 8 bytes on 32-bit1256 // systems so that objects whose first field is a 64-bit1257 // value is aligned to 8 bytes and does not cause a fault on1258 // atomic access. See issue 37262.1259 // TODO(mknyszek): Remove this workaround if/when issue 366061260 // is resolved.1261 off = alignUp(off, 8)1262 } else if size&3 == 0 {1263 off = alignUp(off, 4)1264 } else if size&1 == 0 {1265 off = alignUp(off, 2)1266 }1267 if off+size <= maxTinySize && c.tiny != 0 {1268 // The object fits into existing tiny block.1269 x := unsafe.Pointer(c.tiny + off)1270 c.tinyoffset = off + size1271 c.tinyAllocs++1272 mp.mallocing = 01273 releasem(mp)1274 return x, 01275 }1276 // Allocate a new maxTinySize block.1277 checkGCTrigger := false1278 span := c.alloc[tinySpanClass]1279 v := nextFreeFast(span)1280 if v == 0 {1281 v, span, checkGCTrigger = c.nextFree(tinySpanClass)1282 }1283 x := unsafe.Pointer(v)1284 (*[2]uint64)(x)[0] = 0 // Always zero1285 (*[2]uint64)(x)[1] = 01286 // See if we need to replace the existing tiny block with the new one1287 // based on amount of remaining free space.1288 if !raceenabled && (size < c.tinyoffset || c.tiny == 0) {1289 // Note: disabled when race detector is on, see comment near end of this function.1290 c.tiny = uintptr(x)1291 c.tinyoffset = size1292 }12931294 // Ensure that the stores above that initialize x to1295 // type-safe memory and set the heap bits occur before1296 // the caller can make x observable to the garbage1297 // collector. Otherwise, on weakly ordered machines,1298 // the garbage collector could follow a pointer to x,1299 // but see uninitialized memory or stale heap bits.1300 publicationBarrier()13011302 if writeBarrier.enabled {1303 // Allocate black during GC.1304 // All slots hold nil so no scanning is needed.1305 // This may be racing with GC so do it atomically if there can be1306 // a race marking the bit.1307 gcmarknewobject(span, uintptr(x))1308 } else {1309 // Track the last free index before the mark phase. This field1310 // is only used by the garbage collector. During the mark phase1311 // this is used by the conservative scanner to filter out objects1312 // that are both free and recently-allocated. It's safe to do that1313 // because we allocate-black if the GC is enabled. The conservative1314 // scanner produces pointers out of thin air, so without additional1315 // synchronization it might otherwise observe a partially-initialized1316 // object, which could crash the program.1317 span.freeIndexForScan = span.freeindex1318 }13191320 // Note cache c only valid while m acquired; see #473021321 //1322 // N.B. Use the full size because that matches how the GC1323 // will update the mem profile on the "free" side.1324 //1325 // TODO(mknyszek): We should really count the header as part1326 // of gc_sys or something. The code below just pretends it is1327 // internal fragmentation and matches the GC's accounting by1328 // using the whole allocation slot.1329 c.nextSample -= int64(span.elemsize)1330 if c.nextSample < 0 || MemProfileRate != c.memProfRate {1331 profilealloc(mp, x, span.elemsize)1332 }1333 mp.mallocing = 01334 releasem(mp)13351336 if checkGCTrigger {1337 if t := (gcTrigger{kind: gcTriggerHeap}); t.test() {1338 gcStart(t)1339 }1340 }13411342 if raceenabled {1343 // Pad tinysize allocations so they are aligned with the end1344 // of the tinyalloc region. This ensures that any arithmetic1345 // that goes off the top end of the object will be detectable1346 // by checkptr (issue 38872).1347 // Note that we disable tinyalloc when raceenabled for this to work.1348 // TODO: This padding is only performed when the race detector1349 // is enabled. It would be nice to enable it if any package1350 // was compiled with checkptr, but there's no easy way to1351 // detect that (especially at compile time).1352 // TODO: enable this padding for all allocations, not just1353 // tinyalloc ones. It's tricky because of pointer maps.1354 // Maybe just all noscan objects?1355 x = add(x, span.elemsize-size)1356 }1357 return x, span.elemsize1358}13591360func mallocgcSmallNoscan(size uintptr, typ *_type, needzero bool) (unsafe.Pointer, uintptr) {1361 // Set mp.mallocing to keep from being preempted by GC.1362 mp := acquirem()1363 if doubleCheckMalloc {1364 if mp.mallocing != 0 {1365 throw("malloc deadlock")1366 }1367 if mp.gsignal == getg() {1368 throw("malloc during signal")1369 }1370 if typ != nil && typ.Pointers() {1371 throw("expected noscan type for noscan alloc")1372 }1373 }1374 mp.mallocing = 113751376 checkGCTrigger := false1377 c := getMCache(mp)1378 var sizeclass uint81379 if size <= gc.SmallSizeMax-8 {1380 sizeclass = gc.SizeToSizeClass8[divRoundUp(size, gc.SmallSizeDiv)]1381 } else {1382 sizeclass = gc.SizeToSizeClass128[divRoundUp(size-gc.SmallSizeMax, gc.LargeSizeDiv)]1383 }1384 size = uintptr(gc.SizeClassToSize[sizeclass])1385 spc := makeSpanClass(sizeclass, true)1386 span := c.alloc[spc]13871388 // First, check for a reusable object.1389 if runtimeFreegcEnabled && c.hasReusableNoscan(spc) {1390 // We have a reusable object, use it.1391 x := mallocgcSmallNoscanReuse(c, span, spc, size, needzero)1392 mp.mallocing = 01393 releasem(mp)1394 return x, size1395 }13961397 v := nextFreeFast(span)1398 if v == 0 {1399 v, span, checkGCTrigger = c.nextFree(spc)1400 }1401 x := unsafe.Pointer(v)1402 if needzero && span.needzero != 0 {1403 memclrNoHeapPointers(x, size)1404 }14051406 // Ensure that the stores above that initialize x to1407 // type-safe memory and set the heap bits occur before1408 // the caller can make x observable to the garbage1409 // collector. Otherwise, on weakly ordered machines,1410 // the garbage collector could follow a pointer to x,1411 // but see uninitialized memory or stale heap bits.1412 publicationBarrier()14131414 if writeBarrier.enabled {1415 // Allocate black during GC.1416 // All slots hold nil so no scanning is needed.1417 // This may be racing with GC so do it atomically if there can be1418 // a race marking the bit.1419 gcmarknewobject(span, uintptr(x))1420 } else {1421 // Track the last free index before the mark phase. This field1422 // is only used by the garbage collector. During the mark phase1423 // this is used by the conservative scanner to filter out objects1424 // that are both free and recently-allocated. It's safe to do that1425 // because we allocate-black if the GC is enabled. The conservative1426 // scanner produces pointers out of thin air, so without additional1427 // synchronization it might otherwise observe a partially-initialized1428 // object, which could crash the program.1429 span.freeIndexForScan = span.freeindex1430 }14311432 // Note cache c only valid while m acquired; see #473021433 //1434 // N.B. Use the full size because that matches how the GC1435 // will update the mem profile on the "free" side.1436 //1437 // TODO(mknyszek): We should really count the header as part1438 // of gc_sys or something. The code below just pretends it is1439 // internal fragmentation and matches the GC's accounting by1440 // using the whole allocation slot.1441 c.nextSample -= int64(size)1442 if c.nextSample < 0 || MemProfileRate != c.memProfRate {1443 profilealloc(mp, x, size)1444 }1445 mp.mallocing = 01446 releasem(mp)14471448 if checkGCTrigger {1449 if t := (gcTrigger{kind: gcTriggerHeap}); t.test() {1450 gcStart(t)1451 }1452 }1453 return x, size1454}14551456// mallocgcSmallNoscanReuse returns a previously freed noscan object after preparing it for reuse.1457// It must only be called if hasReusableNoscan returned true.1458func mallocgcSmallNoscanReuse(c *mcache, span *mspan, spc spanClass, size uintptr, needzero bool) unsafe.Pointer {1459 // TODO(thepudds): could nextFreeFast, nextFree and nextReusable return unsafe.Pointer?1460 // Maybe doesn't matter. gclinkptr might be for historical reasons.1461 v, span := c.nextReusableNoScan(span, spc)1462 x := unsafe.Pointer(v)14631464 // Compensate for the GC assist credit deducted in mallocgc (before calling us and1465 // after we return) because this is not a newly allocated object. We use the full slot1466 // size (elemsize) here because that's what mallocgc deducts overall. Note we only1467 // adjust this when gcBlackenEnabled is true, which follows mallocgc behavior.1468 // TODO(thepudds): a follow-up CL adds a more specific test of our assist credit1469 // handling, including for validating internal fragmentation handling.1470 if gcBlackenEnabled != 0 {1471 addAssistCredit(size)1472 }14731474 // This is a previously used object, so only check needzero (and not span.needzero)1475 // for clearing.1476 if needzero {1477 memclrNoHeapPointers(x, size)1478 }14791480 // See publicationBarrier comment in mallocgcSmallNoscan.1481 publicationBarrier()14821483 // Finish and return. Note that we do not update span.freeIndexForScan, profiling info,1484 // nor do we check gcTrigger.1485 // TODO(thepudds): the current approach is viable for a GOEXPERIMENT, but1486 // means we do not profile reused heap objects. Ultimately, we will need a better1487 // approach for profiling, or at least ensure we are not introducing bias in the1488 // profiled allocations.1489 // TODO(thepudds): related, we probably want to adjust how allocs and frees are counted1490 // in the existing stats. Currently, reused objects are not counted as allocs nor1491 // frees, but instead roughly appear as if the original heap object lived on. We1492 // probably will also want some additional runtime/metrics, and generally think about1493 // user-facing observability & diagnostics, though all this likely can wait for an1494 // official proposal.1495 if writeBarrier.enabled {1496 // Allocate black during GC.1497 // All slots hold nil so no scanning is needed.1498 // This may be racing with GC so do it atomically if there can be1499 // a race marking the bit.1500 gcmarknewobject(span, uintptr(x))1501 }1502 return x1503}15041505func mallocgcSmallScanNoHeader(size uintptr, typ *_type) (unsafe.Pointer, uintptr) {1506 // Set mp.mallocing to keep from being preempted by GC.1507 mp := acquirem()1508 if doubleCheckMalloc {1509 if mp.mallocing != 0 {1510 throw("malloc deadlock")1511 }1512 if mp.gsignal == getg() {1513 throw("malloc during signal")1514 }1515 if typ == nil || !typ.Pointers() {1516 throw("noscan allocated in scan-only path")1517 }1518 if !heapBitsInSpan(size) {1519 throw("heap bits in not in span for non-header-only path")1520 }1521 }1522 mp.mallocing = 115231524 checkGCTrigger := false1525 c := getMCache(mp)1526 sizeclass := gc.SizeToSizeClass8[divRoundUp(size, gc.SmallSizeDiv)]1527 spc := makeSpanClass(sizeclass, false)1528 span := c.alloc[spc]1529 v := nextFreeFast(span)1530 if v == 0 {1531 v, span, checkGCTrigger = c.nextFree(spc)1532 }1533 x := unsafe.Pointer(v)1534 if span.needzero != 0 {1535 memclrNoHeapPointers(x, size)1536 }1537 if goarch.PtrSize == 8 && sizeclass == 1 {1538 // initHeapBits already set the pointer bits for the 8-byte sizeclass1539 // on 64-bit platforms.1540 c.scanAlloc += 81541 } else {1542 c.scanAlloc += heapSetTypeNoHeader(uintptr(x), size, typ, span)1543 }1544 size = uintptr(gc.SizeClassToSize[sizeclass])15451546 // Ensure that the stores above that initialize x to1547 // type-safe memory and set the heap bits occur before1548 // the caller can make x observable to the garbage1549 // collector. Otherwise, on weakly ordered machines,1550 // the garbage collector could follow a pointer to x,1551 // but see uninitialized memory or stale heap bits.1552 publicationBarrier()15531554 if writeBarrier.enabled {1555 // Allocate black during GC.1556 // All slots hold nil so no scanning is needed.1557 // This may be racing with GC so do it atomically if there can be1558 // a race marking the bit.1559 gcmarknewobject(span, uintptr(x))1560 } else {1561 // Track the last free index before the mark phase. This field1562 // is only used by the garbage collector. During the mark phase1563 // this is used by the conservative scanner to filter out objects1564 // that are both free and recently-allocated. It's safe to do that1565 // because we allocate-black if the GC is enabled. The conservative1566 // scanner produces pointers out of thin air, so without additional1567 // synchronization it might otherwise observe a partially-initialized1568 // object, which could crash the program.1569 span.freeIndexForScan = span.freeindex1570 }15711572 // Note cache c only valid while m acquired; see #473021573 //1574 // N.B. Use the full size because that matches how the GC1575 // will update the mem profile on the "free" side.1576 //1577 // TODO(mknyszek): We should really count the header as part1578 // of gc_sys or something. The code below just pretends it is1579 // internal fragmentation and matches the GC's accounting by1580 // using the whole allocation slot.1581 c.nextSample -= int64(size)1582 if c.nextSample < 0 || MemProfileRate != c.memProfRate {1583 profilealloc(mp, x, size)1584 }1585 mp.mallocing = 01586 releasem(mp)15871588 if checkGCTrigger {1589 if t := (gcTrigger{kind: gcTriggerHeap}); t.test() {1590 gcStart(t)1591 }1592 }1593 return x, size1594}15951596func mallocgcSmallScanHeader(size uintptr, typ *_type) (unsafe.Pointer, uintptr) {1597 // Set mp.mallocing to keep from being preempted by GC.1598 mp := acquirem()1599 if doubleCheckMalloc {1600 if mp.mallocing != 0 {1601 throw("malloc deadlock")1602 }1603 if mp.gsignal == getg() {1604 throw("malloc during signal")1605 }1606 if typ == nil || !typ.Pointers() {1607 throw("noscan allocated in scan-only path")1608 }1609 if heapBitsInSpan(size) {1610 throw("heap bits in span for header-only path")1611 }1612 }1613 mp.mallocing = 116141615 checkGCTrigger := false1616 c := getMCache(mp)1617 size += gc.MallocHeaderSize1618 var sizeclass uint81619 if size <= gc.SmallSizeMax-8 {1620 sizeclass = gc.SizeToSizeClass8[divRoundUp(size, gc.SmallSizeDiv)]1621 } else {1622 sizeclass = gc.SizeToSizeClass128[divRoundUp(size-gc.SmallSizeMax, gc.LargeSizeDiv)]1623 }1624 size = uintptr(gc.SizeClassToSize[sizeclass])1625 spc := makeSpanClass(sizeclass, false)1626 span := c.alloc[spc]1627 v := nextFreeFast(span)1628 if v == 0 {1629 v, span, checkGCTrigger = c.nextFree(spc)1630 }1631 x := unsafe.Pointer(v)1632 if span.needzero != 0 {1633 memclrNoHeapPointers(x, size)1634 }1635 header := (**_type)(x)1636 x = add(x, gc.MallocHeaderSize)1637 c.scanAlloc += heapSetTypeSmallHeader(uintptr(x), size-gc.MallocHeaderSize, typ, header, span)16381639 // Ensure that the stores above that initialize x to1640 // type-safe memory and set the heap bits occur before1641 // the caller can make x observable to the garbage1642 // collector. Otherwise, on weakly ordered machines,1643 // the garbage collector could follow a pointer to x,1644 // but see uninitialized memory or stale heap bits.1645 publicationBarrier()16461647 if writeBarrier.enabled {1648 // Allocate black during GC.1649 // All slots hold nil so no scanning is needed.1650 // This may be racing with GC so do it atomically if there can be1651 // a race marking the bit.1652 gcmarknewobject(span, uintptr(x))1653 } else {1654 // Track the last free index before the mark phase. This field1655 // is only used by the garbage collector. During the mark phase1656 // this is used by the conservative scanner to filter out objects1657 // that are both free and recently-allocated. It's safe to do that1658 // because we allocate-black if the GC is enabled. The conservative1659 // scanner produces pointers out of thin air, so without additional1660 // synchronization it might otherwise observe a partially-initialized1661 // object, which could crash the program.1662 span.freeIndexForScan = span.freeindex1663 }16641665 // Note cache c only valid while m acquired; see #473021666 //1667 // N.B. Use the full size because that matches how the GC1668 // will update the mem profile on the "free" side.1669 //1670 // TODO(mknyszek): We should really count the header as part1671 // of gc_sys or something. The code below just pretends it is1672 // internal fragmentation and matches the GC's accounting by1673 // using the whole allocation slot.1674 c.nextSample -= int64(size)1675 if c.nextSample < 0 || MemProfileRate != c.memProfRate {1676 profilealloc(mp, x, size)1677 }1678 mp.mallocing = 01679 releasem(mp)16801681 if checkGCTrigger {1682 if t := (gcTrigger{kind: gcTriggerHeap}); t.test() {1683 gcStart(t)1684 }1685 }1686 return x, size1687}16881689func mallocgcLarge(size uintptr, typ *_type, needzero bool) (unsafe.Pointer, uintptr) {1690 // Set mp.mallocing to keep from being preempted by GC.1691 mp := acquirem()1692 if doubleCheckMalloc {1693 if mp.mallocing != 0 {1694 throw("malloc deadlock")1695 }1696 if mp.gsignal == getg() {1697 throw("malloc during signal")1698 }1699 }1700 mp.mallocing = 117011702 c := getMCache(mp)1703 // For large allocations, keep track of zeroed state so that1704 // bulk zeroing can be happen later in a preemptible context.1705 span := c.allocLarge(size, typ == nil || !typ.Pointers())1706 span.freeindex = 11707 span.allocCount = 11708 span.largeType = nil // Tell the GC not to look at this yet.1709 size = span.elemsize1710 x := unsafe.Pointer(span.base())17111712 // Ensure that the store above that sets largeType to1713 // nil happens before the caller can make x observable1714 // to the garbage collector.1715 //1716 // Otherwise, on weakly ordered machines, the garbage1717 // collector could follow a pointer to x, but see a stale1718 // largeType value.1719 publicationBarrier()17201721 if writeBarrier.enabled {1722 // Allocate black during GC.1723 // All slots hold nil so no scanning is needed.1724 // This may be racing with GC so do it atomically if there can be1725 // a race marking the bit.1726 gcmarknewobject(span, uintptr(x))1727 } else {1728 // Track the last free index before the mark phase. This field1729 // is only used by the garbage collector. During the mark phase1730 // this is used by the conservative scanner to filter out objects1731 // that are both free and recently-allocated. It's safe to do that1732 // because we allocate-black if the GC is enabled. The conservative1733 // scanner produces pointers out of thin air, so without additional1734 // synchronization it might otherwise observe a partially-initialized1735 // object, which could crash the program.1736 span.freeIndexForScan = span.freeindex1737 }17381739 // Note cache c only valid while m acquired; see #473021740 //1741 // N.B. Use the full size because that matches how the GC1742 // will update the mem profile on the "free" side.1743 //1744 // TODO(mknyszek): We should really count the header as part1745 // of gc_sys or something. The code below just pretends it is1746 // internal fragmentation and matches the GC's accounting by1747 // using the whole allocation slot.1748 c.nextSample -= int64(size)1749 if c.nextSample < 0 || MemProfileRate != c.memProfRate {1750 profilealloc(mp, x, size)1751 }1752 mp.mallocing = 01753 releasem(mp)17541755 // Check to see if we need to trigger the GC.1756 if t := (gcTrigger{kind: gcTriggerHeap}); t.test() {1757 gcStart(t)1758 }17591760 // Objects can be zeroed late in a context where preemption can occur.1761 //1762 // x will keep the memory alive.1763 if needzero && span.needzero != 0 {1764 // N.B. size == fullSize always in this case.1765 memclrNoHeapPointersChunked(size, x) // This is a possible preemption point: see #473021766 }17671768 // Set the type and run the publication barrier while non-preemptible. We need to make1769 // sure that between heapSetTypeLarge and publicationBarrier we cannot get preempted,1770 // otherwise the GC could potentially observe non-zeroed memory but largeType set on weak1771 // memory architectures.1772 //1773 // The GC can also potentially observe non-zeroed memory if conservative scanning spuriously1774 // observes a partially-allocated object, see the freeIndexForScan update above. This case is1775 // handled by synchronization inside heapSetTypeLarge.1776 mp = acquirem()1777 if typ != nil && typ.Pointers() {1778 // Finish storing the type information, now that we're certain the memory is zeroed.1779 getMCache(mp).scanAlloc += heapSetTypeLarge(uintptr(x), size, typ, span)1780 }1781 // Publish the object again, now with zeroed memory and initialized type information.1782 //1783 // Even if we didn't update any type information, this is necessary to ensure that, for example,1784 // x written to a global without any synchronization still results in other goroutines observing1785 // zeroed memory.1786 publicationBarrier()1787 releasem(mp)1788 return x, size1789}17901791func preMallocgcDebug(size uintptr, typ *_type) unsafe.Pointer {1792 if debug.sbrk != 0 {1793 align := uintptr(16)1794 if typ != nil {1795 // TODO(austin): This should be just1796 // align = uintptr(typ.align)1797 // but that's only 4 on 32-bit platforms,1798 // even if there's a uint64 field in typ (see #599).1799 // This causes 64-bit atomic accesses to panic.1800 // Hence, we use stricter alignment that matches1801 // the normal allocator better.1802 if size&7 == 0 {1803 align = 81804 } else if size&3 == 0 {1805 align = 41806 } else if size&1 == 0 {1807 align = 21808 } else {1809 align = 11810 }1811 }1812 return persistentalloc(size, align, &memstats.other_sys)1813 }1814 if inittrace.active && inittrace.id == getg().goid {1815 // Init functions are executed sequentially in a single goroutine.1816 inittrace.allocs += 11817 }1818 return nil1819}18201821func postMallocgcDebug(x unsafe.Pointer, elemsize uintptr, typ *_type) {1822 if inittrace.active && inittrace.id == getg().goid {1823 // Init functions are executed sequentially in a single goroutine.1824 inittrace.bytes += uint64(elemsize)1825 }18261827 if traceAllocFreeEnabled() {1828 trace := traceAcquire()1829 if trace.ok() {1830 trace.HeapObjectAlloc(uintptr(x), typ)1831 traceRelease(trace)1832 }1833 }18341835 // N.B. elemsize == 0 indicates a tiny allocation, since no new slot was1836 // allocated to fulfill this call to mallocgc. This means checkfinalizer1837 // will only flag an error if there is actually any risk. If an allocation1838 // has the tiny block to itself, it will not get flagged, because we won't1839 // mark the block as a tiny block.1840 if debug.checkfinalizers != 0 && elemsize == 0 {1841 setTinyBlockContext(unsafe.Pointer(alignDown(uintptr(x), maxTinySize)))1842 }1843}18441845// addAssistCredit is like deductAssistCredit,1846// but adds credit rather than removes,1847// and never calls gcAssistAlloc.1848func addAssistCredit(size uintptr) {1849 // Credit the current user G.1850 assistG := getg()1851 if assistG.m.curg != nil { // TODO(thepudds): do we need to do this?1852 assistG = assistG.m.curg1853 }1854 // Credit the size against the G.1855 assistG.gcAssistBytes += int64(size)1856}18571858const (1859 // doubleCheckReusable enables some additional invariant checks for the1860 // runtime.freegc and reusable objects. Note that some of these checks alter timing,1861 // and it is good to test changes with and without this enabled.1862 doubleCheckReusable = false18631864 // debugReusableLog enables some printlns for runtime.freegc and reusable objects.1865 debugReusableLog = false1866)18671868// freegc records that a heap object is reusable and available for1869// immediate reuse in a subsequent mallocgc allocation, without1870// needing to wait for the GC cycle to progress.1871//1872// The information is recorded in a free list stored in the1873// current P's mcache. The caller must pass in the user size1874// and whether the object has pointers, which allows a faster free1875// operation.1876//1877// freegc must be called by the effective owner of ptr who knows1878// the pointer is logically dead, with no possible aliases that might1879// be used past that moment. In other words, ptr must be the1880// last and only pointer to its referent.1881//1882// The intended caller is the compiler.1883//1884// Note: please do not send changes that attempt to add freegc calls1885// to the standard library.1886//1887// ptr must point to a heap object or into the current g's stack,1888// in which case freegc is a no-op. In particular, ptr must not point1889// to memory in the data or bss sections, which is partially enforced.1890// For objects with a malloc header, ptr should point mallocHeaderSize bytes1891// past the base; otherwise, ptr should point to the base of the heap object.1892// In other words, ptr should be the same pointer that was returned by mallocgc.1893//1894// In addition, the caller must know that ptr's object has no specials, such1895// as might have been created by a call to SetFinalizer or AddCleanup.1896// (Internally, the runtime deals appropriately with internally-created1897// specials, such as specials for memory profiling).1898//1899// If the size of ptr's object is less than 16 bytes or greater than1900// 32KiB - gc.MallocHeaderSize bytes, freegc is currently a no-op. It must only1901// be called in alloc-safe places. It currently throws if noscan is false1902// (support for which is implemented in a later CL in our stack).1903//1904// Note that freegc accepts an unsafe.Pointer and hence keeps the pointer1905// alive. It therefore could be a pessimization in some cases (such1906// as a long-lived function) if the caller does not call freegc before1907// or roughly when the liveness analysis of the compiler1908// would otherwise have determined ptr's object is reclaimable by the GC.1909func freegc(ptr unsafe.Pointer, size uintptr, noscan bool) bool {1910 if !runtimeFreegcEnabled || !reusableSize(size) {1911 return false1912 }1913 if sizeSpecializedMallocEnabled && !noscan {1914 // TODO(thepudds): temporarily disable freegc with SizeSpecializedMalloc for pointer types1915 // until we finish integrating.1916 return false1917 }19181919 if ptr == nil {1920 throw("freegc nil")1921 }19221923 // Set mp.mallocing to keep from being preempted by GC.1924 // Otherwise, the GC could flush our mcache or otherwise cause problems.1925 mp := acquirem()1926 if mp.mallocing != 0 {1927 throw("freegc deadlock")1928 }1929 if mp.gsignal == getg() {1930 throw("freegc during signal")1931 }1932 mp.mallocing = 119331934 if mp.curg.stack.lo <= uintptr(ptr) && uintptr(ptr) < mp.curg.stack.hi {1935 // This points into our stack, so free is a no-op.1936 mp.mallocing = 01937 releasem(mp)1938 return false1939 }19401941 if doubleCheckReusable {1942 // TODO(thepudds): we could enforce no free on globals in bss or data. Maybe by1943 // checking span via spanOf or spanOfHeap, or maybe walk from firstmoduledata1944 // like isGoPointerWithoutSpan, or activeModules, or something. If so, we might1945 // be able to delay checking until reuse (e.g., check span just before reusing,1946 // though currently we don't always need to lookup a span on reuse). If we think1947 // no usage patterns could result in globals, maybe enforcement for globals could1948 // be behind -d=checkptr=1 or similar. The compiler can have knowledge of where1949 // a variable is allocated, but stdlib does not, although there are certain1950 // usage patterns that cannot result in a global.1951 // TODO(thepudds): separately, consider a local debugReusableMcacheOnly here1952 // to ignore freed objects if not in mspan in mcache, maybe when freeing and reading,1953 // by checking something like s.base() <= uintptr(v) && uintptr(v) < s.limit. Or1954 // maybe a GODEBUG or compiler debug flag.1955 span := spanOf(uintptr(ptr))1956 if span == nil {1957 throw("nextReusable: nil span for pointer in free list")1958 }1959 if state := span.state.get(); state != mSpanInUse {1960 throw("nextReusable: span is not in use")1961 }1962 }19631964 if debug.clobberfree != 0 {1965 clobberfree(ptr, size)1966 }19671968 // We first check if p is still in our per-P cache.1969 // Get our per-P cache for small objects.1970 c := getMCache(mp)1971 if c == nil {1972 throw("freegc called without a P or outside bootstrapping")1973 }19741975 v := uintptr(ptr)1976 if !noscan && !heapBitsInSpan(size) {1977 // mallocgcSmallScanHeader expects to get the base address of the object back1978 // from the findReusable funcs (as well as from nextFreeFast and nextFree), and1979 // not mallocHeaderSize bytes into a object, so adjust that here.1980 v -= mallocHeaderSize19811982 // The size class lookup wants size to be adjusted by mallocHeaderSize.1983 size += mallocHeaderSize1984 }19851986 // TODO(thepudds): should verify (behind doubleCheckReusable constant) that our calculated1987 // sizeclass here matches what's in span found via spanOf(ptr) or findObject(ptr).1988 var sizeclass uint81989 if size <= gc.SmallSizeMax-8 {1990 sizeclass = gc.SizeToSizeClass8[divRoundUp(size, gc.SmallSizeDiv)]1991 } else {1992 sizeclass = gc.SizeToSizeClass128[divRoundUp(size-gc.SmallSizeMax, gc.LargeSizeDiv)]1993 }19941995 spc := makeSpanClass(sizeclass, noscan)1996 s := c.alloc[spc]19971998 if debugReusableLog {1999 if s.base() <= uintptr(v) && uintptr(v) < s.limit {2000 println("freegc [in mcache]:", hex(uintptr(v)), "sweepgen:", mheap_.sweepgen, "writeBarrier.enabled:", writeBarrier.enabled)
Findings
✓ No findings reported for this file.