1// Code generated by mkmalloc.go; DO NOT EDIT.2// See overview in malloc_stubs.go.34package runtime56import (7	"internal/goarch"8	"internal/goexperiment"9	"internal/runtime/sys"10	"unsafe"11)1213func mallocgcSmallScanNoHeaderSC1(size uintptr, typ *_type, needzero bool) unsafe.Pointer {14	if doubleCheckMalloc {15		if gcphase == _GCmarktermination {16			throw("mallocgc called with gcphase == _GCmarktermination")17		}18	}1920	var mp *m2122	mp = acquirem()2324	forceSlowPath := debug.malloc || gcBlackenEnabled != 0 || (goexperiment.RuntimeSecret && getg().secret > 0)2526	if forceSlowPath {27		releasem(mp)2829		const spc = spanClass(1<<1) | spanClass(0)30		const elemsize = uintptr(8)31		return mallocgcSmallScanSlowPath(size, typ, needzero, spc, elemsize)3233	}3435	lockRankMayQueueFinalizer()3637	const sizeclass = 138	const elemsize = 83940	if doubleCheckMalloc {4142		doubleCheckSmallScanNoHeader(size, typ, mp)4344	}45	mp.mallocing = 14647	checkGCTrigger := false48	c := getMCache(mp)49	const spc = spanClass(sizeclass<<1) | spanClass(0)50	span := c.alloc[spc]5152	var v gclinkptr53	var x unsafe.Pointer5455	{5657		var nextFreeFastResult gclinkptr58		if span.allocCache != 0 {59			theBit := sys.TrailingZeros64(span.allocCache)60			result := span.freeindex + uint16(theBit)61			if result < span.nelems {62				freeidx := result + 163				if !(freeidx%64 == 0 && freeidx != span.nelems) {64					span.allocCache >>= uint(theBit + 1)65					span.freeindex = freeidx66					span.allocCount++67					nextFreeFastResult = gclinkptr(uintptr(result)*elemsize + span.base())68				}69			}70		}71		v = nextFreeFastResult72		if v == 0 {73			v, span, checkGCTrigger = c.nextFree(spc)74		}75		x = unsafe.Pointer(v)76	}7778	if span.needzero != 0 {79		memclrNoHeapPointers(x, elemsize)80	}81	if goarch.PtrSize == 8 && elemsize == 8 {8283		c.scanAlloc += 884	} else {85		dataSize := size86		x := uintptr(x)8788		if doubleCheckHeapSetType && (!heapBitsInSpan(dataSize) || !heapBitsInSpan(8)) {89			throw("tried to write heap bits, but no heap bits in span")90		}9192		src0 := readUintptr(getGCMask(typ))9394		const elemsize = 89596		var scanSize uintptr97		src := src098		if typ.Size_ == goarch.PtrSize {99			src = (1 << (dataSize / goarch.PtrSize)) - 1100101			scanSize = dataSize102		} else {103104			if doubleCheckHeapSetType && !asanenabled && dataSize%typ.Size_ != 0 {105				throw("runtime: (*mspan).writeHeapBitsSmall: dataSize is not a multiple of typ.Size_")106			}107			scanSize = typ.PtrBytes108			for i := typ.Size_; i < dataSize; i += typ.Size_ {109				src |= src0 << (i / goarch.PtrSize)110				scanSize += typ.Size_111			}112		}113114		dstBase, _ := spanHeapBitsRange(span.base(), pageSize, elemsize)115		dst := unsafe.Pointer(dstBase)116		o := (x - span.base()) / goarch.PtrSize117		i := o / ptrBits118		j := o % ptrBits119		var bits uintptr = elemsize / goarch.PtrSize120121		var bitsIsPowerOfTwo = bits&(bits-1) == 0122		if bits > ptrBits || (!bitsIsPowerOfTwo && j+bits > ptrBits) {123124			bits0 := ptrBits - j125			bits1 := bits - bits0126			dst0 := (*uintptr)(add(dst, (i+0)*goarch.PtrSize))127			dst1 := (*uintptr)(add(dst, (i+1)*goarch.PtrSize))128			*dst0 = (*dst0)&(^uintptr(0)>>bits0) | (src << j)129			*dst1 = (*dst1)&^((1<<bits1)-1) | (src >> bits0)130		} else {131132			dst := (*uintptr)(add(dst, i*goarch.PtrSize))133			*dst = (*dst)&^(((1<<(min(bits, ptrBits)))-1)<<j) | (src << j)134		}135136		const doubleCheck = false137		if doubleCheck {138			writeHeapBitsDoubleCheck(span, x, dataSize, src, src0, i, j, bits, typ)139		}140		if doubleCheckHeapSetType {141			doubleCheckHeapType(x, dataSize, typ, nil, span)142		}143		c.scanAlloc += scanSize144	}145146	publicationBarrier()147148	span.freeIndexForScan = span.freeindex149150	c.nextSample -= int64(elemsize)151	if c.nextSample < 0 || MemProfileRate != c.memProfRate {152		profilealloc(mp, x, elemsize)153	}154	mp.mallocing = 0155	releasem(mp)156157	if checkGCTrigger {158		if t := (gcTrigger{kind: gcTriggerHeap}); t.test() {159			gcStart(t)160		}161	}162163	return x164}165166func mallocgcSmallScanNoHeaderSC2(size uintptr, typ *_type, needzero bool) unsafe.Pointer {167	if doubleCheckMalloc {168		if gcphase == _GCmarktermination {169			throw("mallocgc called with gcphase == _GCmarktermination")170		}171	}172173	var mp *m174175	mp = acquirem()176177	forceSlowPath := debug.malloc || gcBlackenEnabled != 0 || (goexperiment.RuntimeSecret && getg().secret > 0)178179	if forceSlowPath {180		releasem(mp)181182		const spc = spanClass(2<<1) | spanClass(0)183		const elemsize = uintptr(16)184		return mallocgcSmallScanSlowPath(size, typ, needzero, spc, elemsize)185186	}187188	lockRankMayQueueFinalizer()189190	const sizeclass = 2191	const elemsize = 16192193	if doubleCheckMalloc {194195		doubleCheckSmallScanNoHeader(size, typ, mp)196197	}198	mp.mallocing = 1199200	checkGCTrigger := false201	c := getMCache(mp)202	const spc = spanClass(sizeclass<<1) | spanClass(0)203	span := c.alloc[spc]204205	var v gclinkptr206	var x unsafe.Pointer207208	{209210		var nextFreeFastResult gclinkptr211		if span.allocCache != 0 {212			theBit := sys.TrailingZeros64(span.allocCache)213			result := span.freeindex + uint16(theBit)214			if result < span.nelems {215				freeidx := result + 1216				if !(freeidx%64 == 0 && freeidx != span.nelems) {217					span.allocCache >>= uint(theBit + 1)218					span.freeindex = freeidx219					span.allocCount++220					nextFreeFastResult = gclinkptr(uintptr(result)*elemsize + span.base())221				}222			}223		}224		v = nextFreeFastResult225		if v == 0 {226			v, span, checkGCTrigger = c.nextFree(spc)227		}228		x = unsafe.Pointer(v)229	}230231	if span.needzero != 0 {232		memclrNoHeapPointers(x, elemsize)233	}234	if goarch.PtrSize == 8 && elemsize == 8 {235236		c.scanAlloc += 8237	} else {238		dataSize := size239		x := uintptr(x)240241		if doubleCheckHeapSetType && (!heapBitsInSpan(dataSize) || !heapBitsInSpan(16)) {242			throw("tried to write heap bits, but no heap bits in span")243		}244245		src0 := readUintptr(getGCMask(typ))246247		const elemsize = 16248249		var scanSize uintptr250		src := src0251		if typ.Size_ == goarch.PtrSize {252			src = (1 << (dataSize / goarch.PtrSize)) - 1253254			scanSize = dataSize255		} else {256257			if doubleCheckHeapSetType && !asanenabled && dataSize%typ.Size_ != 0 {258				throw("runtime: (*mspan).writeHeapBitsSmall: dataSize is not a multiple of typ.Size_")259			}260			scanSize = typ.PtrBytes261			for i := typ.Size_; i < dataSize; i += typ.Size_ {262				src |= src0 << (i / goarch.PtrSize)263				scanSize += typ.Size_264			}265		}266267		dstBase, _ := spanHeapBitsRange(span.base(), pageSize, elemsize)268		dst := unsafe.Pointer(dstBase)269		o := (x - span.base()) / goarch.PtrSize270		i := o / ptrBits271		j := o % ptrBits272		var bits uintptr = elemsize / goarch.PtrSize273274		var bitsIsPowerOfTwo = bits&(bits-1) == 0275		if bits > ptrBits || (!bitsIsPowerOfTwo && j+bits > ptrBits) {276277			bits0 := ptrBits - j278			bits1 := bits - bits0279			dst0 := (*uintptr)(add(dst, (i+0)*goarch.PtrSize))280			dst1 := (*uintptr)(add(dst, (i+1)*goarch.PtrSize))281			*dst0 = (*dst0)&(^uintptr(0)>>bits0) | (src << j)282			*dst1 = (*dst1)&^((1<<bits1)-1) | (src >> bits0)283		} else {284285			dst := (*uintptr)(add(dst, i*goarch.PtrSize))286			*dst = (*dst)&^(((1<<(min(bits, ptrBits)))-1)<<j) | (src << j)287		}288289		const doubleCheck = false290		if doubleCheck {291			writeHeapBitsDoubleCheck(span, x, dataSize, src, src0, i, j, bits, typ)292		}293		if doubleCheckHeapSetType {294			doubleCheckHeapType(x, dataSize, typ, nil, span)295		}296		c.scanAlloc += scanSize297	}298299	publicationBarrier()300301	span.freeIndexForScan = span.freeindex302303	c.nextSample -= int64(elemsize)304	if c.nextSample < 0 || MemProfileRate != c.memProfRate {305		profilealloc(mp, x, elemsize)306	}307	mp.mallocing = 0308	releasem(mp)309310	if checkGCTrigger {311		if t := (gcTrigger{kind: gcTriggerHeap}); t.test() {312			gcStart(t)313		}314	}315316	return x317}318319func mallocgcSmallScanNoHeaderSC3(size uintptr, typ *_type, needzero bool) unsafe.Pointer {320	if doubleCheckMalloc {321		if gcphase == _GCmarktermination {322			throw("mallocgc called with gcphase == _GCmarktermination")323		}324	}325326	var mp *m327328	mp = acquirem()329330	forceSlowPath := debug.malloc || gcBlackenEnabled != 0 || (goexperiment.RuntimeSecret && getg().secret > 0)331332	if forceSlowPath {333		releasem(mp)334335		const spc = spanClass(3<<1) | spanClass(0)336		const elemsize = uintptr(24)337		return mallocgcSmallScanSlowPath(size, typ, needzero, spc, elemsize)338339	}340341	lockRankMayQueueFinalizer()342343	const sizeclass = 3344	const elemsize = 24345346	if doubleCheckMalloc {347348		doubleCheckSmallScanNoHeader(size, typ, mp)349350	}351	mp.mallocing = 1352353	checkGCTrigger := false354	c := getMCache(mp)355	const spc = spanClass(sizeclass<<1) | spanClass(0)356	span := c.alloc[spc]357358	var v gclinkptr359	var x unsafe.Pointer360361	{362363		var nextFreeFastResult gclinkptr364		if span.allocCache != 0 {365			theBit := sys.TrailingZeros64(span.allocCache)366			result := span.freeindex + uint16(theBit)367			if result < span.nelems {368				freeidx := result + 1369				if !(freeidx%64 == 0 && freeidx != span.nelems) {370					span.allocCache >>= uint(theBit + 1)371					span.freeindex = freeidx372					span.allocCount++373					nextFreeFastResult = gclinkptr(uintptr(result)*elemsize + span.base())374				}375			}376		}377		v = nextFreeFastResult378		if v == 0 {379			v, span, checkGCTrigger = c.nextFree(spc)380		}381		x = unsafe.Pointer(v)382	}383384	if span.needzero != 0 {385		memclrNoHeapPointers(x, elemsize)386	}387	if goarch.PtrSize == 8 && elemsize == 8 {388389		c.scanAlloc += 8390	} else {391		dataSize := size392		x := uintptr(x)393394		if doubleCheckHeapSetType && (!heapBitsInSpan(dataSize) || !heapBitsInSpan(24)) {395			throw("tried to write heap bits, but no heap bits in span")396		}397398		src0 := readUintptr(getGCMask(typ))399400		const elemsize = 24401402		var scanSize uintptr403		src := src0404		if typ.Size_ == goarch.PtrSize {405			src = (1 << (dataSize / goarch.PtrSize)) - 1406407			scanSize = dataSize408		} else {409410			if doubleCheckHeapSetType && !asanenabled && dataSize%typ.Size_ != 0 {411				throw("runtime: (*mspan).writeHeapBitsSmall: dataSize is not a multiple of typ.Size_")412			}413			scanSize = typ.PtrBytes414			for i := typ.Size_; i < dataSize; i += typ.Size_ {415				src |= src0 << (i / goarch.PtrSize)416				scanSize += typ.Size_417			}418		}419420		dstBase, _ := spanHeapBitsRange(span.base(), pageSize, elemsize)421		dst := unsafe.Pointer(dstBase)422		o := (x - span.base()) / goarch.PtrSize423		i := o / ptrBits424		j := o % ptrBits425		var bits uintptr = elemsize / goarch.PtrSize426427		var bitsIsPowerOfTwo = bits&(bits-1) == 0428		if bits > ptrBits || (!bitsIsPowerOfTwo && j+bits > ptrBits) {429430			bits0 := ptrBits - j431			bits1 := bits - bits0432			dst0 := (*uintptr)(add(dst, (i+0)*goarch.PtrSize))433			dst1 := (*uintptr)(add(dst, (i+1)*goarch.PtrSize))434			*dst0 = (*dst0)&(^uintptr(0)>>bits0) | (src << j)435			*dst1 = (*dst1)&^((1<<bits1)-1) | (src >> bits0)436		} else {437438			dst := (*uintptr)(add(dst, i*goarch.PtrSize))439			*dst = (*dst)&^(((1<<(min(bits, ptrBits)))-1)<<j) | (src << j)440		}441442		const doubleCheck = false443		if doubleCheck {444			writeHeapBitsDoubleCheck(span, x, dataSize, src, src0, i, j, bits, typ)445		}446		if doubleCheckHeapSetType {447			doubleCheckHeapType(x, dataSize, typ, nil, span)448		}449		c.scanAlloc += scanSize450	}451452	publicationBarrier()453454	span.freeIndexForScan = span.freeindex455456	c.nextSample -= int64(elemsize)457	if c.nextSample < 0 || MemProfileRate != c.memProfRate {458		profilealloc(mp, x, elemsize)459	}460	mp.mallocing = 0461	releasem(mp)462463	if checkGCTrigger {464		if t := (gcTrigger{kind: gcTriggerHeap}); t.test() {465			gcStart(t)466		}467	}468469	return x470}471472func mallocgcSmallScanNoHeaderSC4(size uintptr, typ *_type, needzero bool) unsafe.Pointer {473	if doubleCheckMalloc {474		if gcphase == _GCmarktermination {475			throw("mallocgc called with gcphase == _GCmarktermination")476		}477	}478479	var mp *m480481	mp = acquirem()482483	forceSlowPath := debug.malloc || gcBlackenEnabled != 0 || (goexperiment.RuntimeSecret && getg().secret > 0)484485	if forceSlowPath {486		releasem(mp)487488		const spc = spanClass(4<<1) | spanClass(0)489		const elemsize = uintptr(32)490		return mallocgcSmallScanSlowPath(size, typ, needzero, spc, elemsize)491492	}493494	lockRankMayQueueFinalizer()495496	const sizeclass = 4497	const elemsize = 32498499	if doubleCheckMalloc {500501		doubleCheckSmallScanNoHeader(size, typ, mp)502503	}504	mp.mallocing = 1505506	checkGCTrigger := false507	c := getMCache(mp)508	const spc = spanClass(sizeclass<<1) | spanClass(0)509	span := c.alloc[spc]510511	var v gclinkptr512	var x unsafe.Pointer513514	{515516		var nextFreeFastResult gclinkptr517		if span.allocCache != 0 {518			theBit := sys.TrailingZeros64(span.allocCache)519			result := span.freeindex + uint16(theBit)520			if result < span.nelems {521				freeidx := result + 1522				if !(freeidx%64 == 0 && freeidx != span.nelems) {523					span.allocCache >>= uint(theBit + 1)524					span.freeindex = freeidx525					span.allocCount++526					nextFreeFastResult = gclinkptr(uintptr(result)*elemsize + span.base())527				}528			}529		}530		v = nextFreeFastResult531		if v == 0 {532			v, span, checkGCTrigger = c.nextFree(spc)533		}534		x = unsafe.Pointer(v)535	}536537	if span.needzero != 0 {538		memclrNoHeapPointers(x, elemsize)539	}540	if goarch.PtrSize == 8 && elemsize == 8 {541542		c.scanAlloc += 8543	} else {544		dataSize := size545		x := uintptr(x)546547		if doubleCheckHeapSetType && (!heapBitsInSpan(dataSize) || !heapBitsInSpan(32)) {548			throw("tried to write heap bits, but no heap bits in span")549		}550551		src0 := readUintptr(getGCMask(typ))552553		const elemsize = 32554555		var scanSize uintptr556		src := src0557		if typ.Size_ == goarch.PtrSize {558			src = (1 << (dataSize / goarch.PtrSize)) - 1559560			scanSize = dataSize561		} else {562563			if doubleCheckHeapSetType && !asanenabled && dataSize%typ.Size_ != 0 {564				throw("runtime: (*mspan).writeHeapBitsSmall: dataSize is not a multiple of typ.Size_")565			}566			scanSize = typ.PtrBytes567			for i := typ.Size_; i < dataSize; i += typ.Size_ {568				src |= src0 << (i / goarch.PtrSize)569				scanSize += typ.Size_570			}571		}572573		dstBase, _ := spanHeapBitsRange(span.base(), pageSize, elemsize)574		dst := unsafe.Pointer(dstBase)575		o := (x - span.base()) / goarch.PtrSize576		i := o / ptrBits577		j := o % ptrBits578		var bits uintptr = elemsize / goarch.PtrSize579580		var bitsIsPowerOfTwo = bits&(bits-1) == 0581		if bits > ptrBits || (!bitsIsPowerOfTwo && j+bits > ptrBits) {582583			bits0 := ptrBits - j584			bits1 := bits - bits0585			dst0 := (*uintptr)(add(dst, (i+0)*goarch.PtrSize))586			dst1 := (*uintptr)(add(dst, (i+1)*goarch.PtrSize))587			*dst0 = (*dst0)&(^uintptr(0)>>bits0) | (src << j)588			*dst1 = (*dst1)&^((1<<bits1)-1) | (src >> bits0)589		} else {590591			dst := (*uintptr)(add(dst, i*goarch.PtrSize))592			*dst = (*dst)&^(((1<<(min(bits, ptrBits)))-1)<<j) | (src << j)593		}594595		const doubleCheck = false596		if doubleCheck {597			writeHeapBitsDoubleCheck(span, x, dataSize, src, src0, i, j, bits, typ)598		}599		if doubleCheckHeapSetType {600			doubleCheckHeapType(x, dataSize, typ, nil, span)601		}602		c.scanAlloc += scanSize603	}604605	publicationBarrier()606607	span.freeIndexForScan = span.freeindex608609	c.nextSample -= int64(elemsize)610	if c.nextSample < 0 || MemProfileRate != c.memProfRate {611		profilealloc(mp, x, elemsize)612	}613	mp.mallocing = 0614	releasem(mp)615616	if checkGCTrigger {617		if t := (gcTrigger{kind: gcTriggerHeap}); t.test() {618			gcStart(t)619		}620	}621622	return x623}624625func mallocgcSmallScanNoHeaderSC5(size uintptr, typ *_type, needzero bool) unsafe.Pointer {626	if doubleCheckMalloc {627		if gcphase == _GCmarktermination {628			throw("mallocgc called with gcphase == _GCmarktermination")629		}630	}631632	var mp *m633634	mp = acquirem()635636	forceSlowPath := debug.malloc || gcBlackenEnabled != 0 || (goexperiment.RuntimeSecret && getg().secret > 0)637638	if forceSlowPath {639		releasem(mp)640641		const spc = spanClass(5<<1) | spanClass(0)642		const elemsize = uintptr(48)643		return mallocgcSmallScanSlowPath(size, typ, needzero, spc, elemsize)644645	}646647	lockRankMayQueueFinalizer()648649	const sizeclass = 5650	const elemsize = 48651652	if doubleCheckMalloc {653654		doubleCheckSmallScanNoHeader(size, typ, mp)655656	}657	mp.mallocing = 1658659	checkGCTrigger := false660	c := getMCache(mp)661	const spc = spanClass(sizeclass<<1) | spanClass(0)662	span := c.alloc[spc]663664	var v gclinkptr665	var x unsafe.Pointer666667	{668669		var nextFreeFastResult gclinkptr670		if span.allocCache != 0 {671			theBit := sys.TrailingZeros64(span.allocCache)672			result := span.freeindex + uint16(theBit)673			if result < span.nelems {674				freeidx := result + 1675				if !(freeidx%64 == 0 && freeidx != span.nelems) {676					span.allocCache >>= uint(theBit + 1)677					span.freeindex = freeidx678					span.allocCount++679					nextFreeFastResult = gclinkptr(uintptr(result)*elemsize + span.base())680				}681			}682		}683		v = nextFreeFastResult684		if v == 0 {685			v, span, checkGCTrigger = c.nextFree(spc)686		}687		x = unsafe.Pointer(v)688	}689690	if span.needzero != 0 {691		memclrNoHeapPointers(x, elemsize)692	}693	if goarch.PtrSize == 8 && elemsize == 8 {694695		c.scanAlloc += 8696	} else {697		dataSize := size698		x := uintptr(x)699700		if doubleCheckHeapSetType && (!heapBitsInSpan(dataSize) || !heapBitsInSpan(48)) {701			throw("tried to write heap bits, but no heap bits in span")702		}703704		src0 := readUintptr(getGCMask(typ))705706		const elemsize = 48707708		var scanSize uintptr709		src := src0710		if typ.Size_ == goarch.PtrSize {711			src = (1 << (dataSize / goarch.PtrSize)) - 1712713			scanSize = dataSize714		} else {715716			if doubleCheckHeapSetType && !asanenabled && dataSize%typ.Size_ != 0 {717				throw("runtime: (*mspan).writeHeapBitsSmall: dataSize is not a multiple of typ.Size_")718			}719			scanSize = typ.PtrBytes720			for i := typ.Size_; i < dataSize; i += typ.Size_ {721				src |= src0 << (i / goarch.PtrSize)722				scanSize += typ.Size_723			}724		}725726		dstBase, _ := spanHeapBitsRange(span.base(), pageSize, elemsize)727		dst := unsafe.Pointer(dstBase)728		o := (x - span.base()) / goarch.PtrSize729		i := o / ptrBits730		j := o % ptrBits731		var bits uintptr = elemsize / goarch.PtrSize732733		var bitsIsPowerOfTwo = bits&(bits-1) == 0734		if bits > ptrBits || (!bitsIsPowerOfTwo && j+bits > ptrBits) {735736			bits0 := ptrBits - j737			bits1 := bits - bits0738			dst0 := (*uintptr)(add(dst, (i+0)*goarch.PtrSize))739			dst1 := (*uintptr)(add(dst, (i+1)*goarch.PtrSize))740			*dst0 = (*dst0)&(^uintptr(0)>>bits0) | (src << j)741			*dst1 = (*dst1)&^((1<<bits1)-1) | (src >> bits0)742		} else {743744			dst := (*uintptr)(add(dst, i*goarch.PtrSize))745			*dst = (*dst)&^(((1<<(min(bits, ptrBits)))-1)<<j) | (src << j)746		}747748		const doubleCheck = false749		if doubleCheck {750			writeHeapBitsDoubleCheck(span, x, dataSize, src, src0, i, j, bits, typ)751		}752		if doubleCheckHeapSetType {753			doubleCheckHeapType(x, dataSize, typ, nil, span)754		}755		c.scanAlloc += scanSize756	}757758	publicationBarrier()759760	span.freeIndexForScan = span.freeindex761762	c.nextSample -= int64(elemsize)763	if c.nextSample < 0 || MemProfileRate != c.memProfRate {764		profilealloc(mp, x, elemsize)765	}766	mp.mallocing = 0767	releasem(mp)768769	if checkGCTrigger {770		if t := (gcTrigger{kind: gcTriggerHeap}); t.test() {771			gcStart(t)772		}773	}774775	return x776}777778func mallocgcSmallScanNoHeaderSC6(size uintptr, typ *_type, needzero bool) unsafe.Pointer {779	if doubleCheckMalloc {780		if gcphase == _GCmarktermination {781			throw("mallocgc called with gcphase == _GCmarktermination")782		}783	}784785	var mp *m786787	mp = acquirem()788789	forceSlowPath := debug.malloc || gcBlackenEnabled != 0 || (goexperiment.RuntimeSecret && getg().secret > 0)790791	if forceSlowPath {792		releasem(mp)793794		const spc = spanClass(6<<1) | spanClass(0)795		const elemsize = uintptr(64)796		return mallocgcSmallScanSlowPath(size, typ, needzero, spc, elemsize)797798	}799800	lockRankMayQueueFinalizer()801802	const sizeclass = 6803	const elemsize = 64804805	if doubleCheckMalloc {806807		doubleCheckSmallScanNoHeader(size, typ, mp)808809	}810	mp.mallocing = 1811812	checkGCTrigger := false813	c := getMCache(mp)814	const spc = spanClass(sizeclass<<1) | spanClass(0)815	span := c.alloc[spc]816817	var v gclinkptr818	var x unsafe.Pointer819820	{821822		var nextFreeFastResult gclinkptr823		if span.allocCache != 0 {824			theBit := sys.TrailingZeros64(span.allocCache)825			result := span.freeindex + uint16(theBit)826			if result < span.nelems {827				freeidx := result + 1828				if !(freeidx%64 == 0 && freeidx != span.nelems) {829					span.allocCache >>= uint(theBit + 1)830					span.freeindex = freeidx831					span.allocCount++832					nextFreeFastResult = gclinkptr(uintptr(result)*elemsize + span.base())833				}834			}835		}836		v = nextFreeFastResult837		if v == 0 {838			v, span, checkGCTrigger = c.nextFree(spc)839		}840		x = unsafe.Pointer(v)841	}842843	if span.needzero != 0 {844		memclrNoHeapPointers(x, elemsize)845	}846	if goarch.PtrSize == 8 && elemsize == 8 {847848		c.scanAlloc += 8849	} else {850		dataSize := size851		x := uintptr(x)852853		if doubleCheckHeapSetType && (!heapBitsInSpan(dataSize) || !heapBitsInSpan(64)) {854			throw("tried to write heap bits, but no heap bits in span")855		}856857		src0 := readUintptr(getGCMask(typ))858859		const elemsize = 64860861		var scanSize uintptr862		src := src0863		if typ.Size_ == goarch.PtrSize {864			src = (1 << (dataSize / goarch.PtrSize)) - 1865866			scanSize = dataSize867		} else {868869			if doubleCheckHeapSetType && !asanenabled && dataSize%typ.Size_ != 0 {870				throw("runtime: (*mspan).writeHeapBitsSmall: dataSize is not a multiple of typ.Size_")871			}872			scanSize = typ.PtrBytes873			for i := typ.Size_; i < dataSize; i += typ.Size_ {874				src |= src0 << (i / goarch.PtrSize)875				scanSize += typ.Size_876			}877		}878879		dstBase, _ := spanHeapBitsRange(span.base(), pageSize, elemsize)880		dst := unsafe.Pointer(dstBase)881		o := (x - span.base()) / goarch.PtrSize882		i := o / ptrBits883		j := o % ptrBits884		var bits uintptr = elemsize / goarch.PtrSize885886		var bitsIsPowerOfTwo = bits&(bits-1) == 0887		if bits > ptrBits || (!bitsIsPowerOfTwo && j+bits > ptrBits) {888889			bits0 := ptrBits - j890			bits1 := bits - bits0891			dst0 := (*uintptr)(add(dst, (i+0)*goarch.PtrSize))892			dst1 := (*uintptr)(add(dst, (i+1)*goarch.PtrSize))893			*dst0 = (*dst0)&(^uintptr(0)>>bits0) | (src << j)894			*dst1 = (*dst1)&^((1<<bits1)-1) | (src >> bits0)895		} else {896897			dst := (*uintptr)(add(dst, i*goarch.PtrSize))898			*dst = (*dst)&^(((1<<(min(bits, ptrBits)))-1)<<j) | (src << j)899		}900901		const doubleCheck = false902		if doubleCheck {903			writeHeapBitsDoubleCheck(span, x, dataSize, src, src0, i, j, bits, typ)904		}905		if doubleCheckHeapSetType {906			doubleCheckHeapType(x, dataSize, typ, nil, span)907		}908		c.scanAlloc += scanSize909	}910911	publicationBarrier()912913	span.freeIndexForScan = span.freeindex914915	c.nextSample -= int64(elemsize)916	if c.nextSample < 0 || MemProfileRate != c.memProfRate {917		profilealloc(mp, x, elemsize)918	}919	mp.mallocing = 0920	releasem(mp)921922	if checkGCTrigger {923		if t := (gcTrigger{kind: gcTriggerHeap}); t.test() {924			gcStart(t)925		}926	}927928	return x929}930931func mallocgcSmallScanNoHeaderSC7(size uintptr, typ *_type, needzero bool) unsafe.Pointer {932	if doubleCheckMalloc {933		if gcphase == _GCmarktermination {934			throw("mallocgc called with gcphase == _GCmarktermination")935		}936	}937938	var mp *m939940	mp = acquirem()941942	forceSlowPath := debug.malloc || gcBlackenEnabled != 0 || (goexperiment.RuntimeSecret && getg().secret > 0)943944	if forceSlowPath {945		releasem(mp)946947		const spc = spanClass(7<<1) | spanClass(0)948		const elemsize = uintptr(80)949		return mallocgcSmallScanSlowPath(size, typ, needzero, spc, elemsize)950951	}952953	lockRankMayQueueFinalizer()954955	const sizeclass = 7956	const elemsize = 80957958	if doubleCheckMalloc {959960		doubleCheckSmallScanNoHeader(size, typ, mp)961962	}963	mp.mallocing = 1964965	checkGCTrigger := false966	c := getMCache(mp)967	const spc = spanClass(sizeclass<<1) | spanClass(0)968	span := c.alloc[spc]969970	var v gclinkptr971	var x unsafe.Pointer972973	{974975		var nextFreeFastResult gclinkptr976		if span.allocCache != 0 {977			theBit := sys.TrailingZeros64(span.allocCache)978			result := span.freeindex + uint16(theBit)979			if result < span.nelems {980				freeidx := result + 1981				if !(freeidx%64 == 0 && freeidx != span.nelems) {982					span.allocCache >>= uint(theBit + 1)983					span.freeindex = freeidx984					span.allocCount++985					nextFreeFastResult = gclinkptr(uintptr(result)*elemsize + span.base())986				}987			}988		}989		v = nextFreeFastResult990		if v == 0 {991			v, span, checkGCTrigger = c.nextFree(spc)992		}993		x = unsafe.Pointer(v)994	}995996	if span.needzero != 0 {997		memclrNoHeapPointers(x, elemsize)998	}999	if goarch.PtrSize == 8 && elemsize == 8 {10001001		c.scanAlloc += 81002	} else {1003		dataSize := size1004		x := uintptr(x)10051006		if doubleCheckHeapSetType && (!heapBitsInSpan(dataSize) || !heapBitsInSpan(80)) {1007			throw("tried to write heap bits, but no heap bits in span")1008		}10091010		src0 := readUintptr(getGCMask(typ))10111012		const elemsize = 8010131014		var scanSize uintptr1015		src := src01016		if typ.Size_ == goarch.PtrSize {1017			src = (1 << (dataSize / goarch.PtrSize)) - 110181019			scanSize = dataSize1020		} else {10211022			if doubleCheckHeapSetType && !asanenabled && dataSize%typ.Size_ != 0 {1023				throw("runtime: (*mspan).writeHeapBitsSmall: dataSize is not a multiple of typ.Size_")1024			}1025			scanSize = typ.PtrBytes1026			for i := typ.Size_; i < dataSize; i += typ.Size_ {1027				src |= src0 << (i / goarch.PtrSize)1028				scanSize += typ.Size_1029			}1030		}10311032		dstBase, _ := spanHeapBitsRange(span.base(), pageSize, elemsize)1033		dst := unsafe.Pointer(dstBase)1034		o := (x - span.base()) / goarch.PtrSize1035		i := o / ptrBits1036		j := o % ptrBits1037		var bits uintptr = elemsize / goarch.PtrSize10381039		var bitsIsPowerOfTwo = bits&(bits-1) == 01040		if bits > ptrBits || (!bitsIsPowerOfTwo && j+bits > ptrBits) {10411042			bits0 := ptrBits - j1043			bits1 := bits - bits01044			dst0 := (*uintptr)(add(dst, (i+0)*goarch.PtrSize))1045			dst1 := (*uintptr)(add(dst, (i+1)*goarch.PtrSize))1046			*dst0 = (*dst0)&(^uintptr(0)>>bits0) | (src << j)1047			*dst1 = (*dst1)&^((1<<bits1)-1) | (src >> bits0)1048		} else {10491050			dst := (*uintptr)(add(dst, i*goarch.PtrSize))1051			*dst = (*dst)&^(((1<<(min(bits, ptrBits)))-1)<<j) | (src << j)1052		}10531054		const doubleCheck = false1055		if doubleCheck {1056			writeHeapBitsDoubleCheck(span, x, dataSize, src, src0, i, j, bits, typ)1057		}1058		if doubleCheckHeapSetType {1059			doubleCheckHeapType(x, dataSize, typ, nil, span)1060		}1061		c.scanAlloc += scanSize1062	}10631064	publicationBarrier()10651066	span.freeIndexForScan = span.freeindex10671068	c.nextSample -= int64(elemsize)1069	if c.nextSample < 0 || MemProfileRate != c.memProfRate {1070		profilealloc(mp, x, elemsize)1071	}1072	mp.mallocing = 01073	releasem(mp)10741075	if checkGCTrigger {1076		if t := (gcTrigger{kind: gcTriggerHeap}); t.test() {1077			gcStart(t)1078		}1079	}10801081	return x1082}10831084func mallocgcTinySC2(size uintptr, typ *_type, needzero bool) unsafe.Pointer {1085	if doubleCheckMalloc {1086		if gcphase == _GCmarktermination {1087			throw("mallocgc called with gcphase == _GCmarktermination")1088		}1089	}10901091	var mp *m10921093	mp = acquirem()10941095	forceSlowPath := debug.malloc || gcBlackenEnabled != 0 || (goexperiment.RuntimeSecret && getg().secret > 0)10961097	if forceSlowPath {1098		releasem(mp)10991100		return mallocgcTinySlowPath(size, typ, needzero)11011102	}11031104	lockRankMayQueueFinalizer()11051106	const elemsize = 1611071108	if doubleCheckMalloc {1109		doubleCheckTiny(size, typ, mp)1110	}1111	mp.mallocing = 111121113	c := getMCache(mp)1114	off := c.tinyoffset11151116	if size&7 == 0 {1117		off = alignUp(off, 8)1118	} else if goarch.PtrSize == 4 && size == 12 {11191120		off = alignUp(off, 8)1121	} else if size&3 == 0 {1122		off = alignUp(off, 4)1123	} else if size&1 == 0 {1124		off = alignUp(off, 2)1125	}1126	if off+size <= maxTinySize && c.tiny != 0 {11271128		x := unsafe.Pointer(c.tiny + off)1129		c.tinyoffset = off + size1130		c.tinyAllocs++1131		mp.mallocing = 01132		releasem(mp)1133		const elemsize = 011341135		return x1136	}11371138	checkGCTrigger := false1139	span := c.alloc[tinySpanClass]11401141	const nbytes = 81921142	const nelems = uint16((nbytes - unsafe.Sizeof(spanInlineMarkBits{})) / 16)1143	var nextFreeFastResult gclinkptr1144	if span.allocCache != 0 {1145		theBit := sys.TrailingZeros64(span.allocCache)1146		result := span.freeindex + uint16(theBit)1147		if result < nelems {1148			freeidx := result + 11149			if !(freeidx%64 == 0 && freeidx != nelems) {1150				span.allocCache >>= uint(theBit + 1)1151				span.freeindex = freeidx1152				span.allocCount++1153				nextFreeFastResult = gclinkptr(uintptr(result)*16 + span.base())1154			}1155		}1156	}1157	v := nextFreeFastResult1158	if v == 0 {1159		v, span, checkGCTrigger = c.nextFree(tinySpanClass)1160	}1161	x := unsafe.Pointer(v)1162	(*[2]uint64)(x)[0] = 01163	(*[2]uint64)(x)[1] = 011641165	if !raceenabled && (size < c.tinyoffset || c.tiny == 0) {11661167		c.tiny = uintptr(x)1168		c.tinyoffset = size1169	}11701171	publicationBarrier()11721173	span.freeIndexForScan = span.freeindex11741175	c.nextSample -= int64(elemsize)1176	if c.nextSample < 0 || MemProfileRate != c.memProfRate {1177		profilealloc(mp, x, elemsize)1178	}1179	mp.mallocing = 01180	releasem(mp)11811182	if checkGCTrigger {1183		if t := (gcTrigger{kind: gcTriggerHeap}); t.test() {1184			gcStart(t)1185		}1186	}11871188	return x1189}11901191func mallocgcSmallNoScanSC2(size uintptr, typ *_type, needzero bool) unsafe.Pointer {1192	if doubleCheckMalloc {1193		if gcphase == _GCmarktermination {1194			throw("mallocgc called with gcphase == _GCmarktermination")1195		}1196	}11971198	var mp *m11991200	mp = acquirem()12011202	forceSlowPath := debug.malloc || gcBlackenEnabled != 0 || (goexperiment.RuntimeSecret && getg().secret > 0)12031204	if forceSlowPath {1205		releasem(mp)12061207		const spc = spanClass(2<<1) | spanClass(1)1208		const elemsize = uintptr(16)1209		return mallocgcSmallNoScanSlowPath(size, typ, needzero, spc, elemsize)12101211	}12121213	lockRankMayQueueFinalizer()12141215	const sizeclass = 21216	const elemsize = 1612171218	if doubleCheckMalloc {12191220		doubleCheckSmallNoScan(typ, mp)12211222	}1223	mp.mallocing = 112241225	checkGCTrigger := false1226	c := getMCache(mp)1227	const spc = spanClass(sizeclass<<1) | spanClass(1)1228	span := c.alloc[spc]12291230	var v gclinkptr1231	var x unsafe.Pointer12321233	if runtimeFreegcEnabled && c.hasReusableNoscan(spc) {12341235		x = mallocgcSmallNoscanReuse(c, span, spc, elemsize, needzero)1236		mp.mallocing = 01237		releasem(mp)12381239		return x12401241	}12421243	{12441245		var nextFreeFastResult gclinkptr1246		if span.allocCache != 0 {1247			theBit := sys.TrailingZeros64(span.allocCache)1248			result := span.freeindex + uint16(theBit)1249			if result < span.nelems {1250				freeidx := result + 11251				if !(freeidx%64 == 0 && freeidx != span.nelems) {1252					span.allocCache >>= uint(theBit + 1)1253					span.freeindex = freeidx1254					span.allocCount++1255					nextFreeFastResult = gclinkptr(uintptr(result)*elemsize + span.base())1256				}1257			}1258		}1259		v = nextFreeFastResult1260		if v == 0 {1261			v, span, checkGCTrigger = c.nextFree(spc)1262		}1263		x = unsafe.Pointer(v)1264	}12651266	if needzero && span.needzero != 0 {1267		memclrNoHeapPointers(x, elemsize)1268	}12691270	publicationBarrier()12711272	span.freeIndexForScan = span.freeindex12731274	c.nextSample -= int64(elemsize)1275	if c.nextSample < 0 || MemProfileRate != c.memProfRate {1276		profilealloc(mp, x, elemsize)1277	}1278	mp.mallocing = 01279	releasem(mp)12801281	if checkGCTrigger {1282		if t := (gcTrigger{kind: gcTriggerHeap}); t.test() {1283			gcStart(t)1284		}1285	}12861287	return x1288}12891290func mallocgcSmallNoScanSC3(size uintptr, typ *_type, needzero bool) unsafe.Pointer {1291	if doubleCheckMalloc {1292		if gcphase == _GCmarktermination {1293			throw("mallocgc called with gcphase == _GCmarktermination")1294		}1295	}12961297	var mp *m12981299	mp = acquirem()13001301	forceSlowPath := debug.malloc || gcBlackenEnabled != 0 || (goexperiment.RuntimeSecret && getg().secret > 0)13021303	if forceSlowPath {1304		releasem(mp)13051306		const spc = spanClass(3<<1) | spanClass(1)1307		const elemsize = uintptr(24)1308		return mallocgcSmallNoScanSlowPath(size, typ, needzero, spc, elemsize)13091310	}13111312	lockRankMayQueueFinalizer()13131314	const sizeclass = 31315	const elemsize = 2413161317	if doubleCheckMalloc {13181319		doubleCheckSmallNoScan(typ, mp)13201321	}1322	mp.mallocing = 113231324	checkGCTrigger := false1325	c := getMCache(mp)1326	const spc = spanClass(sizeclass<<1) | spanClass(1)1327	span := c.alloc[spc]13281329	var v gclinkptr1330	var x unsafe.Pointer13311332	if runtimeFreegcEnabled && c.hasReusableNoscan(spc) {13331334		x = mallocgcSmallNoscanReuse(c, span, spc, elemsize, needzero)1335		mp.mallocing = 01336		releasem(mp)13371338		return x13391340	}13411342	{13431344		var nextFreeFastResult gclinkptr1345		if span.allocCache != 0 {1346			theBit := sys.TrailingZeros64(span.allocCache)1347			result := span.freeindex + uint16(theBit)1348			if result < span.nelems {1349				freeidx := result + 11350				if !(freeidx%64 == 0 && freeidx != span.nelems) {1351					span.allocCache >>= uint(theBit + 1)1352					span.freeindex = freeidx1353					span.allocCount++1354					nextFreeFastResult = gclinkptr(uintptr(result)*elemsize + span.base())1355				}1356			}1357		}1358		v = nextFreeFastResult1359		if v == 0 {1360			v, span, checkGCTrigger = c.nextFree(spc)1361		}1362		x = unsafe.Pointer(v)1363	}13641365	if needzero && span.needzero != 0 {1366		memclrNoHeapPointers(x, elemsize)1367	}13681369	publicationBarrier()13701371	span.freeIndexForScan = span.freeindex13721373	c.nextSample -= int64(elemsize)1374	if c.nextSample < 0 || MemProfileRate != c.memProfRate {1375		profilealloc(mp, x, elemsize)1376	}1377	mp.mallocing = 01378	releasem(mp)13791380	if checkGCTrigger {1381		if t := (gcTrigger{kind: gcTriggerHeap}); t.test() {1382			gcStart(t)1383		}1384	}13851386	return x1387}13881389func mallocgcSmallNoScanSC4(size uintptr, typ *_type, needzero bool) unsafe.Pointer {1390	if doubleCheckMalloc {1391		if gcphase == _GCmarktermination {1392			throw("mallocgc called with gcphase == _GCmarktermination")1393		}1394	}13951396	var mp *m13971398	mp = acquirem()13991400	forceSlowPath := debug.malloc || gcBlackenEnabled != 0 || (goexperiment.RuntimeSecret && getg().secret > 0)14011402	if forceSlowPath {1403		releasem(mp)14041405		const spc = spanClass(4<<1) | spanClass(1)1406		const elemsize = uintptr(32)1407		return mallocgcSmallNoScanSlowPath(size, typ, needzero, spc, elemsize)14081409	}14101411	lockRankMayQueueFinalizer()14121413	const sizeclass = 41414	const elemsize = 3214151416	if doubleCheckMalloc {14171418		doubleCheckSmallNoScan(typ, mp)14191420	}1421	mp.mallocing = 114221423	checkGCTrigger := false1424	c := getMCache(mp)1425	const spc = spanClass(sizeclass<<1) | spanClass(1)1426	span := c.alloc[spc]14271428	var v gclinkptr1429	var x unsafe.Pointer14301431	if runtimeFreegcEnabled && c.hasReusableNoscan(spc) {14321433		x = mallocgcSmallNoscanReuse(c, span, spc, elemsize, needzero)1434		mp.mallocing = 01435		releasem(mp)14361437		return x14381439	}14401441	{14421443		var nextFreeFastResult gclinkptr1444		if span.allocCache != 0 {1445			theBit := sys.TrailingZeros64(span.allocCache)1446			result := span.freeindex + uint16(theBit)1447			if result < span.nelems {1448				freeidx := result + 11449				if !(freeidx%64 == 0 && freeidx != span.nelems) {1450					span.allocCache >>= uint(theBit + 1)1451					span.freeindex = freeidx1452					span.allocCount++1453					nextFreeFastResult = gclinkptr(uintptr(result)*elemsize + span.base())1454				}1455			}1456		}1457		v = nextFreeFastResult1458		if v == 0 {1459			v, span, checkGCTrigger = c.nextFree(spc)1460		}1461		x = unsafe.Pointer(v)1462	}14631464	if needzero && span.needzero != 0 {1465		memclrNoHeapPointers(x, elemsize)1466	}14671468	publicationBarrier()14691470	span.freeIndexForScan = span.freeindex14711472	c.nextSample -= int64(elemsize)1473	if c.nextSample < 0 || MemProfileRate != c.memProfRate {1474		profilealloc(mp, x, elemsize)1475	}1476	mp.mallocing = 01477	releasem(mp)14781479	if checkGCTrigger {1480		if t := (gcTrigger{kind: gcTriggerHeap}); t.test() {1481			gcStart(t)1482		}1483	}14841485	return x1486}14871488func mallocgcSmallNoScanSC5(size uintptr, typ *_type, needzero bool) unsafe.Pointer {1489	if doubleCheckMalloc {1490		if gcphase == _GCmarktermination {1491			throw("mallocgc called with gcphase == _GCmarktermination")1492		}1493	}14941495	var mp *m14961497	mp = acquirem()14981499	forceSlowPath := debug.malloc || gcBlackenEnabled != 0 || (goexperiment.RuntimeSecret && getg().secret > 0)15001501	if forceSlowPath {1502		releasem(mp)15031504		const spc = spanClass(5<<1) | spanClass(1)1505		const elemsize = uintptr(48)1506		return mallocgcSmallNoScanSlowPath(size, typ, needzero, spc, elemsize)15071508	}15091510	lockRankMayQueueFinalizer()15111512	const sizeclass = 51513	const elemsize = 4815141515	if doubleCheckMalloc {15161517		doubleCheckSmallNoScan(typ, mp)15181519	}1520	mp.mallocing = 115211522	checkGCTrigger := false1523	c := getMCache(mp)1524	const spc = spanClass(sizeclass<<1) | spanClass(1)1525	span := c.alloc[spc]15261527	var v gclinkptr1528	var x unsafe.Pointer15291530	if runtimeFreegcEnabled && c.hasReusableNoscan(spc) {15311532		x = mallocgcSmallNoscanReuse(c, span, spc, elemsize, needzero)1533		mp.mallocing = 01534		releasem(mp)15351536		return x15371538	}15391540	{15411542		var nextFreeFastResult gclinkptr1543		if span.allocCache != 0 {1544			theBit := sys.TrailingZeros64(span.allocCache)1545			result := span.freeindex + uint16(theBit)1546			if result < span.nelems {1547				freeidx := result + 11548				if !(freeidx%64 == 0 && freeidx != span.nelems) {1549					span.allocCache >>= uint(theBit + 1)1550					span.freeindex = freeidx1551					span.allocCount++1552					nextFreeFastResult = gclinkptr(uintptr(result)*elemsize + span.base())1553				}1554			}1555		}1556		v = nextFreeFastResult1557		if v == 0 {1558			v, span, checkGCTrigger = c.nextFree(spc)1559		}1560		x = unsafe.Pointer(v)1561	}15621563	if needzero && span.needzero != 0 {1564		memclrNoHeapPointers(x, elemsize)1565	}15661567	publicationBarrier()15681569	span.freeIndexForScan = span.freeindex15701571	c.nextSample -= int64(elemsize)1572	if c.nextSample < 0 || MemProfileRate != c.memProfRate {1573		profilealloc(mp, x, elemsize)1574	}1575	mp.mallocing = 01576	releasem(mp)15771578	if checkGCTrigger {1579		if t := (gcTrigger{kind: gcTriggerHeap}); t.test() {1580			gcStart(t)1581		}1582	}15831584	return x1585}15861587func mallocgcSmallNoScanSC6(size uintptr, typ *_type, needzero bool) unsafe.Pointer {1588	if doubleCheckMalloc {1589		if gcphase == _GCmarktermination {1590			throw("mallocgc called with gcphase == _GCmarktermination")1591		}1592	}15931594	var mp *m15951596	mp = acquirem()15971598	forceSlowPath := debug.malloc || gcBlackenEnabled != 0 || (goexperiment.RuntimeSecret && getg().secret > 0)15991600	if forceSlowPath {1601		releasem(mp)16021603		const spc = spanClass(6<<1) | spanClass(1)1604		const elemsize = uintptr(64)1605		return mallocgcSmallNoScanSlowPath(size, typ, needzero, spc, elemsize)16061607	}16081609	lockRankMayQueueFinalizer()16101611	const sizeclass = 61612	const elemsize = 6416131614	if doubleCheckMalloc {16151616		doubleCheckSmallNoScan(typ, mp)16171618	}1619	mp.mallocing = 116201621	checkGCTrigger := false1622	c := getMCache(mp)1623	const spc = spanClass(sizeclass<<1) | spanClass(1)1624	span := c.alloc[spc]16251626	var v gclinkptr1627	var x unsafe.Pointer16281629	if runtimeFreegcEnabled && c.hasReusableNoscan(spc) {16301631		x = mallocgcSmallNoscanReuse(c, span, spc, elemsize, needzero)1632		mp.mallocing = 01633		releasem(mp)16341635		return x16361637	}16381639	{16401641		var nextFreeFastResult gclinkptr1642		if span.allocCache != 0 {1643			theBit := sys.TrailingZeros64(span.allocCache)1644			result := span.freeindex + uint16(theBit)1645			if result < span.nelems {1646				freeidx := result + 11647				if !(freeidx%64 == 0 && freeidx != span.nelems) {1648					span.allocCache >>= uint(theBit + 1)1649					span.freeindex = freeidx1650					span.allocCount++1651					nextFreeFastResult = gclinkptr(uintptr(result)*elemsize + span.base())1652				}1653			}1654		}1655		v = nextFreeFastResult1656		if v == 0 {1657			v, span, checkGCTrigger = c.nextFree(spc)1658		}1659		x = unsafe.Pointer(v)1660	}16611662	if needzero && span.needzero != 0 {1663		memclrNoHeapPointers(x, elemsize)1664	}16651666	publicationBarrier()16671668	span.freeIndexForScan = span.freeindex16691670	c.nextSample -= int64(elemsize)1671	if c.nextSample < 0 || MemProfileRate != c.memProfRate {1672		profilealloc(mp, x, elemsize)1673	}1674	mp.mallocing = 01675	releasem(mp)16761677	if checkGCTrigger {1678		if t := (gcTrigger{kind: gcTriggerHeap}); t.test() {1679			gcStart(t)1680		}1681	}16821683	return x1684}16851686func mallocgcSmallNoScanSC7(size uintptr, typ *_type, needzero bool) unsafe.Pointer {1687	if doubleCheckMalloc {1688		if gcphase == _GCmarktermination {1689			throw("mallocgc called with gcphase == _GCmarktermination")1690		}1691	}16921693	var mp *m16941695	mp = acquirem()16961697	forceSlowPath := debug.malloc || gcBlackenEnabled != 0 || (goexperiment.RuntimeSecret && getg().secret > 0)16981699	if forceSlowPath {1700		releasem(mp)17011702		const spc = spanClass(7<<1) | spanClass(1)1703		const elemsize = uintptr(80)1704		return mallocgcSmallNoScanSlowPath(size, typ, needzero, spc, elemsize)17051706	}17071708	lockRankMayQueueFinalizer()17091710	const sizeclass = 71711	const elemsize = 8017121713	if doubleCheckMalloc {17141715		doubleCheckSmallNoScan(typ, mp)17161717	}1718	mp.mallocing = 117191720	checkGCTrigger := false1721	c := getMCache(mp)1722	const spc = spanClass(sizeclass<<1) | spanClass(1)1723	span := c.alloc[spc]17241725	var v gclinkptr1726	var x unsafe.Pointer17271728	if runtimeFreegcEnabled && c.hasReusableNoscan(spc) {17291730		x = mallocgcSmallNoscanReuse(c, span, spc, elemsize, needzero)1731		mp.mallocing = 01732		releasem(mp)17331734		return x17351736	}17371738	{17391740		var nextFreeFastResult gclinkptr1741		if span.allocCache != 0 {1742			theBit := sys.TrailingZeros64(span.allocCache)1743			result := span.freeindex + uint16(theBit)1744			if result < span.nelems {1745				freeidx := result + 11746				if !(freeidx%64 == 0 && freeidx != span.nelems) {1747					span.allocCache >>= uint(theBit + 1)1748					span.freeindex = freeidx1749					span.allocCount++1750					nextFreeFastResult = gclinkptr(uintptr(result)*elemsize + span.base())1751				}1752			}1753		}1754		v = nextFreeFastResult1755		if v == 0 {1756			v, span, checkGCTrigger = c.nextFree(spc)1757		}1758		x = unsafe.Pointer(v)1759	}17601761	if needzero && span.needzero != 0 {1762		memclrNoHeapPointers(x, elemsize)1763	}17641765	publicationBarrier()17661767	span.freeIndexForScan = span.freeindex17681769	c.nextSample -= int64(elemsize)1770	if c.nextSample < 0 || MemProfileRate != c.memProfRate {1771		profilealloc(mp, x, elemsize)1772	}1773	mp.mallocing = 01774	releasem(mp)17751776	if checkGCTrigger {1777		if t := (gcTrigger{kind: gcTriggerHeap}); t.test() {1778			gcStart(t)1779		}1780	}17811782	return x1783}17841785func mallocgcTinySlowPath(size uintptr, typ *_type, needzero bool) unsafe.Pointer {1786	if doubleCheckMalloc {1787		if gcphase == _GCmarktermination {1788			throw("mallocgc called with gcphase == _GCmarktermination")1789		}1790	}17911792	var mp *m17931794	gp := getg()1795	if goexperiment.RuntimeSecret && gp.secret > 0 {1796		return mallocgcSmallNoScanSC2(size, typ, needzero)1797	}17981799	lockRankMayQueueFinalizer()18001801	if debug.malloc {1802		if x := preMallocgcDebug(size, typ); x != nil {1803			return x1804		}1805	}18061807	if gcBlackenEnabled != 0 {18081809		assistG := getg()1810		if assistG.m.curg != nil {1811			assistG = assistG.m.curg1812		}1813		assistG.gcAssistBytes -= int64(size)1814		if assistG.gcAssistBytes < 0 {1815			gcAssistAlloc(assistG)1816		}1817	}18181819	mp = acquirem()18201821	const elemsize = 1618221823	if doubleCheckMalloc {1824		doubleCheckTiny(size, typ, mp)1825	}1826	mp.mallocing = 118271828	c := getMCache(mp)1829	off := c.tinyoffset18301831	if size&7 == 0 {1832		off = alignUp(off, 8)1833	} else if goarch.PtrSize == 4 && size == 12 {18341835		off = alignUp(off, 8)1836	} else if size&3 == 0 {1837		off = alignUp(off, 4)1838	} else if size&1 == 0 {1839		off = alignUp(off, 2)1840	}1841	if off+size <= maxTinySize && c.tiny != 0 {18421843		x := unsafe.Pointer(c.tiny + off)1844		c.tinyoffset = off + size1845		c.tinyAllocs++1846		mp.mallocing = 01847		releasem(mp)1848		const elemsize = 01849		if gcBlackenEnabled != 0 && elemsize != 0 {1850			if assistG := getg().m.curg; assistG != nil {1851				assistG.gcAssistBytes -= int64(elemsize - size)1852			}1853		}18541855		if debug.malloc {1856			postMallocgcDebug(x, elemsize, typ)1857		}18581859		return x1860	}18611862	checkGCTrigger := false1863	span := c.alloc[tinySpanClass]18641865	const nbytes = 81921866	const nelems = uint16((nbytes - unsafe.Sizeof(spanInlineMarkBits{})) / 16)1867	var nextFreeFastResult gclinkptr1868	if span.allocCache != 0 {1869		theBit := sys.TrailingZeros64(span.allocCache)1870		result := span.freeindex + uint16(theBit)1871		if result < nelems {1872			freeidx := result + 11873			if !(freeidx%64 == 0 && freeidx != nelems) {1874				span.allocCache >>= uint(theBit + 1)1875				span.freeindex = freeidx1876				span.allocCount++1877				nextFreeFastResult = gclinkptr(uintptr(result)*16 + span.base())1878			}1879		}1880	}1881	v := nextFreeFastResult1882	if v == 0 {1883		v, span, checkGCTrigger = c.nextFree(tinySpanClass)1884	}1885	x := unsafe.Pointer(v)1886	(*[2]uint64)(x)[0] = 01887	(*[2]uint64)(x)[1] = 018881889	if !raceenabled && (size < c.tinyoffset || c.tiny == 0) {18901891		c.tiny = uintptr(x)1892		c.tinyoffset = size1893	}18941895	publicationBarrier()18961897	if writeBarrier.enabled {18981899		gcmarknewobject(span, uintptr(x))1900	} else {19011902		span.freeIndexForScan = span.freeindex1903	}19041905	c.nextSample -= int64(elemsize)1906	if c.nextSample < 0 || MemProfileRate != c.memProfRate {1907		profilealloc(mp, x, elemsize)1908	}1909	mp.mallocing = 01910	releasem(mp)19111912	if checkGCTrigger {1913		if t := (gcTrigger{kind: gcTriggerHeap}); t.test() {1914			gcStart(t)1915		}1916	}1917	if gcBlackenEnabled != 0 && elemsize != 0 {1918		if assistG := getg().m.curg; assistG != nil {1919			assistG.gcAssistBytes -= int64(elemsize - size)1920		}1921	}19221923	if debug.malloc {1924		postMallocgcDebug(x, elemsize, typ)1925	}19261927	return x1928}19291930func mallocgcSmallScanSlowPath(size uintptr, typ *_type, needzero bool, spc spanClass, elemsize uintptr) unsafe.Pointer {19311932	if doubleCheckMalloc {1933		if gcphase == _GCmarktermination {1934			throw("mallocgc called with gcphase == _GCmarktermination")1935		}1936	}19371938	var mp *m19391940	lockRankMayQueueFinalizer()19411942	if debug.malloc {1943		if x := preMallocgcDebug(size, typ); x != nil {1944			return x1945		}1946	}19471948	if gcBlackenEnabled != 0 {19491950		assistG := getg()1951		if assistG.m.curg != nil {1952			assistG = assistG.m.curg1953		}1954		assistG.gcAssistBytes -= int64(size)1955		if assistG.gcAssistBytes < 0 {1956			gcAssistAlloc(assistG)1957		}1958	}19591960	mp = acquirem()19611962	if doubleCheckMalloc {19631964		doubleCheckSmallScanNoHeader(size, typ, mp)19651966	}1967	mp.mallocing = 119681969	checkGCTrigger := false1970	c := getMCache(mp)19711972	span := c.alloc[spc]19731974	var v gclinkptr1975	var x unsafe.Pointer19761977	{19781979		var nextFreeFastResult gclinkptr1980		if span.allocCache != 0 {1981			theBit := sys.TrailingZeros64(span.allocCache)1982			result := span.freeindex + uint16(theBit)1983			if result < span.nelems {1984				freeidx := result + 11985				if !(freeidx%64 == 0 && freeidx != span.nelems) {1986					span.allocCache >>= uint(theBit + 1)1987					span.freeindex = freeidx1988					span.allocCount++1989					nextFreeFastResult = gclinkptr(uintptr(result)*elemsize + span.base())1990				}1991			}1992		}1993		v = nextFreeFastResult1994		if v == 0 {1995			v, span, checkGCTrigger = c.nextFree(spc)1996		}1997		x = unsafe.Pointer(v)1998	}19992000	if span.needzero != 0 {