1// Copyright 2014 The Go Authors. All rights reserved.2// Use of this source code is governed by a BSD-style3// license that can be found in the LICENSE file.45package runtime67import (8	"internal/abi"9	"internal/cpu"10	"internal/goarch"11	"internal/goexperiment"12	"internal/goos"13	"internal/runtime/atomic"14	"internal/runtime/exithook"15	"internal/runtime/maps"16	"internal/runtime/sys"17	"internal/strconv"18	"internal/stringslite"19	"unsafe"20)2122// set using cmd/go/internal/modload.ModInfoProg23var modinfo string2425// Goroutine scheduler26// The scheduler's job is to distribute ready-to-run goroutines over worker threads.27//28// The main concepts are:29// G - goroutine.30// M - worker thread, or machine.31// P - processor, a resource that is required to execute Go code.32//     M must have an associated P to execute Go code, however it can be33//     blocked or in a syscall w/o an associated P.34//35// Design doc at https://golang.org/s/go11sched.3637// Worker thread parking/unparking.38// We need to balance between keeping enough running worker threads to utilize39// available hardware parallelism and parking excessive running worker threads40// to conserve CPU resources and power. This is not simple for two reasons:41// (1) scheduler state is intentionally distributed (in particular, per-P work42// queues), so it is not possible to compute global predicates on fast paths;43// (2) for optimal thread management we would need to know the future (don't park44// a worker thread when a new goroutine will be readied in near future).45//46// Three rejected approaches that would work badly:47// 1. Centralize all scheduler state (would inhibit scalability).48// 2. Direct goroutine handoff. That is, when we ready a new goroutine and there49//    is a spare P, unpark a thread and handoff it the thread and the goroutine.50//    This would lead to thread state thrashing, as the thread that readied the51//    goroutine can be out of work the very next moment, we will need to park it.52//    Also, it would destroy locality of computation as we want to preserve53//    dependent goroutines on the same thread; and introduce additional latency.54// 3. Unpark an additional thread whenever we ready a goroutine and there is an55//    idle P, but don't do handoff. This would lead to excessive thread parking/56//    unparking as the additional threads will instantly park without discovering57//    any work to do.58//59// The current approach:60//61// This approach applies to three primary sources of potential work: readying a62// goroutine, new/modified-earlier timers, and idle-priority GC. See below for63// additional details.64//65// We unpark an additional thread when we submit work if (this is wakep()):66// 1. There is an idle P, and67// 2. There are no "spinning" worker threads.68//69// A worker thread is considered spinning if it is out of local work and did70// not find work in the global run queue or netpoller; the spinning state is71// denoted in m.spinning and in sched.nmspinning. Threads unparked this way are72// also considered spinning; we don't do goroutine handoff so such threads are73// out of work initially. Spinning threads spin on looking for work in per-P74// run queues and timer heaps or from the GC before parking. If a spinning75// thread finds work it takes itself out of the spinning state and proceeds to76// execution. If it does not find work it takes itself out of the spinning77// state and then parks.78//79// If there is at least one spinning thread (sched.nmspinning>1), we don't80// unpark new threads when submitting work. To compensate for that, if the last81// spinning thread finds work and stops spinning, it must unpark a new spinning82// thread. This approach smooths out unjustified spikes of thread unparking,83// but at the same time guarantees eventual maximal CPU parallelism84// utilization.85//86// The main implementation complication is that we need to be very careful87// during spinning->non-spinning thread transition. This transition can race88// with submission of new work, and either one part or another needs to unpark89// another worker thread. If they both fail to do that, we can end up with90// semi-persistent CPU underutilization.91//92// The general pattern for submission is:93// 1. Submit work to the local or global run queue, timer heap, or GC state.94// 2. #StoreLoad-style memory barrier.95// 3. Check sched.nmspinning.96//97// The general pattern for spinning->non-spinning transition is:98// 1. Decrement nmspinning.99// 2. #StoreLoad-style memory barrier.100// 3. Check all per-P work queues and GC for new work.101//102// Note that all this complexity does not apply to global run queue as we are103// not sloppy about thread unparking when submitting to global queue. Also see104// comments for nmspinning manipulation.105//106// How these different sources of work behave varies, though it doesn't affect107// the synchronization approach:108// * Ready goroutine: this is an obvious source of work; the goroutine is109//   immediately ready and must run on some thread eventually.110// * New/modified-earlier timer: The current timer implementation (see time.go)111//   uses netpoll in a thread with no work available to wait for the soonest112//   timer. If there is no thread waiting, we want a new spinning thread to go113//   wait.114// * Idle-priority GC: The GC wakes a stopped idle thread to contribute to115//   background GC work (note: currently disabled per golang.org/issue/19112).116//   Also see golang.org/issue/44313, as this should be extended to all GC117//   workers.118119var (120	m0           m121	g0           g122	mcache0      *mcache123	raceprocctx0 uintptr124	raceFiniLock mutex125)126127// This slice records the initializing tasks that need to be128// done to start up the runtime. It is built by the linker.129var runtime_inittasks []*initTask130131// mainInitDone is a signal used by cgocallbackg that initialization132// has been completed. If this is false, wait on mainInitDoneChan.133var mainInitDone atomic.Bool134135// mainInitDoneChan is closed after initialization has been completed.136// It is made before _cgo_notify_runtime_init_done, so all cgo137// calls can rely on it existing.138var mainInitDoneChan chan bool139140//go:linkname main_main main.main141func main_main()142143// mainStarted indicates that the main M has started.144var mainStarted bool145146// runtimeInitTime is the nanotime() at which the runtime started.147var runtimeInitTime int64148149// Value to use for signal mask for newly created M's.150var initSigmask sigset151152// The main goroutine.153func main() {154	mp := getg().m155156	// Racectx of m0->g0 is used only as the parent of the main goroutine.157	// It must not be used for anything else.158	mp.g0.racectx = 0159160	// Max stack size is 1 GB on 64-bit, 250 MB on 32-bit.161	// Using decimal instead of binary GB and MB because162	// they look nicer in the stack overflow failure message.163	if goarch.PtrSize == 8 {164		maxstacksize = 1000000000165	} else {166		maxstacksize = 250000000167	}168169	// An upper limit for max stack size. Used to avoid random crashes170	// after calling SetMaxStack and trying to allocate a stack that is too big,171	// since stackalloc works with 32-bit sizes.172	maxstackceiling = 2 * maxstacksize173174	// Allow newproc to start new Ms.175	mainStarted = true176177	if haveSysmon {178		systemstack(func() {179			newm(sysmon, nil, -1)180		})181	}182183	// Lock the main goroutine onto this, the main OS thread,184	// during initialization. Most programs won't care, but a few185	// do require certain calls to be made by the main thread.186	// Those can arrange for main.main to run in the main thread187	// by calling runtime.LockOSThread during initialization188	// to preserve the lock.189	lockOSThread()190191	if mp != &m0 {192		throw("runtime.main not on m0")193	}194195	// Record when the world started.196	// Must be before doInit for tracing init.197	runtimeInitTime = nanotime()198	if runtimeInitTime == 0 {199		throw("nanotime returning zero")200	}201202	if debug.inittrace != 0 {203		inittrace.id = getg().goid204		inittrace.active = true205	}206207	doInit(runtime_inittasks) // Must be before defer.208209	// Defer unlock so that runtime.Goexit during init does the unlock too.210	needUnlock := true211	defer func() {212		if needUnlock {213			unlockOSThread()214		}215	}()216217	gcenable()218	defaultGOMAXPROCSUpdateEnable() // don't STW before runtime initialized.219220	// If we encountered a removed GODEBUG during startup we can panic now.221	if k := invalidGODEBUG.key; k != "" {222		v := invalidGODEBUG.value223		r := strconv.Itoa(invalidGODEBUG.removed)224		fatal(`removed GODEBUG "` + k + `" set to old value "` + v + `" in environment (https://go.dev/doc/godebug#go-1` + r + `)`)225	}226227	mainInitDoneChan = make(chan bool)228	if iscgo {229		if _cgo_pthread_key_created == nil {230			throw("_cgo_pthread_key_created missing")231		}232233		if GOOS != "windows" {234			if _cgo_thread_start == nil {235				throw("_cgo_thread_start missing")236			}237			if _cgo_setenv == nil {238				throw("_cgo_setenv missing")239			}240			if _cgo_unsetenv == nil {241				throw("_cgo_unsetenv missing")242			}243		}244		if _cgo_notify_runtime_init_done == nil {245			throw("_cgo_notify_runtime_init_done missing")246		}247248		// Set the x_crosscall2_ptr C function pointer variable point to crosscall2.249		if set_crosscall2 == nil {250			throw("set_crosscall2 missing")251		}252		set_crosscall2()253254		// Start the template thread in case we enter Go from255		// a C-created thread and need to create a new thread.256		startTemplateThread()257		cgocall(_cgo_notify_runtime_init_done, nil)258	}259260	// Run the initializing tasks. Depending on build mode this261	// list can arrive a few different ways, but it will always262	// contain the init tasks computed by the linker for all the263	// packages in the program (excluding those added at runtime264	// by package plugin). Run through the modules in dependency265	// order (the order they are initialized by the dynamic266	// loader, i.e. they are added to the moduledata linked list).267	last := lastmoduledatap // grab before loop starts. Any added modules after this point will do their own doInit calls.268	for m := &firstmoduledata; true; m = m.next {269		doInit(m.inittasks)270		if m == last {271			break272		}273	}274275	// Disable init tracing after main init done to avoid overhead276	// of collecting statistics in malloc and newproc277	inittrace.active = false278279	mainInitDone.Store(true)280	close(mainInitDoneChan)281282	needUnlock = false283	unlockOSThread()284285	if isarchive || islibrary {286		// A program compiled with -buildmode=c-archive or c-shared287		// has a main, but it is not executed.288		if GOARCH == "wasm" {289			// On Wasm, pause makes it return to the host.290			// Unlike cgo callbacks where Ms are created on demand,291			// on Wasm we have only one M. So we keep this M (and this292			// G) for callbacks.293			// Using the caller's SP unwinds this frame and backs to294			// goexit. The -16 is: 8 for goexit's (fake) return PC,295			// and pause's epilogue pops 8.296			pause(sys.GetCallerSP() - 16) // should not return297			panic("unreachable")298		}299		return300	}301	fn := main_main // make an indirect call, as the linker doesn't know the address of the main package when laying down the runtime302	fn()303304	// Check for C memory leaks if using ASAN and we've made cgo calls,305	// or if we are running as a library in a C program.306	// We always make one cgo call, above, to notify_runtime_init_done,307	// so we ignore that one.308	// No point in leak checking if no cgo calls, since leak checking309	// just looks for objects allocated using malloc and friends.310	// Just checking iscgo doesn't help because asan implies iscgo.311	exitHooksRun := false312	if asanenabled && (isarchive || islibrary || NumCgoCall() > 1) {313		runExitHooks(0) // lsandoleakcheck may not return314		exitHooksRun = true315		lsandoleakcheck()316	}317318	// Make racy client program work: if panicking on319	// another goroutine at the same time as main returns,320	// let the other goroutine finish printing the panic trace.321	// Once it does, it will exit. See issues 3934 and 20018.322	if runningPanicDefers.Load() != 0 {323		// Running deferred functions should not take long.324		for c := 0; c < 1000; c++ {325			if runningPanicDefers.Load() == 0 {326				break327			}328			Gosched()329		}330	}331	if panicking.Load() != 0 {332		gopark(nil, nil, waitReasonPanicWait, traceBlockForever, 1)333	}334	if !exitHooksRun {335		runExitHooks(0)336	}337	if raceenabled {338		racefini() // does not return339	}340341	exit(0)342	for {343		var x *int32344		*x = 0345	}346}347348// os_beforeExit is called from os.Exit(0).349//350//go:linkname os_beforeExit os.runtime_beforeExit351func os_beforeExit(exitCode int) {352	runExitHooks(exitCode)353	if exitCode == 0 && raceenabled {354		racefini()355	}356357	// See comment in main, above.358	if exitCode == 0 && asanenabled && (isarchive || islibrary || NumCgoCall() > 1) {359		lsandoleakcheck()360	}361}362363func init() {364	exithook.Gosched = Gosched365	exithook.Goid = func() uint64 { return getg().goid }366	exithook.Throw = throw367}368369func runExitHooks(code int) {370	exithook.Run(code)371}372373// start forcegc helper goroutine374func init() {375	go forcegchelper()376}377378func forcegchelper() {379	forcegc.g = getg()380	lockInit(&forcegc.lock, lockRankForcegc)381	for {382		lock(&forcegc.lock)383		if forcegc.idle.Load() {384			throw("forcegc: phase error")385		}386		forcegc.idle.Store(true)387		goparkunlock(&forcegc.lock, waitReasonForceGCIdle, traceBlockSystemGoroutine, 1)388		// this goroutine is explicitly resumed by sysmon389		if debug.gctrace > 0 {390			println("GC forced")391		}392		// Time-triggered, fully concurrent.393		gcStart(gcTrigger{kind: gcTriggerTime, now: nanotime()})394	}395}396397// Gosched yields the processor, allowing other goroutines to run. It does not398// suspend the current goroutine, so execution resumes automatically.399//400//go:nosplit401func Gosched() {402	checkTimeouts()403	mcall(gosched_m)404}405406// goschedguarded yields the processor like gosched, but also checks407// for forbidden states and opts out of the yield in those cases.408//409//go:nosplit410func goschedguarded() {411	mcall(goschedguarded_m)412}413414// goschedIfBusy yields the processor like gosched, but only does so if415// there are no idle Ps or if we're on the only P and there's nothing in416// the run queue. In both cases, there is freely available idle time.417//418//go:nosplit419func goschedIfBusy() {420	gp := getg()421	// Call gosched if gp.preempt is set; we may be in a tight loop that422	// doesn't otherwise yield.423	if !gp.preempt && sched.npidle.Load() > 0 {424		return425	}426	mcall(gosched_m)427}428429// Puts the current goroutine into a waiting state and calls unlockf on the430// system stack.431//432// If unlockf returns false, the goroutine is resumed.433//434// unlockf must not access this G's stack, as it may be moved between435// the call to gopark and the call to unlockf.436//437// Note that because unlockf is called after putting the G into a waiting438// state, the G may have already been readied by the time unlockf is called439// unless there is external synchronization preventing the G from being440// readied. If unlockf returns false, it must guarantee that the G cannot be441// externally readied.442//443// Reason explains why the goroutine has been parked. It is displayed in stack444// traces and heap dumps. Reasons should be unique and descriptive. Do not445// re-use reasons, add new ones.446//447// gopark should be an internal detail,448// but widely used packages access it using linkname.449// Notable members of the hall of shame include:450//   - gvisor.dev/gvisor451//   - github.com/sagernet/gvisor452//453// Do not remove or change the type signature.454// See go.dev/issue/67401.455//456//go:linkname gopark457func gopark(unlockf func(*g, unsafe.Pointer) bool, lock unsafe.Pointer, reason waitReason, traceReason traceBlockReason, traceskip int) {458	if reason != waitReasonSleep {459		checkTimeouts() // timeouts may expire while two goroutines keep the scheduler busy460	}461	mp := acquirem()462	gp := mp.curg463	status := readgstatus(gp)464	if status != _Grunning && status != _Gscanrunning {465		throw("gopark: bad g status")466	}467	mp.waitlock = lock468	mp.waitunlockf = unlockf469	gp.waitreason = reason470	mp.waitTraceBlockReason = traceReason471	mp.waitTraceSkip = traceskip472	releasem(mp)473	// can't do anything that might move the G between Ms here.474	mcall(park_m)475}476477// Puts the current goroutine into a waiting state and unlocks the lock.478// The goroutine can be made runnable again by calling goready(gp).479func goparkunlock(lock *mutex, reason waitReason, traceReason traceBlockReason, traceskip int) {480	gopark(parkunlock_c, unsafe.Pointer(lock), reason, traceReason, traceskip)481}482483// goready should be an internal detail,484// but widely used packages access it using linkname.485// Notable members of the hall of shame include:486//   - gvisor.dev/gvisor487//   - github.com/sagernet/gvisor488//489// Do not remove or change the type signature.490// See go.dev/issue/67401.491//492//go:linkname goready493func goready(gp *g, traceskip int) {494	systemstack(func() {495		ready(gp, traceskip, true)496	})497}498499//go:nosplit500func acquireSudog() *sudog {501	// Delicate dance: the semaphore implementation calls502	// acquireSudog, acquireSudog calls new(sudog),503	// new calls malloc, malloc can call the garbage collector,504	// and the garbage collector calls the semaphore implementation505	// in stopTheWorld.506	// Break the cycle by doing acquirem/releasem around new(sudog).507	// The acquirem/releasem increments m.locks during new(sudog),508	// which keeps the garbage collector from being invoked.509	mp := acquirem()510	pp := mp.p.ptr()511	if len(pp.sudogcache) == 0 {512		lock(&sched.sudoglock)513		// First, try to grab a batch from central cache.514		for len(pp.sudogcache) < cap(pp.sudogcache)/2 && sched.sudogcache != nil {515			s := sched.sudogcache516			sched.sudogcache = s.next517			s.next = nil518			pp.sudogcache = append(pp.sudogcache, s)519		}520		unlock(&sched.sudoglock)521		// If the central cache is empty, allocate a new one.522		if len(pp.sudogcache) == 0 {523			pp.sudogcache = append(pp.sudogcache, new(sudog))524		}525	}526	n := len(pp.sudogcache)527	s := pp.sudogcache[n-1]528	pp.sudogcache[n-1] = nil529	pp.sudogcache = pp.sudogcache[:n-1]530	if s.elem.get() != nil {531		throw("acquireSudog: found s.elem != nil in cache")532	}533	releasem(mp)534	return s535}536537//go:nosplit538func releaseSudog(s *sudog) {539	if s.elem.get() != nil {540		throw("runtime: sudog with non-nil elem")541	}542	if s.isSelect {543		throw("runtime: sudog with non-false isSelect")544	}545	if s.next != nil {546		throw("runtime: sudog with non-nil next")547	}548	if s.prev != nil {549		throw("runtime: sudog with non-nil prev")550	}551	if s.waitlink != nil {552		throw("runtime: sudog with non-nil waitlink")553	}554	if s.c.get() != nil {555		throw("runtime: sudog with non-nil c")556	}557	gp := getg()558	if gp.param != nil {559		throw("runtime: releaseSudog with non-nil gp.param")560	}561	mp := acquirem() // avoid rescheduling to another P562	pp := mp.p.ptr()563	if len(pp.sudogcache) == cap(pp.sudogcache) {564		// Transfer half of local cache to the central cache.565		var first, last *sudog566		for len(pp.sudogcache) > cap(pp.sudogcache)/2 {567			n := len(pp.sudogcache)568			p := pp.sudogcache[n-1]569			pp.sudogcache[n-1] = nil570			pp.sudogcache = pp.sudogcache[:n-1]571			if first == nil {572				first = p573			} else {574				last.next = p575			}576			last = p577		}578		lock(&sched.sudoglock)579		last.next = sched.sudogcache580		sched.sudogcache = first581		unlock(&sched.sudoglock)582	}583	pp.sudogcache = append(pp.sudogcache, s)584	releasem(mp)585}586587// called from assembly.588func badmcall(fn func(*g)) {589	throw("runtime: mcall called on m->g0 stack")590}591592func badmcall2(fn func(*g)) {593	throw("runtime: mcall function returned")594}595596func badreflectcall() {597	panic(plainError("arg size to reflect.call more than 1GB"))598}599600//go:nosplit601//go:nowritebarrierrec602func badmorestackg0() {603	if !crashStackImplemented {604		writeErrStr("fatal: morestack on g0\n")605		return606	}607608	g := getg()609	switchToCrashStack(func() {610		print("runtime: morestack on g0, stack [", hex(g.stack.lo), " ", hex(g.stack.hi), "], sp=", hex(g.sched.sp), ", called from\n")611		g.m.traceback = 2 // include pc and sp in stack trace612		traceback1(g.sched.pc, g.sched.sp, g.sched.lr, g, 0)613		print("\n")614615		throw("morestack on g0")616	})617}618619//go:nosplit620//go:nowritebarrierrec621func badmorestackgsignal() {622	writeErrStr("fatal: morestack on gsignal\n")623}624625//go:nosplit626func badctxt() {627	throw("ctxt != 0")628}629630// gcrash is a fake g that can be used when crashing due to bad631// stack conditions.632var gcrash g633634var crashingG atomic.Pointer[g]635636// Switch to crashstack and call fn, with special handling of637// concurrent and recursive cases.638//639// Nosplit as it is called in a bad stack condition (we know640// morestack would fail).641//642//go:nosplit643//go:nowritebarrierrec644func switchToCrashStack(fn func()) {645	me := getg()646	if crashingG.CompareAndSwapNoWB(nil, me) {647		switchToCrashStack0(fn) // should never return648		abort()649	}650	if crashingG.Load() == me {651		// recursive crashing. too bad.652		writeErrStr("fatal: recursive switchToCrashStack\n")653		abort()654	}655	// Another g is crashing. Give it some time, hopefully it will finish traceback.656	usleep_no_g(100)657	writeErrStr("fatal: concurrent switchToCrashStack\n")658	abort()659}660661// Disable crash stack on Windows for now. Apparently, throwing an exception662// on a non-system-allocated crash stack causes EXCEPTION_STACK_OVERFLOW and663// hangs the process (see issue 63938).664const crashStackImplemented = GOOS != "windows"665666//go:noescape667func switchToCrashStack0(fn func()) // in assembly668669func lockedOSThread() bool {670	gp := getg()671	return gp.lockedm != 0 && gp.m.lockedg != 0672}673674var (675	// allgs contains all Gs ever created (including dead Gs), and thus676	// never shrinks.677	//678	// Access via the slice is protected by allglock or stop-the-world.679	// Readers that cannot take the lock may (carefully!) use the atomic680	// variables below.681	allglock mutex682	allgs    []*g683684	// allglen and allgptr are atomic variables that contain len(allgs) and685	// &allgs[0] respectively. Proper ordering depends on totally-ordered686	// loads and stores. Writes are protected by allglock.687	//688	// allgptr is updated before allglen. Readers should read allglen689	// before allgptr to ensure that allglen is always <= len(allgptr). New690	// Gs appended during the race can be missed. For a consistent view of691	// all Gs, allglock must be held.692	//693	// allgptr copies should always be stored as a concrete type or694	// unsafe.Pointer, not uintptr, to ensure that GC can still reach it695	// even if it points to a stale array.696	allglen uintptr697	allgptr **g698)699700func allgadd(gp *g) {701	if readgstatus(gp) == _Gidle {702		throw("allgadd: bad status Gidle")703	}704705	lock(&allglock)706	allgs = append(allgs, gp)707	if &allgs[0] != allgptr {708		atomicstorep(unsafe.Pointer(&allgptr), unsafe.Pointer(&allgs[0]))709	}710	atomic.Storeuintptr(&allglen, uintptr(len(allgs)))711	unlock(&allglock)712}713714// allGsSnapshot returns a snapshot of the slice of all Gs.715//716// The world must be stopped or allglock must be held.717func allGsSnapshot() []*g {718	assertWorldStoppedOrLockHeld(&allglock)719720	// Because the world is stopped or allglock is held, allgadd721	// cannot happen concurrently with this. allgs grows722	// monotonically and existing entries never change, so we can723	// simply return a copy of the slice header. For added safety,724	// we trim everything past len because that can still change.725	return allgs[:len(allgs):len(allgs)]726}727728// atomicAllG returns &allgs[0] and len(allgs) for use with atomicAllGIndex.729func atomicAllG() (**g, uintptr) {730	length := atomic.Loaduintptr(&allglen)731	ptr := (**g)(atomic.Loadp(unsafe.Pointer(&allgptr)))732	return ptr, length733}734735// atomicAllGIndex returns ptr[i] with the allgptr returned from atomicAllG.736func atomicAllGIndex(ptr **g, i uintptr) *g {737	return *(**g)(add(unsafe.Pointer(ptr), i*goarch.PtrSize))738}739740// forEachG calls fn on every G from allgs.741//742// forEachG takes a lock to exclude concurrent addition of new Gs.743func forEachG(fn func(gp *g)) {744	lock(&allglock)745	for _, gp := range allgs {746		fn(gp)747	}748	unlock(&allglock)749}750751// forEachGRace calls fn on every G from allgs.752//753// forEachGRace avoids locking, but does not exclude addition of new Gs during754// execution, which may be missed.755func forEachGRace(fn func(gp *g)) {756	ptr, length := atomicAllG()757	for i := uintptr(0); i < length; i++ {758		gp := atomicAllGIndex(ptr, i)759		fn(gp)760	}761	return762}763764const (765	// Number of goroutine ids to grab from sched.goidgen to local per-P cache at once.766	// 16 seems to provide enough amortization, but other than that it's mostly arbitrary number.767	_GoidCacheBatch = 16768)769770// cpuinit sets up CPU feature flags and calls internal/cpu.Initialize. env should be the complete771// value of the GODEBUG environment variable.772func cpuinit(env string) {773	cpu.Initialize(env)774775	// Support cpu feature variables are used in code generated by the compiler776	// to guard execution of instructions that can not be assumed to be always supported.777	switch GOARCH {778	case "386", "amd64":779		x86HasAVX = cpu.X86.HasAVX780		x86HasFMA = cpu.X86.HasFMA781		x86HasPOPCNT = cpu.X86.HasPOPCNT782		x86HasSSE41 = cpu.X86.HasSSE41783784	case "arm":785		armHasVFPv4 = cpu.ARM.HasVFPv4786787	case "arm64":788		arm64HasATOMICS = cpu.ARM64.HasATOMICS789790	case "loong64":791		loong64HasLAMCAS = cpu.Loong64.HasLAMCAS792		loong64HasLAM_BH = cpu.Loong64.HasLAM_BH793		loong64HasDBAR_HINTS = cpu.Loong64.HasDBAR_HINTS794		loong64HasLSX = cpu.Loong64.HasLSX795796	case "riscv64":797		riscv64HasZbb = cpu.RISCV64.HasZbb798	}799}800801// getGodebugEarly extracts the environment variable GODEBUG from the environment on802// Unix-like operating systems and returns it. This function exists to extract GODEBUG803// early before much of the runtime is initialized.804//805// Returns nil, false if OS doesn't provide env vars early in the init sequence.806func getGodebugEarly() (string, bool) {807	const prefix = "GODEBUG="808	var env string809	switch GOOS {810	case "aix", "darwin", "ios", "dragonfly", "freebsd", "netbsd", "openbsd", "illumos", "solaris", "linux":811		// Similar to goenv_unix but extracts the environment value for812		// GODEBUG directly.813		// TODO(moehrmann): remove when general goenvs() can be called before cpuinit()814		n := int32(0)815		for argv_index(argv, argc+1+n) != nil {816			n++817		}818819		for i := int32(0); i < n; i++ {820			p := argv_index(argv, argc+1+i)821			s := unsafe.String(p, findnull(p))822823			if stringslite.HasPrefix(s, prefix) {824				env = gostringnocopy(p)[len(prefix):]825				break826			}827		}828		break829830	default:831		return "", false832	}833	return env, true834}835836// The bootstrap sequence is:837//838//	call osinit839//	call schedinit840//	make & queue new G841//	call runtime·mstart842//843// The new G calls runtime·main.844func schedinit() {845	lockInit(&sched.lock, lockRankSched)846	lockInit(&sched.sysmonlock, lockRankSysmon)847	lockInit(&sched.deferlock, lockRankDefer)848	lockInit(&sched.sudoglock, lockRankSudog)849	lockInit(&deadlock, lockRankDeadlock)850	lockInit(&paniclk, lockRankPanic)851	lockInit(&allglock, lockRankAllg)852	lockInit(&allpLock, lockRankAllp)853	lockInit(&reflectOffs.lock, lockRankReflectOffs)854	lockInit(&finlock, lockRankFin)855	lockInit(&cpuprof.lock, lockRankCpuprof)856	lockInit(&computeMaxProcsLock, lockRankComputeMaxProcs)857	allocmLock.init(lockRankAllocmR, lockRankAllocmRInternal, lockRankAllocmW)858	execLock.init(lockRankExecR, lockRankExecRInternal, lockRankExecW)859	traceLockInit()860	// Enforce that this lock is always a leaf lock.861	// All of this lock's critical sections should be862	// extremely short.863	lockInit(&memstats.heapStats.noPLock, lockRankLeafRank)864865	lockVerifyMSize()866867	sched.midle.init(unsafe.Offsetof(m{}.idleNode))868869	// raceinit must be the first call to race detector.870	// In particular, it must be done before mallocinit below calls racemapshadow.871	gp := getg()872	if raceenabled {873		gp.racectx, raceprocctx0 = raceinit()874	}875876	sched.maxmcount = 10000877	crashFD.Store(^uintptr(0))878879	// The world starts stopped.880	worldStopped()881882	godebug, parsedGodebug := getGodebugEarly()883	if parsedGodebug {884		parseRuntimeDebugVars(godebug)885	}886	ticks.init() // run as early as possible887	moduledataverify()888	stackinit()889	randinit() // must run before mallocinit, AlgInit, mcommoninit890	mallocinit()891	cpuinit(godebug) // must run before AlgInit892	maps.AlgInit()   // maps, hash, rand must not be used before this call893	mcommoninit(gp.m, -1)894	modulesinit()   // provides activeModules895	typelinksinit() // uses maps, activeModules896	itabsinit()     // uses activeModules897	stkobjinit()    // must run before GC starts898899	sigsave(&gp.m.sigmask)900	initSigmask = gp.m.sigmask901902	goargs()903	goenvs()904	secure()905	checkfds()906	if !parsedGodebug {907		// Some platforms, e.g., Windows, didn't make env vars available "early",908		// so try again now.909		parseRuntimeDebugVars(gogetenv("GODEBUG"))910	}911	finishDebugVarsSetup()912	gcinit()913914	// Allocate stack space that can be used when crashing due to bad stack915	// conditions, e.g. morestack on g0.916	gcrash.stack = stackalloc(16384)917	gcrash.stackguard0 = gcrash.stack.lo + 1000918	gcrash.stackguard1 = gcrash.stack.lo + 1000919920	// if disableMemoryProfiling is set, update MemProfileRate to 0 to turn off memprofile.921	// Note: parsedebugvars may update MemProfileRate, but when disableMemoryProfiling is922	// set to true by the linker, it means that nothing is consuming the profile, it is923	// safe to set MemProfileRate to 0.924	if disableMemoryProfiling {925		MemProfileRate = 0926	}927928	// mcommoninit runs before parsedebugvars, so init profstacks again.929	mProfStackInit(gp.m)930	defaultGOMAXPROCSInit()931932	lock(&sched.lock)933	sched.lastpoll.Store(nanotime())934	var procs int32935	if n, err := strconv.ParseInt(gogetenv("GOMAXPROCS"), 10, 32); err == nil && n > 0 {936		procs = int32(n)937		sched.customGOMAXPROCS = true938	} else {939		// Use numCPUStartup for initial GOMAXPROCS for two reasons:940		//941		// 1. We just computed it in osinit, recomputing is (minorly) wasteful.942		//943		// 2. More importantly, if debug.containermaxprocs == 0 &&944		//    debug.updatemaxprocs == 0, we want to guarantee that945		//    runtime.GOMAXPROCS(0) always equals runtime.NumCPU (which is946		//    just numCPUStartup).947		procs = defaultGOMAXPROCS(numCPUStartup)948	}949	if procresize(procs) != nil {950		throw("unknown runnable goroutine during bootstrap")951	}952	unlock(&sched.lock)953954	// World is effectively started now, as P's can run.955	worldStarted()956957	if buildVersion == "" {958		// Condition should never trigger. This code just serves959		// to ensure runtime·buildVersion is kept in the resulting binary.960		buildVersion = "unknown"961	}962	if len(modinfo) == 1 {963		// Condition should never trigger. This code just serves964		// to ensure runtime·modinfo is kept in the resulting binary.965		modinfo = ""966	}967}968969func dumpgstatus(gp *g) {970	thisg := getg()971	print("runtime:   gp: gp=", gp, ", goid=", gp.goid, ", gp->atomicstatus=", readgstatus(gp), "\n")972	print("runtime: getg:  g=", thisg, ", goid=", thisg.goid, ",  g->atomicstatus=", readgstatus(thisg), "\n")973}974975// sched.lock must be held.976func checkmcount() {977	assertLockHeld(&sched.lock)978979	// Exclude extra M's, which are used for cgocallback from threads980	// created in C.981	//982	// The purpose of the SetMaxThreads limit is to avoid accidental fork983	// bomb from something like millions of goroutines blocking on system984	// calls, causing the runtime to create millions of threads. By985	// definition, this isn't a problem for threads created in C, so we986	// exclude them from the limit. See https://go.dev/issue/60004.987	count := mcount() - int32(extraMInUse.Load()) - int32(extraMLength.Load())988	if count > sched.maxmcount {989		print("runtime: program exceeds ", sched.maxmcount, "-thread limit\n")990		throw("thread exhaustion")991	}992}993994// mReserveID returns the next ID to use for a new m. This new m is immediately995// considered 'running' by checkdead.996//997// sched.lock must be held.998func mReserveID() int64 {999	assertLockHeld(&sched.lock)10001001	if sched.mnext+1 < sched.mnext {1002		throw("runtime: thread ID overflow")1003	}1004	id := sched.mnext1005	sched.mnext++1006	checkmcount()1007	return id1008}10091010// Pre-allocated ID may be passed as 'id', or omitted by passing -1.1011func mcommoninit(mp *m, id int64) {1012	gp := getg()10131014	// g0 stack won't make sense for user (and is not necessary unwindable).1015	if gp != gp.m.g0 {1016		callers(1, mp.createstack[:])1017	}10181019	lock(&sched.lock)10201021	if id >= 0 {1022		mp.id = id1023	} else {1024		mp.id = mReserveID()1025	}10261027	mp.self = newMWeakPointer(mp)10281029	mrandinit(mp)10301031	mpreinit(mp)1032	if mp.gsignal != nil {1033		mp.gsignal.stackguard1 = mp.gsignal.stack.lo + stackGuard1034	}10351036	// Add to allm so garbage collector doesn't free g->m1037	// when it is just in a register or thread-local storage.1038	mp.alllink = allm10391040	// NumCgoCall and others iterate over allm w/o schedlock,1041	// so we need to publish it safely.1042	atomicstorep(unsafe.Pointer(&allm), unsafe.Pointer(mp))1043	unlock(&sched.lock)10441045	// Allocate memory to hold a cgo traceback if the cgo call crashes.1046	if iscgo || GOOS == "solaris" || GOOS == "illumos" || GOOS == "windows" {1047		mp.cgoCallers = new(cgoCallers)1048	}1049	mProfStackInit(mp)1050}10511052// mProfStackInit is used to eagerly initialize stack trace buffers for1053// profiling. Lazy allocation would have to deal with reentrancy issues in1054// malloc and runtime locks for mLockProfile.1055// TODO(mknyszek): Implement lazy allocation if this becomes a problem.1056func mProfStackInit(mp *m) {1057	if debug.profstackdepth == 0 {1058		// debug.profstack is set to 0 by the user, or we're being called from1059		// schedinit before parsedebugvars.1060		return1061	}1062	mp.profStack = makeProfStackFP()1063	mp.mLockProfile.stack = makeProfStackFP()1064}10651066// makeProfStackFP creates a buffer large enough to hold a maximum-sized stack1067// trace as well as any additional frames needed for frame pointer unwinding1068// with delayed inline expansion.1069func makeProfStackFP() []uintptr {1070	// The "1" term is to account for the first stack entry being1071	// taken up by a "skip" sentinel value for profilers which1072	// defer inline frame expansion until the profile is reported.1073	// The "maxSkip" term is for frame pointer unwinding, where we1074	// want to end up with debug.profstackdebth frames but will discard1075	// some "physical" frames to account for skipping.1076	return make([]uintptr, 1+maxSkip+debug.profstackdepth)1077}10781079// makeProfStack returns a buffer large enough to hold a maximum-sized stack1080// trace.1081func makeProfStack() []uintptr { return make([]uintptr, debug.profstackdepth) }10821083//go:linkname pprof_makeProfStack1084func pprof_makeProfStack() []uintptr { return makeProfStack() }10851086func (mp *m) becomeSpinning() {1087	mp.spinning = true1088	sched.nmspinning.Add(1)1089	sched.needspinning.Store(0)1090}10911092// Take a snapshot of allp, for use after dropping the P.1093//1094// Must be called with a P, but the returned slice may be used after dropping1095// the P. The M holds a reference on the snapshot to keep the backing array1096// alive.1097//1098//go:yeswritebarrierrec1099func (mp *m) snapshotAllp() []*p {1100	mp.allpSnapshot = allp1101	return mp.allpSnapshot1102}11031104// Clear the saved allp snapshot. Should be called as soon as the snapshot is1105// no longer required.1106//1107// Must be called after reacquiring a P, as it requires a write barrier.1108//1109//go:yeswritebarrierrec1110func (mp *m) clearAllpSnapshot() {1111	mp.allpSnapshot = nil1112}11131114func (mp *m) hasCgoOnStack() bool {1115	return mp.ncgo > 0 || mp.isextra1116}11171118const (1119	// osHasLowResTimer indicates that the platform's internal timer system has a low resolution,1120	// typically on the order of 1 ms or more.1121	osHasLowResTimer = GOOS == "windows" || GOOS == "openbsd" || GOOS == "netbsd" || GOOS == "plan9"11221123	// osHasLowResClockInt is osHasLowResClock but in integer form, so it can be used to create1124	// constants conditionally.1125	osHasLowResClockInt = goos.IsWindows11261127	// osHasLowResClock indicates that timestamps produced by nanotime on the platform have a1128	// low resolution, typically on the order of 1 ms or more.1129	osHasLowResClock = osHasLowResClockInt > 01130)11311132// Mark gp ready to run.1133func ready(gp *g, traceskip int, next bool) {1134	status := readgstatus(gp)11351136	// Mark runnable.1137	mp := acquirem() // disable preemption because it can be holding p in a local var1138	if status&^_Gscan != _Gwaiting {1139		dumpgstatus(gp)1140		throw("bad g->status in ready")1141	}11421143	// status is Gwaiting or Gscanwaiting, make Grunnable and put on runq1144	trace := traceAcquire()1145	casgstatus(gp, _Gwaiting, _Grunnable)1146	if trace.ok() {1147		trace.GoUnpark(gp, traceskip)1148		traceRelease(trace)1149	}1150	runqput(mp.p.ptr(), gp, next)1151	wakep()1152	releasem(mp)1153}11541155// freezeStopWait is a large value that freezetheworld sets1156// sched.stopwait to in order to request that all Gs permanently stop.1157const freezeStopWait = 0x7fffffff11581159// freezing is set to non-zero if the runtime is trying to freeze the1160// world.1161var freezing atomic.Bool11621163// Similar to stopTheWorld but best-effort and can be called several times.1164// There is no reverse operation, used during crashing.1165// This function must not lock any mutexes.1166func freezetheworld() {1167	freezing.Store(true)1168	if debug.dontfreezetheworld > 0 {1169		// Don't prempt Ps to stop goroutines. That will perturb1170		// scheduler state, making debugging more difficult. Instead,1171		// allow goroutines to continue execution.1172		//1173		// fatalpanic will tracebackothers to trace all goroutines. It1174		// is unsafe to trace a running goroutine, so tracebackothers1175		// will skip running goroutines. That is OK and expected, we1176		// expect users of dontfreezetheworld to use core files anyway.1177		//1178		// However, allowing the scheduler to continue running free1179		// introduces a race: a goroutine may be stopped when1180		// tracebackothers checks its status, and then start running1181		// later when we are in the middle of traceback, potentially1182		// causing a crash.1183		//1184		// To mitigate this, when an M naturally enters the scheduler,1185		// schedule checks if freezing is set and if so stops1186		// execution. This guarantees that while Gs can transition from1187		// running to stopped, they can never transition from stopped1188		// to running.1189		//1190		// The sleep here allows racing Ms that missed freezing and are1191		// about to run a G to complete the transition to running1192		// before we start traceback.1193		usleep(1000)1194		return1195	}11961197	// stopwait and preemption requests can be lost1198	// due to races with concurrently executing threads,1199	// so try several times1200	for i := 0; i < 5; i++ {1201		// this should tell the scheduler to not start any new goroutines1202		sched.stopwait = freezeStopWait1203		sched.gcwaiting.Store(true)1204		// this should stop running goroutines1205		if !preemptall() {1206			break // no running goroutines1207		}1208		usleep(1000)1209	}1210	// to be sure1211	usleep(1000)1212	preemptall()1213	usleep(1000)1214}12151216// All reads and writes of g's status go through readgstatus, casgstatus1217// castogscanstatus, casfrom_Gscanstatus.1218//1219//go:nosplit1220func readgstatus(gp *g) uint32 {1221	return gp.atomicstatus.Load()1222}12231224// The Gscanstatuses are acting like locks and this releases them.1225// If it proves to be a performance hit we should be able to make these1226// simple atomic stores but for now we are going to throw if1227// we see an inconsistent state.1228func casfrom_Gscanstatus(gp *g, oldval, newval uint32) {1229	success := false12301231	// Check that transition is valid.1232	switch oldval {1233	default:1234		print("runtime: casfrom_Gscanstatus bad oldval gp=", gp, ", oldval=", hex(oldval), ", newval=", hex(newval), "\n")1235		dumpgstatus(gp)1236		throw("casfrom_Gscanstatus:top gp->status is not in scan state")1237	case _Gscanrunnable,1238		_Gscanwaiting,1239		_Gscanrunning,1240		_Gscansyscall,1241		_Gscanleaked,1242		_Gscanpreempted,1243		_Gscandeadextra:1244		if newval == oldval&^_Gscan {1245			success = gp.atomicstatus.CompareAndSwap(oldval, newval)1246		}1247	}1248	if !success {1249		print("runtime: casfrom_Gscanstatus failed gp=", gp, ", oldval=", hex(oldval), ", newval=", hex(newval), "\n")1250		dumpgstatus(gp)1251		throw("casfrom_Gscanstatus: gp->status is not in scan state")1252	}1253	releaseLockRankAndM(lockRankGscan)1254}12551256// This will return false if the gp is not in the expected status and the cas fails.1257// This acts like a lock acquire while the casfromgstatus acts like a lock release.1258func castogscanstatus(gp *g, oldval, newval uint32) bool {1259	switch oldval {1260	case _Grunnable,1261		_Grunning,1262		_Gwaiting,1263		_Gleaked,1264		_Gsyscall,1265		_Gdeadextra:1266		if newval == oldval|_Gscan {1267			r := gp.atomicstatus.CompareAndSwap(oldval, newval)1268			if r {1269				acquireLockRankAndM(lockRankGscan)1270			}1271			return r12721273		}1274	}1275	print("runtime: castogscanstatus oldval=", hex(oldval), " newval=", hex(newval), "\n")1276	throw("bad oldval passed to castogscanstatus")1277	return false1278}12791280// casgstatusAlwaysTrack is a debug flag that causes casgstatus to always track1281// various latencies on every transition instead of sampling them.1282var casgstatusAlwaysTrack = false12831284// If asked to move to or from a Gscanstatus this will throw. Use the castogscanstatus1285// and casfrom_Gscanstatus instead.1286// casgstatus will loop if the g->atomicstatus is in a Gscan status until the routine that1287// put it in the Gscan state is finished.1288//1289//go:nosplit1290func casgstatus(gp *g, oldval, newval uint32) {1291	if (oldval&_Gscan != 0) || (newval&_Gscan != 0) || oldval == newval {1292		systemstack(func() {1293			// Call on the systemstack to prevent print and throw from counting1294			// against the nosplit stack reservation.1295			print("runtime: casgstatus: oldval=", hex(oldval), " newval=", hex(newval), "\n")1296			throw("casgstatus: bad incoming values")1297		})1298	}12991300	lockWithRankMayAcquire(nil, lockRankGscan)13011302	// See https://golang.org/cl/21503 for justification of the yield delay.1303	const yieldDelay = 5 * 10001304	var nextYield int6413051306	// loop if gp->atomicstatus is in a scan state giving1307	// GC time to finish and change the state to oldval.1308	for i := 0; !gp.atomicstatus.CompareAndSwap(oldval, newval); i++ {1309		if oldval == _Gwaiting && gp.atomicstatus.Load() == _Grunnable {1310			systemstack(func() {1311				// Call on the systemstack to prevent throw from counting1312				// against the nosplit stack reservation.1313				throw("casgstatus: waiting for Gwaiting but is Grunnable")1314			})1315		}1316		if i == 0 {1317			nextYield = nanotime() + yieldDelay1318		}1319		if nanotime() < nextYield {1320			for x := 0; x < 10 && gp.atomicstatus.Load() != oldval; x++ {1321				procyield(1)1322			}1323		} else {1324			osyield()1325			nextYield = nanotime() + yieldDelay/21326		}1327	}13281329	if gp.bubble != nil {1330		systemstack(func() {1331			gp.bubble.changegstatus(gp, oldval, newval)1332		})1333	}13341335	if (oldval == _Grunning || oldval == _Gsyscall) && (newval != _Grunning && newval != _Gsyscall) {1336		// Track every gTrackingPeriod time a goroutine transitions out of _Grunning or _Gsyscall.1337		// Do not track _Grunning <-> _Gsyscall transitions, since they're two very similar states.1338		if casgstatusAlwaysTrack || gp.trackingSeq%gTrackingPeriod == 0 {1339			gp.tracking = true1340		}1341		gp.trackingSeq++1342	}1343	if !gp.tracking {1344		return1345	}13461347	// Handle various kinds of tracking.1348	//1349	// Currently:1350	// - Time spent in runnable.1351	// - Time spent blocked on a sync.Mutex or sync.RWMutex.1352	switch oldval {1353	case _Grunnable:1354		// We transitioned out of runnable, so measure how much1355		// time we spent in this state and add it to1356		// runnableTime.1357		now := nanotime()1358		gp.runnableTime += now - gp.trackingStamp1359		gp.trackingStamp = 01360	case _Gwaiting:1361		if !gp.waitreason.isMutexWait() {1362			// Not blocking on a lock.1363			break1364		}1365		// Blocking on a lock, measure it. Note that because we're1366		// sampling, we have to multiply by our sampling period to get1367		// a more representative estimate of the absolute value.1368		// gTrackingPeriod also represents an accurate sampling period1369		// because we can only enter this state from _Grunning.1370		now := nanotime()1371		sched.totalMutexWaitTime.Add((now - gp.trackingStamp) * gTrackingPeriod)1372		gp.trackingStamp = 01373	}1374	switch newval {1375	case _Gwaiting:1376		if !gp.waitreason.isMutexWait() {1377			// Not blocking on a lock.1378			break1379		}1380		// Blocking on a lock. Write down the timestamp.1381		now := nanotime()1382		gp.trackingStamp = now1383	case _Grunnable:1384		// We just transitioned into runnable, so record what1385		// time that happened.1386		now := nanotime()1387		gp.trackingStamp = now1388	case _Grunning:1389		// We're transitioning into running, so turn off1390		// tracking and record how much time we spent in1391		// runnable.1392		gp.tracking = false1393		sched.timeToRun.record(gp.runnableTime)1394		gp.runnableTime = 01395	}1396}13971398// casGToWaiting transitions gp from old to _Gwaiting, and sets the wait reason.1399//1400// Use this over casgstatus when possible to ensure that a waitreason is set.1401func casGToWaiting(gp *g, old uint32, reason waitReason) {1402	// Set the wait reason before calling casgstatus, because casgstatus will use it.1403	gp.waitreason = reason1404	casgstatus(gp, old, _Gwaiting)1405}14061407// casGToWaitingForSuspendG transitions gp from old to _Gwaiting, and sets the wait reason.1408// The wait reason must be a valid isWaitingForSuspendG wait reason.1409//1410// While a goroutine is in this state, it's stack is effectively pinned.1411// The garbage collector must not shrink or otherwise mutate the goroutine's stack.1412//1413// Use this over casgstatus when possible to ensure that a waitreason is set.1414func casGToWaitingForSuspendG(gp *g, old uint32, reason waitReason) {1415	if !reason.isWaitingForSuspendG() {1416		throw("casGToWaitingForSuspendG with non-isWaitingForSuspendG wait reason")1417	}1418	casGToWaiting(gp, old, reason)1419}14201421// casGToPreemptScan transitions gp from _Grunning to _Gscan|_Gpreempted.1422//1423// TODO(austin): This is the only status operation that both changes1424// the status and locks the _Gscan bit. Rethink this.1425func casGToPreemptScan(gp *g, old, new uint32) {1426	if old != _Grunning || new != _Gscan|_Gpreempted {1427		throw("bad g transition")1428	}1429	acquireLockRankAndM(lockRankGscan)1430	for !gp.atomicstatus.CompareAndSwap(_Grunning, _Gscan|_Gpreempted) {1431	}1432	// We never notify gp.bubble that the goroutine state has moved1433	// from _Grunning to _Gpreempted. We call bubble.changegstatus1434	// after status changes happen, but doing so here would violate the1435	// ordering between the gscan and synctest locks. The bubble doesn't1436	// distinguish between _Grunning and _Gpreempted anyway, so not1437	// notifying it is fine.1438}14391440// casGFromPreempted attempts to transition gp from _Gpreempted to1441// _Gwaiting. If successful, the caller is responsible for1442// re-scheduling gp.1443func casGFromPreempted(gp *g, old, new uint32) bool {1444	if old != _Gpreempted || new != _Gwaiting {1445		throw("bad g transition")1446	}1447	gp.waitreason = waitReasonPreempted1448	if !gp.atomicstatus.CompareAndSwap(_Gpreempted, _Gwaiting) {1449		return false1450	}1451	if bubble := gp.bubble; bubble != nil {1452		bubble.changegstatus(gp, _Gpreempted, _Gwaiting)1453	}1454	return true1455}14561457// stwReason is an enumeration of reasons the world is stopping.1458type stwReason uint814591460// Reasons to stop-the-world.1461//1462// Avoid reusing reasons and add new ones instead.1463const (1464	stwUnknown                     stwReason = iota // "unknown"1465	stwGCMarkTerm                                   // "GC mark termination"1466	stwGCSweepTerm                                  // "GC sweep termination"1467	stwWriteHeapDump                                // "write heap dump"1468	stwGoroutineProfile                             // "goroutine profile"1469	stwGoroutineProfileCleanup                      // "goroutine profile cleanup"1470	stwAllGoroutinesStack                           // "all goroutines stack trace"1471	stwReadMemStats                                 // "read mem stats"1472	stwAllThreadsSyscall                            // "AllThreadsSyscall"1473	stwGOMAXPROCS                                   // "GOMAXPROCS"1474	stwStartTrace                                   // "start trace"1475	stwStopTrace                                    // "stop trace"1476	stwForTestCountPagesInUse                       // "CountPagesInUse (test)"1477	stwForTestReadMetricsSlow                       // "ReadMetricsSlow (test)"1478	stwForTestReadMemStatsSlow                      // "ReadMemStatsSlow (test)"1479	stwForTestPageCachePagesLeaked                  // "PageCachePagesLeaked (test)"1480	stwForTestResetDebugLog                         // "ResetDebugLog (test)"1481)14821483func (r stwReason) String() string {1484	return stwReasonStrings[r]1485}14861487func (r stwReason) isGC() bool {1488	return r == stwGCMarkTerm || r == stwGCSweepTerm1489}14901491// If you add to this list, also add it to src/internal/trace/parser.go.1492// If you change the values of any of the stw* constants, bump the trace1493// version number and make a copy of this.1494var stwReasonStrings = [...]string{1495	stwUnknown:                     "unknown",1496	stwGCMarkTerm:                  "GC mark termination",1497	stwGCSweepTerm:                 "GC sweep termination",1498	stwWriteHeapDump:               "write heap dump",1499	stwGoroutineProfile:            "goroutine profile",1500	stwGoroutineProfileCleanup:     "goroutine profile cleanup",1501	stwAllGoroutinesStack:          "all goroutines stack trace",1502	stwReadMemStats:                "read mem stats",1503	stwAllThreadsSyscall:           "AllThreadsSyscall",1504	stwGOMAXPROCS:                  "GOMAXPROCS",1505	stwStartTrace:                  "start trace",1506	stwStopTrace:                   "stop trace",1507	stwForTestCountPagesInUse:      "CountPagesInUse (test)",1508	stwForTestReadMetricsSlow:      "ReadMetricsSlow (test)",1509	stwForTestReadMemStatsSlow:     "ReadMemStatsSlow (test)",1510	stwForTestPageCachePagesLeaked: "PageCachePagesLeaked (test)",1511	stwForTestResetDebugLog:        "ResetDebugLog (test)",1512}15131514// worldStop provides context from the stop-the-world required by the1515// start-the-world.1516type worldStop struct {1517	reason           stwReason1518	startedStopping  int641519	finishedStopping int641520	stoppingCPUTime  int641521}15221523// Temporary variable for stopTheWorld, when it can't write to the stack.1524//1525// Protected by worldsema.1526var stopTheWorldContext worldStop15271528// stopTheWorld stops all P's from executing goroutines, interrupting1529// all goroutines at GC safe points and records reason as the reason1530// for the stop. On return, only the current goroutine's P is running.1531// stopTheWorld must not be called from a system stack and the caller1532// must not hold worldsema. The caller must call startTheWorld when1533// other P's should resume execution.1534//1535// stopTheWorld is safe for multiple goroutines to call at the1536// same time. Each will execute its own stop, and the stops will1537// be serialized.1538//1539// This is also used by routines that do stack dumps. If the system is1540// in panic or being exited, this may not reliably stop all1541// goroutines.1542//1543// Returns the STW context. When starting the world, this context must be1544// passed to startTheWorld.1545func stopTheWorld(reason stwReason) worldStop {1546	semacquire(&worldsema)1547	gp := getg()1548	gp.m.preemptoff = reason.String()1549	systemstack(func() {1550		stopTheWorldContext = stopTheWorldWithSema(reason) // avoid write to stack1551	})1552	return stopTheWorldContext1553}15541555// startTheWorld undoes the effects of stopTheWorld.1556//1557// w must be the worldStop returned by stopTheWorld.1558func startTheWorld(w worldStop) {1559	systemstack(func() { startTheWorldWithSema(0, w) })15601561	// worldsema must be held over startTheWorldWithSema to ensure1562	// gomaxprocs cannot change while worldsema is held.1563	//1564	// Release worldsema with direct handoff to the next waiter, but1565	// acquirem so that semrelease1 doesn't try to yield our time.1566	//1567	// Otherwise if e.g. ReadMemStats is being called in a loop,1568	// it might stomp on other attempts to stop the world, such as1569	// for starting or ending GC. The operation this blocks is1570	// so heavy-weight that we should just try to be as fair as1571	// possible here.1572	//1573	// We don't want to just allow us to get preempted between now1574	// and releasing the semaphore because then we keep everyone1575	// (including, for example, GCs) waiting longer.1576	mp := acquirem()1577	mp.preemptoff = ""1578	semrelease1(&worldsema, true, 0)1579	releasem(mp)1580}15811582// stopTheWorldGC has the same effect as stopTheWorld, but blocks1583// until the GC is not running. It also blocks a GC from starting1584// until startTheWorldGC is called.1585func stopTheWorldGC(reason stwReason) worldStop {1586	semacquire(&gcsema)1587	return stopTheWorld(reason)1588}15891590// startTheWorldGC undoes the effects of stopTheWorldGC.1591//1592// w must be the worldStop returned by stopTheWorld.1593func startTheWorldGC(w worldStop) {1594	startTheWorld(w)1595	semrelease(&gcsema)1596}15971598// Holding worldsema grants an M the right to try to stop the world.1599var worldsema uint32 = 116001601// Holding gcsema grants the M the right to block a GC, and blocks1602// until the current GC is done. In particular, it prevents gomaxprocs1603// from changing concurrently.1604//1605// TODO(mknyszek): Once gomaxprocs and the execution tracer can handle1606// being changed/enabled during a GC, remove this.1607var gcsema uint32 = 116081609// stopTheWorldWithSema is the core implementation of stopTheWorld.1610// The caller is responsible for acquiring worldsema and disabling1611// preemption first and then should stopTheWorldWithSema on the system1612// stack:1613//1614//	semacquire(&worldsema, 0)1615//	m.preemptoff = "reason"1616//	var stw worldStop1617//	systemstack(func() {1618//		stw = stopTheWorldWithSema(reason)1619//	})1620//1621// When finished, the caller must either call startTheWorld or undo1622// these three operations separately:1623//1624//	m.preemptoff = ""1625//	systemstack(func() {1626//		now = startTheWorldWithSema(stw)1627//	})1628//	semrelease(&worldsema)1629//1630// It is allowed to acquire worldsema once and then execute multiple1631// startTheWorldWithSema/stopTheWorldWithSema pairs.1632// Other P's are able to execute between successive calls to1633// startTheWorldWithSema and stopTheWorldWithSema.1634// Holding worldsema causes any other goroutines invoking1635// stopTheWorld to block.1636//1637// Returns the STW context. When starting the world, this context must be1638// passed to startTheWorldWithSema.1639//1640//go:systemstack1641func stopTheWorldWithSema(reason stwReason) worldStop {1642	// Mark the goroutine which called stopTheWorld preemptible so its1643	// stack may be scanned by the GC or observed by the execution tracer.1644	//1645	// This lets a mark worker scan us or the execution tracer take our1646	// stack while we try to stop the world since otherwise we could get1647	// in a mutual preemption deadlock.1648	//1649	// casGToWaitingForSuspendG marks the goroutine as ineligible for a1650	// stack shrink, effectively pinning the stack in memory for the duration.1651	//1652	// N.B. The execution tracer is not aware of this status transition and1653	// handles it specially based on the wait reason.1654	casGToWaitingForSuspendG(getg().m.curg, _Grunning, waitReasonStoppingTheWorld)16551656	trace := traceAcquire()1657	if trace.ok() {1658		trace.STWStart(reason)1659		traceRelease(trace)1660	}1661	gp := getg()16621663	// If we hold a lock, then we won't be able to stop another M1664	// that is blocked trying to acquire the lock.1665	if gp.m.locks > 0 {1666		throw("stopTheWorld: holding locks")1667	}16681669	lock(&sched.lock)1670	start := nanotime() // exclude time waiting for sched.lock from start and total time metrics.1671	sched.stopwait = gomaxprocs1672	sched.gcwaiting.Store(true)1673	preemptall()16741675	// Stop current P.1676	gp.m.p.ptr().status = _Pgcstop // Pgcstop is only diagnostic.1677	gp.m.p.ptr().gcStopTime = start1678	sched.stopwait--16791680	// Try to retake all P's in syscalls.1681	for _, pp := range allp {1682		if thread, ok := setBlockOnExitSyscall(pp); ok {1683			thread.gcstopP()1684			thread.resume()1685		}1686	}16871688	// Stop idle Ps.1689	now := nanotime()1690	for {1691		pp, _ := pidleget(now)1692		if pp == nil {1693			break1694		}1695		pp.status = _Pgcstop1696		pp.gcStopTime = nanotime()1697		sched.stopwait--1698	}1699	wait := sched.stopwait > 01700	unlock(&sched.lock)17011702	// Wait for remaining Ps to stop voluntarily.1703	if wait {1704		for {1705			// wait for 100us, then try to re-preempt in case of any races1706			if notetsleep(&sched.stopnote, 100*1000) {1707				noteclear(&sched.stopnote)1708				break1709			}1710			preemptall()1711		}1712	}17131714	finish := nanotime()1715	startTime := finish - start1716	if reason.isGC() {1717		sched.stwStoppingTimeGC.record(startTime)1718	} else {1719		sched.stwStoppingTimeOther.record(startTime)1720	}17211722	// Double-check we actually stopped everything, and all the invariants hold.1723	// Also accumulate all the time spent by each P in _Pgcstop up to the point1724	// where everything was stopped. This will be accumulated into the total pause1725	// CPU time by the caller.1726	stoppingCPUTime := int64(0)1727	bad := ""1728	if sched.stopwait != 0 {1729		bad = "stopTheWorld: not stopped (stopwait != 0)"1730	} else {1731		for _, pp := range allp {1732			if pp.status != _Pgcstop {1733				bad = "stopTheWorld: not stopped (status != _Pgcstop)"1734			}1735			if pp.gcStopTime == 0 && bad == "" {1736				bad = "stopTheWorld: broken CPU time accounting"1737			}1738			stoppingCPUTime += finish - pp.gcStopTime1739			pp.gcStopTime = 01740		}1741	}1742	if freezing.Load() {1743		// Some other thread is panicking. This can cause the1744		// sanity checks above to fail if the panic happens in1745		// the signal handler on a stopped thread. Either way,1746		// we should halt this thread.1747		lock(&deadlock)1748		lock(&deadlock)1749	}1750	if bad != "" {1751		throw(bad)1752	}17531754	worldStopped()17551756	// Switch back to _Grunning, now that the world is stopped.1757	casgstatus(getg().m.curg, _Gwaiting, _Grunning)17581759	return worldStop{1760		reason:           reason,1761		startedStopping:  start,1762		finishedStopping: finish,1763		stoppingCPUTime:  stoppingCPUTime,1764	}1765}17661767// reason is the same STW reason passed to stopTheWorld. start is the start1768// time returned by stopTheWorld.1769//1770// now is the current time; prefer to pass 0 to capture a fresh timestamp.1771//1772// stattTheWorldWithSema returns now.1773func startTheWorldWithSema(now int64, w worldStop) int64 {1774	assertWorldStopped()17751776	mp := acquirem() // disable preemption because it can be holding p in a local var1777	if netpollinited() {1778		list, delta := netpoll(0) // non-blocking1779		injectglist(&list)1780		netpollAdjustWaiters(delta)1781	}1782	lock(&sched.lock)17831784	procs := gomaxprocs1785	if newprocs != 0 {1786		procs = newprocs1787		newprocs = 01788	}1789	p1 := procresize(procs)1790	sched.gcwaiting.Store(false)1791	if sched.sysmonwait.Load() {1792		sched.sysmonwait.Store(false)1793		notewakeup(&sched.sysmonnote)1794	}1795	unlock(&sched.lock)17961797	worldStarted()17981799	for p1 != nil {1800		p := p11801		p1 = p1.link.ptr()1802		if p.m != 0 {1803			mp := p.m.ptr()1804			p.m = 01805			if mp.nextp != 0 {1806				throw("startTheWorld: inconsistent mp->nextp")1807			}1808			mp.nextp.set(p)1809			notewakeup(&mp.park)1810		} else {1811			// Start M to run P.  Do not start another M below.1812			newm(nil, p, -1)1813		}1814	}18151816	// Capture start-the-world time before doing clean-up tasks.1817	if now == 0 {1818		now = nanotime()1819	}1820	totalTime := now - w.startedStopping1821	if w.reason.isGC() {1822		sched.stwTotalTimeGC.record(totalTime)1823	} else {1824		sched.stwTotalTimeOther.record(totalTime)1825	}1826	trace := traceAcquire()1827	if trace.ok() {1828		trace.STWDone()1829		traceRelease(trace)1830	}18311832	// Wakeup an additional proc in case we have excessive runnable goroutines1833	// in local queues or in the global queue. If we don't, the proc will park itself.1834	// If we have lots of excessive work, resetspinning will unpark additional procs as necessary.1835	wakep()18361837	releasem(mp)18381839	return now1840}18411842// usesLibcall indicates whether this runtime performs system calls1843// via libcall.1844func usesLibcall() bool {1845	switch GOOS {1846	case "aix", "darwin", "illumos", "ios", "openbsd", "solaris", "windows":1847		return true1848	}1849	return false1850}18511852// mStackIsSystemAllocated indicates whether this runtime starts on a1853// system-allocated stack.1854func mStackIsSystemAllocated() bool {1855	switch GOOS {1856	case "aix", "darwin", "plan9", "illumos", "ios", "openbsd", "solaris", "windows":1857		return true1858	}1859	return false1860}18611862// mstart is the entry-point for new Ms.1863// It is written in assembly, uses ABI0, is marked TOPFRAME, and calls mstart0.1864func mstart()18651866// mstart0 is the Go entry-point for new Ms.1867// This must not split the stack because we may not even have stack1868// bounds set up yet.1869//1870// May run during STW (because it doesn't have a P yet), so write1871// barriers are not allowed.1872//1873//go:nosplit1874//go:nowritebarrierrec1875func mstart0() {1876	gp := getg()18771878	osStack := gp.stack.lo == 01879	if osStack {1880		// Initialize stack bounds from system stack.1881		// Cgo may have left stack size in stack.hi.1882		// minit may update the stack bounds.1883		//1884		// Note: these bounds may not be very accurate.1885		// We set hi to &size, but there are things above1886		// it. The 1024 is supposed to compensate this,1887		// but is somewhat arbitrary.1888		size := gp.stack.hi1889		if size == 0 {1890			size = 16384 * sys.StackGuardMultiplier1891		}1892		gp.stack.hi = uintptr(noescape(unsafe.Pointer(&size)))1893		gp.stack.lo = gp.stack.hi - size + 10241894	}1895	// Initialize stack guard so that we can start calling regular1896	// Go code.1897	gp.stackguard0 = gp.stack.lo + stackGuard1898	// This is the g0, so we can also call go:systemstack1899	// functions, which check stackguard1.1900	gp.stackguard1 = gp.stackguard01901	mstart1()19021903	// Exit this thread.1904	if mStackIsSystemAllocated() {1905		// Windows, Solaris, illumos, Darwin, AIX and Plan 9 always system-allocate1906		// the stack, but put it in gp.stack before mstart,1907		// so the logic above hasn't set osStack yet.1908		osStack = true1909	}1910	mexit(osStack)1911}19121913// The go:noinline is to guarantee the sys.GetCallerPC/sys.GetCallerSP below are safe,1914// so that we can set up g0.sched to return to the call of mstart1 above.1915//1916//go:noinline1917func mstart1() {1918	gp := getg()19191920	if gp != gp.m.g0 {1921		throw("bad runtime·mstart")1922	}19231924	// Set up m.g0.sched as a label returning to just1925	// after the mstart1 call in mstart0 above, for use by goexit0 and mcall.1926	// We're never coming back to mstart1 after we call schedule,1927	// so other calls can reuse the current frame.1928	// And goexit0 does a gogo that needs to return from mstart11929	// and let mstart0 exit the thread.1930	gp.sched.g = guintptr(unsafe.Pointer(gp))1931	gp.sched.pc = sys.GetCallerPC()1932	gp.sched.sp = sys.GetCallerSP()1933	gp.sched.bp = getcallerfp()19341935	asminit()1936	minit()19371938	// Install signal handlers; after minit so that minit can1939	// prepare the thread to be able to handle the signals.1940	if gp.m == &m0 {1941		mstartm0()1942	}19431944	if debug.dataindependenttiming == 1 {1945		sys.EnableDIT()1946	}19471948	if fn := gp.m.mstartfn; fn != nil {1949		fn()1950	}19511952	if gp.m != &m0 {1953		acquirep(gp.m.nextp.ptr())1954		gp.m.nextp = 01955	}1956	schedule()1957}19581959// mstartm0 implements part of mstart1 that only runs on the m0.1960//1961// Write barriers are allowed here because we know the GC can't be1962// running yet, so they'll be no-ops.1963//1964//go:yeswritebarrierrec1965func mstartm0() {1966	// Create an extra M for callbacks on threads not created by Go.1967	// An extra M is also needed on Windows for callbacks created by1968	// syscall.NewCallback. See issue #6751 for details.1969	if (iscgo || GOOS == "windows") && !cgoHasExtraM {1970		cgoHasExtraM = true1971		newextram()1972	}1973	initsig(false)1974}19751976// mPark causes a thread to park itself, returning once woken.1977//1978//go:nosplit1979func mPark() {1980	gp := getg()1981	// This M might stay parked through an entire GC cycle.1982	// Erase any leftovers on the signal stack.1983	if goexperiment.RuntimeSecret {1984		eraseSecretsSignalStk()1985	}1986	notesleep(&gp.m.park)1987	noteclear(&gp.m.park)1988}19891990// mexit tears down and exits the current thread.1991//1992// Don't call this directly to exit the thread, since it must run at1993// the top of the thread stack. Instead, use gogo(&gp.m.g0.sched) to1994// unwind the stack to the point that exits the thread.1995//1996// It is entered with m.p != nil, so write barriers are allowed. It1997// will release the P before exiting.1998//1999//go:yeswritebarrierrec2000func mexit(osStack bool) {