1// Copyright 2009 The Go Authors. All rights reserved.2// Use of this source code is governed by a BSD-style3// license that can be found in the LICENSE file.45#include "go_asm.h"6#include "go_tls.h"7#include "funcdata.h"8#include "textflag.h"9#include "cgo/abi_amd64.h"1011// _rt0_amd64 is common startup code for most amd64 systems when using12// internal linking. This is the entry point for the program from the13// kernel for an ordinary -buildmode=exe program. The stack holds the14// number of arguments and the C-style argv.15TEXT _rt0_amd64(SB),NOSPLIT,$-816	MOVQ	0(SP), DI	// argc17	LEAQ	8(SP), SI	// argv18	JMP	runtime·rt0_go(SB)1920// main is common startup code for most amd64 systems when using21// external linking. The C startup code will call the symbol "main"22// passing argc and argv in the usual C ABI registers DI and SI.23TEXT main(SB),NOSPLIT,$-824	JMP	runtime·rt0_go(SB)2526// _rt0_amd64_lib is common startup code for most amd64 systems when27// using -buildmode=c-archive or -buildmode=c-shared. The linker will28// arrange to invoke this function as a global constructor (for29// c-archive) or when the shared library is loaded (for c-shared).30// We expect argc and argv to be passed in the usual C ABI registers31// DI and SI.32TEXT _rt0_amd64_lib(SB),NOSPLIT|NOFRAME,$033	// Transition from C ABI to Go ABI.34	PUSH_REGS_HOST_TO_ABI0()3536	MOVQ	DI, _rt0_amd64_lib_argc<>(SB)37	MOVQ	SI, _rt0_amd64_lib_argv<>(SB)3839#ifdef GOOS_windows40	// Set up a dummy TLS value on Windows so that the autogenerated41	// ABI wrappers don't crash when trying to load G from TLS before42	// wintls has set up the real TLS slot in rt0_go.43	MOVQ	$zeroTLS<>(SB), DI44	CALL	runtime·settls(SB)45#endif4647	CALL	runtime·libInit(SB)4849	POP_REGS_HOST_TO_ABI0()50	RET5152// rt0_lib_go initializes the Go runtime.53// This is started in a separate thread by _rt0_amd64_lib.54TEXT runtime·rt0_lib_go<ABIInternal>(SB),NOSPLIT,$055	MOVQ	_rt0_amd64_lib_argc<>(SB), DI56	MOVQ	_rt0_amd64_lib_argv<>(SB), SI57	JMP	runtime·rt0_go(SB)5859DATA _rt0_amd64_lib_argc<>(SB)/8, $060GLOBL _rt0_amd64_lib_argc<>(SB),NOPTR, $861DATA _rt0_amd64_lib_argv<>(SB)/8, $062GLOBL _rt0_amd64_lib_argv<>(SB),NOPTR, $86364#ifdef GOAMD64_v265DATA bad_cpu_msg<>+0x00(SB)/84, $"This program can only be run on AMD64 processors with v2 microarchitecture support.\n"66#endif6768#ifdef GOAMD64_v369DATA bad_cpu_msg<>+0x00(SB)/84, $"This program can only be run on AMD64 processors with v3 microarchitecture support.\n"70#endif7172#ifdef GOAMD64_v473DATA bad_cpu_msg<>+0x00(SB)/84, $"This program can only be run on AMD64 processors with v4 microarchitecture support.\n"74#endif7576GLOBL bad_cpu_msg<>(SB), RODATA, $847778// Define a list of AMD64 microarchitecture level features79// https://en.wikipedia.org/wiki/X86-64#Microarchitecture_levels8081                     // SSE3     SSSE3    CMPXCHNG16 SSE4.1    SSE4.2    POPCNT82#define V2_FEATURES_CX (1 << 0 | 1 << 9 | 1 << 13  | 1 << 19 | 1 << 20 | 1 << 23)83                         // LAHF/SAHF84#define V2_EXT_FEATURES_CX (1 << 0)85                                      // FMA       MOVBE     OSXSAVE   AVX       F16C86#define V3_FEATURES_CX (V2_FEATURES_CX | 1 << 12 | 1 << 22 | 1 << 27 | 1 << 28 | 1 << 29)87                                              // ABM (FOR LZNCT)88#define V3_EXT_FEATURES_CX (V2_EXT_FEATURES_CX | 1 << 5)89                         // BMI1     AVX2     BMI290#define V3_EXT_FEATURES_BX (1 << 3 | 1 << 5 | 1 << 8)91                       // XMM      YMM92#define V3_OS_SUPPORT_AX (1 << 1 | 1 << 2)9394#define V4_FEATURES_CX V3_FEATURES_CX9596#define V4_EXT_FEATURES_CX V3_EXT_FEATURES_CX97                                              // AVX512F   AVX512DQ  AVX512CD  AVX512BW  AVX512VL98#define V4_EXT_FEATURES_BX (V3_EXT_FEATURES_BX | 1 << 16 | 1 << 17 | 1 << 28 | 1 << 30 | 1 << 31)99                                          // OPMASK   ZMM100#define V4_OS_SUPPORT_AX (V3_OS_SUPPORT_AX | 1 << 5 | (1 << 6 | 1 << 7))101102#ifdef GOAMD64_v2103#define NEED_MAX_CPUID 0x80000001104#define NEED_FEATURES_CX V2_FEATURES_CX105#define NEED_EXT_FEATURES_CX V2_EXT_FEATURES_CX106#endif107108#ifdef GOAMD64_v3109#define NEED_MAX_CPUID 0x80000001110#define NEED_FEATURES_CX V3_FEATURES_CX111#define NEED_EXT_FEATURES_CX V3_EXT_FEATURES_CX112#define NEED_EXT_FEATURES_BX V3_EXT_FEATURES_BX113#define NEED_OS_SUPPORT_AX V3_OS_SUPPORT_AX114#endif115116#ifdef GOAMD64_v4117#define NEED_MAX_CPUID 0x80000001118#define NEED_FEATURES_CX V4_FEATURES_CX119#define NEED_EXT_FEATURES_CX V4_EXT_FEATURES_CX120#define NEED_EXT_FEATURES_BX V4_EXT_FEATURES_BX121122// Darwin requires a different approach to check AVX512 support, see CL 285572.123#ifdef GOOS_darwin124#define NEED_OS_SUPPORT_AX V3_OS_SUPPORT_AX125// These values are from:126// https://github.com/apple/darwin-xnu/blob/xnu-4570.1.46/osfmk/i386/cpu_capabilities.h127#define commpage64_base_address         0x00007fffffe00000128#define commpage64_cpu_capabilities64   (commpage64_base_address+0x010)129#define commpage64_version              (commpage64_base_address+0x01E)130#define AVX512F                         0x0000004000000000131#define AVX512CD                        0x0000008000000000132#define AVX512DQ                        0x0000010000000000133#define AVX512BW                        0x0000020000000000134#define AVX512VL                        0x0000100000000000135#define NEED_DARWIN_SUPPORT             (AVX512F | AVX512DQ | AVX512CD | AVX512BW | AVX512VL)136#else137#define NEED_OS_SUPPORT_AX V4_OS_SUPPORT_AX138#endif139140#endif141142TEXT runtime·rt0_go(SB),NOSPLIT|NOFRAME|TOPFRAME,$0143	// copy arguments forward on an even stack144	MOVQ	DI, AX		// argc145	MOVQ	SI, BX		// argv146	SUBQ	$(5*8), SP		// 3args 2auto147	ANDQ	$~15, SP148	MOVQ	AX, 24(SP)149	MOVQ	BX, 32(SP)150151	// This is typically the entry point for Go programs.152	// Call stack unwinding must not proceed past this frame.153	// Set the frame pointer register to 0 so that frame pointer-based unwinders154	// (which don't use debug info for performance reasons)155	// won't attempt to unwind past this function.156	// See go.dev/issue/63630157	MOVQ	$0, BP158159	// create istack out of the given (operating system) stack.160	// _cgo_init may update stackguard.161	MOVQ	$runtime·g0(SB), DI162	LEAQ	(-64*1024)(SP), BX163	MOVQ	BX, g_stackguard0(DI)164	MOVQ	BX, g_stackguard1(DI)165	MOVQ	BX, (g_stack+stack_lo)(DI)166	MOVQ	SP, (g_stack+stack_hi)(DI)167168	// find out information about the processor we're on169	MOVL	$0, AX170	CPUID171	CMPL	AX, $0172	JE	nocpuinfo173174	CMPL	BX, $0x756E6547  // "Genu"175	JNE	notintel176	CMPL	DX, $0x49656E69  // "ineI"177	JNE	notintel178	CMPL	CX, $0x6C65746E  // "ntel"179	JNE	notintel180	MOVB	$1, runtime·isIntel(SB)181182notintel:183	// Load EAX=1 cpuid flags184	MOVL	$1, AX185	CPUID186	MOVL	AX, runtime·processorVersionInfo(SB)187188nocpuinfo:189	// if there is an _cgo_init, call it.190	MOVQ	_cgo_init(SB), AX191	TESTQ	AX, AX192	JZ	needtls193	// arg 1: g0, already in DI194	MOVQ	$setg_gcc<>(SB), SI // arg 2: setg_gcc195	MOVQ	$0, DX	// arg 3, 4: not used when using platform's TLS196	MOVQ	$0, CX197#ifdef GOOS_android198	MOVQ	$runtime·tls_g(SB), DX 	// arg 3: &tls_g199	// arg 4: TLS base, stored in slot 0 (Android's TLS_SLOT_SELF).200	// Compensate for tls_g (+16).201	MOVQ	-16(TLS), CX202#endif203#ifdef GOOS_windows204	MOVQ	$runtime·tls_g(SB), DX 	// arg 3: &tls_g205	// Adjust for the Win64 calling convention.206	MOVQ	CX, R9 // arg 4207	MOVQ	DX, R8 // arg 3208	MOVQ	SI, DX // arg 2209	MOVQ	DI, CX // arg 1210#endif211	CALL	AX212213	// update stackguard after _cgo_init214	MOVQ	$runtime·g0(SB), CX215	MOVQ	(g_stack+stack_lo)(CX), AX216	ADDQ	$const_stackGuard, AX217	MOVQ	AX, g_stackguard0(CX)218	MOVQ	AX, g_stackguard1(CX)219220#ifndef GOOS_windows221	JMP ok222#endif223needtls:224#ifdef GOOS_plan9225	// skip TLS setup on Plan 9226	JMP ok227#endif228#ifdef GOOS_solaris229	// skip TLS setup on Solaris230	JMP ok231#endif232#ifdef GOOS_illumos233	// skip TLS setup on illumos234	JMP ok235#endif236#ifdef GOOS_darwin237	// skip TLS setup on Darwin238	JMP ok239#endif240#ifdef GOOS_openbsd241	// skip TLS setup on OpenBSD242	JMP ok243#endif244245#ifdef GOOS_windows246	CALL	runtime·wintls(SB)247#endif248249	LEAQ	runtime·m0+m_tls(SB), DI250	CALL	runtime·settls(SB)251252	// store through it, to make sure it works253	get_tls(BX)254	MOVQ	$0x123, g(BX)255	MOVQ	runtime·m0+m_tls(SB), AX256	CMPQ	AX, $0x123257	JEQ 2(PC)258	CALL	runtime·abort(SB)259ok:260	// set the per-goroutine and per-mach "registers"261	get_tls(BX)262	LEAQ	runtime·g0(SB), CX263	MOVQ	CX, g(BX)264	LEAQ	runtime·m0(SB), AX265266	// save m->g0 = g0267	MOVQ	CX, m_g0(AX)268	// save m0 to g0->m269	MOVQ	AX, g_m(CX)270271	CLD				// convention is D is always left cleared272273	// Check GOAMD64 requirements274	// We need to do this after setting up TLS, so that275	// we can report an error if there is a failure. See issue 49586.276#ifdef NEED_FEATURES_CX277	MOVL	$0, AX278	CPUID279	CMPL	AX, $0280	JE	bad_cpu281	MOVL	$1, AX282	CPUID283	ANDL	$NEED_FEATURES_CX, CX284	CMPL	CX, $NEED_FEATURES_CX285	JNE	bad_cpu286#endif287288#ifdef NEED_MAX_CPUID289	MOVL	$0x80000000, AX290	CPUID291	CMPL	AX, $NEED_MAX_CPUID292	JL	bad_cpu293#endif294295#ifdef NEED_EXT_FEATURES_BX296	MOVL	$7, AX297	MOVL	$0, CX298	CPUID299	ANDL	$NEED_EXT_FEATURES_BX, BX300	CMPL	BX, $NEED_EXT_FEATURES_BX301	JNE	bad_cpu302#endif303304#ifdef NEED_EXT_FEATURES_CX305	MOVL	$0x80000001, AX306	CPUID307	ANDL	$NEED_EXT_FEATURES_CX, CX308	CMPL	CX, $NEED_EXT_FEATURES_CX309	JNE	bad_cpu310#endif311312#ifdef NEED_OS_SUPPORT_AX313	XORL    CX, CX314	XGETBV315	ANDL	$NEED_OS_SUPPORT_AX, AX316	CMPL	AX, $NEED_OS_SUPPORT_AX317	JNE	bad_cpu318#endif319320#ifdef NEED_DARWIN_SUPPORT321	MOVQ	$commpage64_version, BX322	CMPW	(BX), $13  // cpu_capabilities64 undefined in versions < 13323	JL	bad_cpu324	MOVQ	$commpage64_cpu_capabilities64, BX325	MOVQ	(BX), BX326	MOVQ	$NEED_DARWIN_SUPPORT, CX327	ANDQ	CX, BX328	CMPQ	BX, CX329	JNE	bad_cpu330#endif331332	CALL	runtime·check(SB)333334	MOVL	24(SP), AX		// copy argc335	MOVL	AX, 0(SP)336	MOVQ	32(SP), AX		// copy argv337	MOVQ	AX, 8(SP)338	CALL	runtime·args(SB)339	CALL	runtime·osinit(SB)340	CALL	runtime·schedinit(SB)341342	// create a new goroutine to start program343	MOVQ	$runtime·mainPC(SB), AX		// entry344	PUSHQ	AX345	CALL	runtime·newproc(SB)346	POPQ	AX347348	// start this M349	CALL	runtime·mstart(SB)350351	CALL	runtime·abort(SB)	// mstart should never return352	RET353354bad_cpu: // show that the program requires a certain microarchitecture level.355	MOVQ	$2, 0(SP)356	MOVQ	$bad_cpu_msg<>(SB), AX357	MOVQ	AX, 8(SP)358	MOVQ	$84, 16(SP)359	CALL	runtime·write(SB)360	MOVQ	$1, 0(SP)361	CALL	runtime·exit(SB)362	CALL	runtime·abort(SB)363	RET364365	// Prevent dead-code elimination of debugCallV2 and debugPinnerV1, which are366	// intended to be called by debuggers.367	MOVQ	$runtime·debugPinnerV1<ABIInternal>(SB), AX368	MOVQ	$runtime·debugCallV2<ABIInternal>(SB), AX369	RET370371// mainPC is a function value for runtime.main, to be passed to newproc.372// The reference to runtime.main is made via ABIInternal, since the373// actual function (not the ABI0 wrapper) is needed by newproc.374DATA	runtime·mainPC+0(SB)/8,$runtime·main<ABIInternal>(SB)375GLOBL	runtime·mainPC(SB),RODATA,$8376377TEXT runtime·breakpoint(SB),NOSPLIT,$0-0378	BYTE	$0xcc379	RET380381TEXT runtime·asminit(SB),NOSPLIT,$0-0382	// No per-thread init.383	RET384385TEXT runtime·mstart(SB),NOSPLIT|TOPFRAME|NOFRAME,$0386	// This is the root frame of new Go-created OS threads.387	// Call stack unwinding must not proceed past this frame.388	// Set the frame pointer register to 0 so that frame pointer-based unwinders389	// (which don't use debug info for performance reasons)390	// won't attempt to unwind past this function.391	// See go.dev/issue/63630392	MOVD	$0, BP393	CALL	runtime·mstart0(SB)394	RET // not reached395396/*397 *  go-routine398 */399400// func gogo(buf *gobuf)401// restore state from Gobuf; longjmp402TEXT runtime·gogo(SB), NOSPLIT, $0-8403	MOVQ	buf+0(FP), BX		// gobuf404	MOVQ	gobuf_g(BX), DX405	MOVQ	0(DX), CX		// make sure g != nil406	JMP	gogo<>(SB)407408TEXT gogo<>(SB), NOSPLIT, $0409	get_tls(CX)410	MOVQ	DX, g(CX)411	MOVQ	DX, R14		// set the g register412	MOVQ	gobuf_sp(BX), SP	// restore SP413	MOVQ	gobuf_ctxt(BX), DX414	MOVQ	gobuf_bp(BX), BP415	MOVQ	$0, gobuf_sp(BX)	// clear to help garbage collector416	MOVQ	$0, gobuf_ctxt(BX)417	MOVQ	$0, gobuf_bp(BX)418	MOVQ	gobuf_pc(BX), BX419	JMP	BX420421// func mcall(fn func(*g))422// Switch to m->g0's stack, call fn(g).423// Fn must never return. It should gogo(&g->sched)424// to keep running g.425TEXT runtime·mcall<ABIInternal>(SB), NOSPLIT, $0-8426#ifdef GOEXPERIMENT_runtimesecret427	CMPL	g_secret(R14), $0428	JEQ	nosecret429	CALL	·secretEraseRegistersMcall(SB)430nosecret:431#endif432433	MOVQ	AX, DX	// DX = fn434435	// Save state in g->sched. The caller's SP and PC are restored by gogo to436	// resume execution in the caller's frame (implicit return). The caller's BP437	// is also restored to support frame pointer unwinding.438	MOVQ	SP, BX	// hide (SP) reads from vet439	MOVQ	8(BX), BX	// caller's PC440	MOVQ	BX, (g_sched+gobuf_pc)(R14)441	LEAQ	fn+0(FP), BX	// caller's SP442	MOVQ	BX, (g_sched+gobuf_sp)(R14)443	// Get the caller's frame pointer by dereferencing BP. Storing BP as it is444	// can cause a frame pointer cycle, see CL 476235.445	MOVQ	(BP), BX // caller's BP446	MOVQ	BX, (g_sched+gobuf_bp)(R14)447448	// switch to m->g0 & its stack, call fn449	MOVQ	g_m(R14), BX450	MOVQ	m_g0(BX), SI	// SI = g.m.g0451	CMPQ	SI, R14	// if g == m->g0 call badmcall452	JNE	goodm453	JMP	runtime·badmcall(SB)454goodm:455	MOVQ	R14, AX		// AX (and arg 0) = g456	MOVQ	SI, R14		// g = g.m.g0457	get_tls(CX)		// Set G in TLS458	MOVQ	R14, g(CX)459	MOVQ	(g_sched+gobuf_sp)(R14), SP	// sp = g0.sched.sp460	MOVQ	$0, BP	// clear frame pointer, as caller may execute on another M461	PUSHQ	AX	// open up space for fn's arg spill slot462	MOVQ	0(DX), R12463	CALL	R12		// fn(g)464	// The Windows native stack unwinder incorrectly classifies the next instruction465	// as part of the function epilogue, producing a wrong call stack.466	// Add a NOP to work around this issue. See go.dev/issue/67007.467	BYTE	$0x90468	POPQ	AX469	JMP	runtime·badmcall2(SB)470	RET471472// systemstack_switch is a dummy routine that systemstack leaves at the bottom473// of the G stack. We need to distinguish the routine that474// lives at the bottom of the G stack from the one that lives475// at the top of the system stack because the one at the top of476// the system stack terminates the stack walk (see topofstack()).477// The frame layout needs to match systemstack478// so that it can pretend to be systemstack_switch.479TEXT runtime·systemstack_switch(SB), NOSPLIT, $0-0480	// Align for consistency with offset used in gosave_systemstack_switch481	PCALIGN	$8482	UNDEF483	// Make sure this function is not leaf,484	// so the frame is saved.485	CALL	runtime·abort(SB)486	RET487488// func systemstack(fn func())489TEXT runtime·systemstack(SB), NOSPLIT, $0-8490#ifdef GOEXPERIMENT_runtimesecret491	// If in secret mode, erase registers on transition492	// from G stack to M stack,493	get_tls(CX)494	MOVQ	g(CX), AX495	CMPL	g_secret(AX), $0496	JEQ	nosecret497	CALL	·secretEraseRegisters(SB)498nosecret:499#endif500501	MOVQ	fn+0(FP), DI	// DI = fn502	get_tls(CX)503	MOVQ	g(CX), AX	// AX = g504	MOVQ	g_m(AX), BX	// BX = m505506	CMPQ	AX, m_gsignal(BX)507	JEQ	noswitch508509	MOVQ	m_g0(BX), DX	// DX = g0510	CMPQ	AX, DX511	JEQ	noswitch512513	CMPQ	AX, m_curg(BX)514	JNE	bad515516	// Switch stacks.517	// The original frame pointer is stored in BP,518	// which is useful for stack unwinding.519	// Save our state in g->sched. Pretend to520	// be systemstack_switch if the G stack is scanned.521	CALL	gosave_systemstack_switch<>(SB)522523	// switch to g0524	MOVQ	DX, g(CX)525	MOVQ	DX, R14 // set the g register526	MOVQ	(g_sched+gobuf_sp)(DX), SP527528	// call target function529	MOVQ	DI, DX530	MOVQ	0(DI), DI531	CALL	DI532533	// switch back to g534	get_tls(CX)535	MOVQ	g(CX), AX536	MOVQ	g_m(AX), BX537	MOVQ	m_curg(BX), AX538	MOVQ	AX, g(CX)539	MOVQ	(g_sched+gobuf_sp)(AX), SP540	MOVQ	(g_sched+gobuf_bp)(AX), BP541	MOVQ	$0, (g_sched+gobuf_sp)(AX)542	MOVQ	$0, (g_sched+gobuf_bp)(AX)543	RET544545noswitch:546	// already on m stack; tail call the function547	// Using a tail call here cleans up tracebacks since we won't stop548	// at an intermediate systemstack.549	MOVQ	DI, DX550	MOVQ	0(DI), DI551	// The function epilogue is not called on a tail call.552	// Pop BP from the stack to simulate it.553	POPQ	BP554	JMP	DI555556bad:557	// Bad: g is not gsignal, not g0, not curg. What is it?558	MOVQ	$runtime·badsystemstack(SB), AX559	CALL	AX560	INT	$3561562// func switchToCrashStack0(fn func())563TEXT runtime·switchToCrashStack0<ABIInternal>(SB), NOSPLIT, $0-8564	MOVQ	g_m(R14), BX // curm565566	// set g to gcrash567	LEAQ	runtime·gcrash(SB), R14 // g = &gcrash568	MOVQ	BX, g_m(R14)            // g.m = curm569	MOVQ	R14, m_g0(BX)           // curm.g0 = g570	get_tls(CX)571	MOVQ	R14, g(CX)572573	// switch to crashstack574	MOVQ	(g_stack+stack_hi)(R14), BX575	SUBQ	$(4*8), BX576	MOVQ	BX, SP577578	// call target function579	MOVQ	AX, DX580	MOVQ	0(AX), AX581	CALL	AX582583	// should never return584	CALL	runtime·abort(SB)585	UNDEF586587/*588 * support for morestack589 */590591// Called during function prolog when more stack is needed.592//593// The traceback routines see morestack on a g0 as being594// the top of a stack (for example, morestack calling newstack595// calling the scheduler calling newm calling gc), so we must596// record an argument size. For that purpose, it has no arguments.597TEXT runtime·morestack(SB),NOSPLIT|NOFRAME,$0-0598	// Cannot grow scheduler stack (m->g0).599	get_tls(CX)600	MOVQ	g(CX), DI     // DI = g601	MOVQ	g_m(DI), BX   // BX = m602603	// Set g->sched to context in f.604	MOVQ	0(SP), AX // f's PC605	MOVQ	AX, (g_sched+gobuf_pc)(DI)606	LEAQ	8(SP), AX // f's SP607	MOVQ	AX, (g_sched+gobuf_sp)(DI)608	MOVQ	BP, (g_sched+gobuf_bp)(DI)609	MOVQ	DX, (g_sched+gobuf_ctxt)(DI)610611	MOVQ	m_g0(BX), SI  // SI = m.g0612	CMPQ	DI, SI613	JNE	3(PC)614	CALL	runtime·badmorestackg0(SB)615	CALL	runtime·abort(SB)616617	// Cannot grow signal stack (m->gsignal).618	MOVQ	m_gsignal(BX), SI619	CMPQ	DI, SI620	JNE	3(PC)621	CALL	runtime·badmorestackgsignal(SB)622	CALL	runtime·abort(SB)623624	// Called from f.625	// Set m->morebuf to f's caller.626	NOP	SP	// tell vet SP changed - stop checking offsets627	MOVQ	8(SP), AX	// f's caller's PC628	MOVQ	AX, (m_morebuf+gobuf_pc)(BX)629	LEAQ	16(SP), AX	// f's caller's SP630	MOVQ	AX, (m_morebuf+gobuf_sp)(BX)631	MOVQ	DI, (m_morebuf+gobuf_g)(BX)632633	// If in secret mode, erase registers on transition634	// from G stack to M stack,635#ifdef GOEXPERIMENT_runtimesecret636	CMPL	g_secret(DI), $0637	JEQ	nosecret638	CALL	·secretEraseRegisters(SB)639	get_tls(CX)640	MOVQ	g(CX), DI     // DI = g641	MOVQ	g_m(DI), BX   // BX = m642nosecret:643#endif644645	// Call newstack on m->g0's stack.646	MOVQ	m_g0(BX), BX647	MOVQ	BX, g(CX)648	MOVQ	(g_sched+gobuf_sp)(BX), SP649	MOVQ	$0, BP			// clear frame pointer, as caller may execute on another M650	CALL	runtime·newstack(SB)651	CALL	runtime·abort(SB)	// crash if newstack returns652	RET653654// morestack but not preserving ctxt.655TEXT runtime·morestack_noctxt(SB),NOSPLIT,$0656	MOVL	$0, DX657	JMP	runtime·morestack(SB)658659// spillArgs stores return values from registers to a *internal/abi.RegArgs in R12.660TEXT ·spillArgs(SB),NOSPLIT,$0-0661	MOVQ AX, 0(R12)662	MOVQ BX, 8(R12)663	MOVQ CX, 16(R12)664	MOVQ DI, 24(R12)665	MOVQ SI, 32(R12)666	MOVQ R8, 40(R12)667	MOVQ R9, 48(R12)668	MOVQ R10, 56(R12)669	MOVQ R11, 64(R12)670	MOVQ X0, 72(R12)671	MOVQ X1, 80(R12)672	MOVQ X2, 88(R12)673	MOVQ X3, 96(R12)674	MOVQ X4, 104(R12)675	MOVQ X5, 112(R12)676	MOVQ X6, 120(R12)677	MOVQ X7, 128(R12)678	MOVQ X8, 136(R12)679	MOVQ X9, 144(R12)680	MOVQ X10, 152(R12)681	MOVQ X11, 160(R12)682	MOVQ X12, 168(R12)683	MOVQ X13, 176(R12)684	MOVQ X14, 184(R12)685	RET686687// unspillArgs loads args into registers from a *internal/abi.RegArgs in R12.688TEXT ·unspillArgs(SB),NOSPLIT,$0-0689	MOVQ 0(R12), AX690	MOVQ 8(R12), BX691	MOVQ 16(R12), CX692	MOVQ 24(R12), DI693	MOVQ 32(R12), SI694	MOVQ 40(R12), R8695	MOVQ 48(R12), R9696	MOVQ 56(R12), R10697	MOVQ 64(R12), R11698	MOVQ 72(R12), X0699	MOVQ 80(R12), X1700	MOVQ 88(R12), X2701	MOVQ 96(R12), X3702	MOVQ 104(R12), X4703	MOVQ 112(R12), X5704	MOVQ 120(R12), X6705	MOVQ 128(R12), X7706	MOVQ 136(R12), X8707	MOVQ 144(R12), X9708	MOVQ 152(R12), X10709	MOVQ 160(R12), X11710	MOVQ 168(R12), X12711	MOVQ 176(R12), X13712	MOVQ 184(R12), X14713	RET714715// reflectcall: call a function with the given argument list716// func call(stackArgsType *_type, f *FuncVal, stackArgs *byte, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs).717// we don't have variable-sized frames, so we use a small number718// of constant-sized-frame functions to encode a few bits of size in the pc.719// Caution: ugly multiline assembly macros in your future!720721#define DISPATCH(NAME,MAXSIZE)		\722	CMPQ	CX, $MAXSIZE;		\723	JA	3(PC);			\724	MOVQ	$NAME(SB), AX;		\725	JMP	AX726// Note: can't just "JMP NAME(SB)" - bad inlining results.727728TEXT ·reflectcall(SB), NOSPLIT, $0-48729	MOVLQZX frameSize+32(FP), CX730	DISPATCH(runtime·call16, 16)731	DISPATCH(runtime·call32, 32)732	DISPATCH(runtime·call64, 64)733	DISPATCH(runtime·call128, 128)734	DISPATCH(runtime·call256, 256)735	DISPATCH(runtime·call512, 512)736	DISPATCH(runtime·call1024, 1024)737	DISPATCH(runtime·call2048, 2048)738	DISPATCH(runtime·call4096, 4096)739	DISPATCH(runtime·call8192, 8192)740	DISPATCH(runtime·call16384, 16384)741	DISPATCH(runtime·call32768, 32768)742	DISPATCH(runtime·call65536, 65536)743	DISPATCH(runtime·call131072, 131072)744	DISPATCH(runtime·call262144, 262144)745	DISPATCH(runtime·call524288, 524288)746	DISPATCH(runtime·call1048576, 1048576)747	DISPATCH(runtime·call2097152, 2097152)748	DISPATCH(runtime·call4194304, 4194304)749	DISPATCH(runtime·call8388608, 8388608)750	DISPATCH(runtime·call16777216, 16777216)751	DISPATCH(runtime·call33554432, 33554432)752	DISPATCH(runtime·call67108864, 67108864)753	DISPATCH(runtime·call134217728, 134217728)754	DISPATCH(runtime·call268435456, 268435456)755	DISPATCH(runtime·call536870912, 536870912)756	DISPATCH(runtime·call1073741824, 1073741824)757	MOVQ	$runtime·badreflectcall(SB), AX758	JMP	AX759760#define CALLFN(NAME,MAXSIZE)			\761TEXT NAME(SB), WRAPPER, $MAXSIZE-48;		\762	NO_LOCAL_POINTERS;			\763	/* copy arguments to stack */		\764	MOVQ	stackArgs+16(FP), SI;		\765	MOVLQZX stackArgsSize+24(FP), CX;		\766	MOVQ	SP, DI;				\767	REP;MOVSB;				\768	/* set up argument registers */		\769	MOVQ    regArgs+40(FP), R12;		\770	CALL    ·unspillArgs(SB);		\771	/* call function */			\772	MOVQ	f+8(FP), DX;			\773	PCDATA  $PCDATA_StackMapIndex, $0;	\774	MOVQ	(DX), R12;			\775	CALL	R12;				\776	/* copy register return values back */		\777	MOVQ    regArgs+40(FP), R12;		\778	CALL    ·spillArgs(SB);		\779	MOVLQZX	stackArgsSize+24(FP), CX;		\780	MOVLQZX	stackRetOffset+28(FP), BX;		\781	MOVQ	stackArgs+16(FP), DI;		\782	MOVQ	stackArgsType+0(FP), DX;		\783	MOVQ	SP, SI;				\784	ADDQ	BX, DI;				\785	ADDQ	BX, SI;				\786	SUBQ	BX, CX;				\787	CALL	callRet<>(SB);			\788	RET789790// callRet copies return values back at the end of call*. This is a791// separate function so it can allocate stack space for the arguments792// to reflectcallmove. It does not follow the Go ABI; it expects its793// arguments in registers.794TEXT callRet<>(SB), NOSPLIT, $40-0795	NO_LOCAL_POINTERS796	MOVQ	DX, 0(SP)797	MOVQ	DI, 8(SP)798	MOVQ	SI, 16(SP)799	MOVQ	CX, 24(SP)800	MOVQ	R12, 32(SP)801	CALL	runtime·reflectcallmove(SB)802	RET803804CALLFN(·call16, 16)805CALLFN(·call32, 32)806CALLFN(·call64, 64)807CALLFN(·call128, 128)808CALLFN(·call256, 256)809CALLFN(·call512, 512)810CALLFN(·call1024, 1024)811CALLFN(·call2048, 2048)812CALLFN(·call4096, 4096)813CALLFN(·call8192, 8192)814CALLFN(·call16384, 16384)815CALLFN(·call32768, 32768)816CALLFN(·call65536, 65536)817CALLFN(·call131072, 131072)818CALLFN(·call262144, 262144)819CALLFN(·call524288, 524288)820CALLFN(·call1048576, 1048576)821CALLFN(·call2097152, 2097152)822CALLFN(·call4194304, 4194304)823CALLFN(·call8388608, 8388608)824CALLFN(·call16777216, 16777216)825CALLFN(·call33554432, 33554432)826CALLFN(·call67108864, 67108864)827CALLFN(·call134217728, 134217728)828CALLFN(·call268435456, 268435456)829CALLFN(·call536870912, 536870912)830CALLFN(·call1073741824, 1073741824)831832TEXT runtime·procyieldAsm(SB),NOSPLIT,$0-0833	MOVL	cycles+0(FP), AX834	TESTL	AX, AX835	JZ	done836again:837	PAUSE838	SUBL	$1, AX839	JNZ	again840done:841	RET842843844TEXT ·publicationBarrier<ABIInternal>(SB),NOSPLIT,$0-0845	// Stores are already ordered on x86, so this is just a846	// compile barrier.847	RET848849// Save state of caller into g->sched,850// but using fake PC from systemstack_switch.851// Must only be called from functions with frame pointer852// and without locals ($0) or else unwinding from853// systemstack_switch is incorrect.854// Smashes R9.855TEXT gosave_systemstack_switch<>(SB),NOSPLIT|NOFRAME,$0856	// Take systemstack_switch PC and add 8 bytes to skip857	// the prologue. Keep 8 bytes offset consistent with858	// PCALIGN $8 in systemstack_swtich, pointing start of859	// UNDEF instruction beyond prologue.860	MOVQ	$runtime·systemstack_switch+8(SB), R9861	MOVQ	R9, (g_sched+gobuf_pc)(R14)862	LEAQ	8(SP), R9863	MOVQ	R9, (g_sched+gobuf_sp)(R14)864	MOVQ	BP, (g_sched+gobuf_bp)(R14)865	// Assert ctxt is zero. See func save.866	MOVQ	(g_sched+gobuf_ctxt)(R14), R9867	TESTQ	R9, R9868	JZ	2(PC)869	CALL	runtime·abort(SB)870	RET871872// func asmcgocall_no_g(fn, arg unsafe.Pointer)873// Call fn(arg) aligned appropriately for the gcc ABI.874// Called on a system stack, and there may be no g yet (during needm).875TEXT ·asmcgocall_no_g(SB),NOSPLIT,$32-16876	MOVQ	fn+0(FP), AX877	MOVQ	arg+8(FP), BX878	MOVQ	SP, DX879	ANDQ	$~15, SP	// alignment880	MOVQ	DX, 8(SP)881	MOVQ	BX, DI		// DI = first argument in AMD64 ABI882	MOVQ	BX, CX		// CX = first argument in Win64883	CALL	AX884	MOVQ	8(SP), DX885	MOVQ	DX, SP886	RET887888// asmcgocall_landingpad calls AX with BX as argument.889// Must be called on the system stack.890TEXT ·asmcgocall_landingpad(SB),NOSPLIT,$0-0891#ifdef GOOS_windows892	// Make sure we have enough room for 4 stack-backed fast-call893	// registers as per Windows amd64 calling convention.894	ADJSP	$32895	// On Windows, asmcgocall_landingpad acts as landing pad for exceptions896	// thrown in the cgo call. Exceptions that reach this function will be897	// handled by runtime.sehtramp thanks to the SEH metadata added898	// by the compiler.899	// Note that runtime.sehtramp can't be attached directly to asmcgocall900	// because its initial stack pointer can be outside the system stack bounds,901	// and Windows stops the stack unwinding without calling the exception handler902	// when it reaches that point.903	MOVQ	BX, CX		// CX = first argument in Win64904	CALL	AX905	// The exception handler is not called if the next instruction is part of906	// the epilogue, which includes the RET instruction, so we need to add a NOP here.907	BYTE	$0x90908	ADJSP	$-32909	RET910#endif911	// Tail call AX on non-Windows, as the extra stack frame is not needed.912	MOVQ	BX, DI		// DI = first argument in AMD64 ABI913	JMP	AX914915// func asmcgocall(fn, arg unsafe.Pointer) int32916// Call fn(arg) on the scheduler stack,917// aligned appropriately for the gcc ABI.918// See cgocall.go for more details.919TEXT ·asmcgocall(SB),NOSPLIT,$0-20920	// Figure out if we need to switch to m->g0 stack.921	// We get called to create new OS threads too, and those922	// come in on the m->g0 stack already. Or we might already923	// be on the m->gsignal stack.924	get_tls(CX)925	MOVQ	g(CX), DI926	CMPQ	DI, $0927	JEQ	nosave928	MOVQ	g_m(DI), R8929	MOVQ	m_gsignal(R8), SI930	CMPQ	DI, SI931	JEQ	nosave932	MOVQ	m_g0(R8), SI933	CMPQ	DI, SI934	JEQ	nosave935936	// Running on a user G937	// Figure out if we're running secret code and clear the registers938	// so that the C code we're about to call doesn't spill confidential939	// information into memory940#ifdef GOEXPERIMENT_runtimesecret941	CMPL	g_secret(DI), $0942	JEQ	nosecret943	CALL	·secretEraseRegisters(SB)944945	get_tls(CX)946	MOVQ    g(CX), DI947	MOVQ    g_m(DI), R8948	MOVQ    m_g0(R8), SI949950nosecret:951#endif952	MOVQ	fn+0(FP), AX953	MOVQ	arg+8(FP), BX954	MOVQ	SP, DX955956	// Switch to system stack.957	// The original frame pointer is stored in BP,958	// which is useful for stack unwinding.959	CALL	gosave_systemstack_switch<>(SB)960	MOVQ	SI, g(CX)961	MOVQ	(g_sched+gobuf_sp)(SI), SP962963	// Now on a scheduling stack (a pthread-created stack).964	SUBQ	$16, SP965	ANDQ	$~15, SP	// alignment for gcc ABI966	MOVQ	DI, 8(SP)	// save g967	MOVQ	(g_stack+stack_hi)(DI), DI968	SUBQ	DX, DI969	MOVQ	DI, 0(SP)	// save depth in stack (can't just save SP, as stack might be copied during a callback)970	CALL	runtime·asmcgocall_landingpad(SB)971972	// Restore registers, g, stack pointer.973	get_tls(CX)974	MOVQ	8(SP), DI975	MOVQ	(g_stack+stack_hi)(DI), SI976	SUBQ	0(SP), SI977	MOVQ	DI, g(CX)978	MOVQ	SI, SP979980	MOVL	AX, ret+16(FP)981	RET982983nosave:984	// Running on a system stack, perhaps even without a g.985	// Having no g can happen during thread creation or thread teardown986	// (see needm/dropm on Solaris, for example).987	// This code is like the above sequence but without saving/restoring g988	// and without worrying about the stack moving out from under us989	// (because we're on a system stack, not a goroutine stack).990	MOVQ	fn+0(FP), AX991	MOVQ	arg+8(FP), BX992	MOVQ	SP, DX993994	SUBQ	$16, SP995	ANDQ	$~15, SP996	MOVQ	$0, 8(SP)		// where above code stores g, in case someone looks during debugging997	MOVQ	DX, 0(SP)	// save original stack pointer998	CALL	runtime·asmcgocall_landingpad(SB)999	MOVQ	0(SP), SI	// restore original stack pointer1000	MOVQ	SI, SP1001	MOVL	AX, ret+16(FP)1002	RET10031004#ifdef GOOS_windows1005// Dummy TLS that's used on Windows so that we don't crash trying1006// to restore the G register in needm. needm and its callees are1007// very careful never to actually use the G, the TLS just can't be1008// unset since we're in Go code.1009GLOBL zeroTLS<>(SB),RODATA,$const_tlsSize1010#endif10111012// func cgocallback(fn, frame unsafe.Pointer, ctxt uintptr)1013// See cgocall.go for more details.1014TEXT ·cgocallback(SB),NOSPLIT,$24-241015	NO_LOCAL_POINTERS10161017	// Skip cgocallbackg, just dropm when fn is nil, and frame is the saved g.1018	// It is used to dropm while thread is exiting.1019	MOVQ	fn+0(FP), AX1020	CMPQ	AX, $01021	JNE	loadg1022	// Restore the g from frame.1023	get_tls(CX)1024	MOVQ	frame+8(FP), BX1025	MOVQ	BX, g(CX)1026	JMP	dropm10271028loadg:1029	// If g is nil, Go did not create the current thread,1030	// or if this thread never called into Go on pthread platforms.1031	// Call needm to obtain one m for temporary use.1032	// In this case, we're running on the thread stack, so there's1033	// lots of space, but the linker doesn't know. Hide the call from1034	// the linker analysis by using an indirect call through AX.1035	get_tls(CX)1036#ifdef GOOS_windows1037	MOVL	$0, BX1038	CMPQ	CX, $01039	JEQ	2(PC)1040#endif1041	MOVQ	g(CX), BX1042	CMPQ	BX, $01043	JEQ	needm1044	MOVQ	g_m(BX), BX1045	MOVQ	BX, savedm-8(SP)	// saved copy of oldm1046	JMP	havem1047needm:1048#ifdef GOOS_windows1049	// Set up a dummy TLS value. needm is careful not to use it,1050	// but it needs to be there to prevent autogenerated code from1051	// crashing when it loads from it.1052	// We don't need to clear it or anything later because needm1053	// will set up TLS properly.1054	MOVQ	$zeroTLS<>(SB), DI1055	CALL	runtime·settls(SB)1056#endif1057	// On some platforms (Windows) we cannot call needm through1058	// an ABI wrapper because there's no TLS set up, and the ABI1059	// wrapper will try to restore the G register (R14) from TLS.1060	// Clear X15 because Go expects it and we're not calling1061	// through a wrapper, but otherwise avoid setting the G1062	// register in the wrapper and call needm directly. It1063	// takes no arguments and doesn't return any values so1064	// there's no need to handle that. Clear R14 so that there's1065	// a bad value in there, in case needm tries to use it.1066	XORPS	X15, X151067#ifndef GOAMD64_v31068#ifndef GOAMD64_v41069	CMPB	internal∕cpu·X86+const_offsetX86HasAVX(SB), $11070	JNE	2(PC)1071#endif1072#endif1073	VXORPS	X15, X15, X151074	XORQ    R14, R141075	MOVQ	$runtime·needAndBindM<ABIInternal>(SB), AX1076	CALL	AX1077	MOVQ	$0, savedm-8(SP)1078	get_tls(CX)1079	MOVQ	g(CX), BX1080	MOVQ	g_m(BX), BX10811082	// Set m->sched.sp = SP, so that if a panic happens1083	// during the function we are about to execute, it will1084	// have a valid SP to run on the g0 stack.1085	// The next few lines (after the havem label)1086	// will save this SP onto the stack and then write1087	// the same SP back to m->sched.sp. That seems redundant,1088	// but if an unrecovered panic happens, unwindm will1089	// restore the g->sched.sp from the stack location1090	// and then systemstack will try to use it. If we don't set it here,1091	// that restored SP will be uninitialized (typically 0) and1092	// will not be usable.1093	MOVQ	m_g0(BX), SI1094	MOVQ	SP, (g_sched+gobuf_sp)(SI)10951096havem:1097	// Now there's a valid m, and we're running on its m->g0.1098	// Save current m->g0->sched.sp on stack and then set it to SP.1099	// Save current sp in m->g0->sched.sp in preparation for1100	// switch back to m->curg stack.1101	// NOTE: unwindm knows that the saved g->sched.sp is at 0(SP).1102	MOVQ	m_g0(BX), SI1103	MOVQ	(g_sched+gobuf_sp)(SI), AX1104	MOVQ	AX, 0(SP)1105	MOVQ	SP, (g_sched+gobuf_sp)(SI)11061107	// Switch to m->curg stack and call runtime.cgocallbackg.1108	// Because we are taking over the execution of m->curg1109	// but *not* resuming what had been running, we need to1110	// save that information (m->curg->sched) so we can restore it.1111	// We can restore m->curg->sched.sp easily, because calling1112	// runtime.cgocallbackg leaves SP unchanged upon return.1113	// To save m->curg->sched.pc, we push it onto the curg stack and1114	// open a frame the same size as cgocallback's g0 frame.1115	// Once we switch to the curg stack, the pushed PC will appear1116	// to be the return PC of cgocallback, so that the traceback1117	// will seamlessly trace back into the earlier calls.1118	MOVQ	m_curg(BX), SI1119	MOVQ	SI, g(CX)1120	MOVQ	SI, R14 // set the g register, as required by ABIInternal.1121	XORPS	X15, X15 // clear X15, as required by ABIInternal.1122	MOVQ	(g_sched+gobuf_sp)(SI), DI  // prepare stack as DI1123	MOVQ	(g_sched+gobuf_pc)(SI), BX1124	MOVQ	BX, -8(DI)  // "push" return PC on the g stack1125	// Gather our arguments into registers.1126	MOVQ	fn+0(FP), AX1127	MOVQ	frame+8(FP), BX1128	MOVQ	ctxt+16(FP), CX1129	// Compute the size of the frame, including the return PC and1130	// saved frame pointer1131	LEAQ	fn+0(FP), R81132	SUBQ	SP, R8   // R8 is our actual frame size1133	SUBQ	R8, DI   // Allocate the same frame size on the g stack1134	MOVQ	DI, SP11351136	MOVQ	$runtime·cgocallbackg<ABIInternal>(SB), DX1137	CALL	DX	// indirect call to bypass nosplit check. We're on a different stack now.11381139	// Compute the size of the frame again. FP and SP have1140	// completely different values here than they did above,1141	// but only their difference matters.1142	LEAQ	fn+0(FP), AX1143	SUBQ	SP, AX11441145	// Restore g->sched (== m->curg->sched) from saved values.1146	get_tls(CX)1147	MOVQ	g(CX), SI1148	MOVQ	SP, DI1149	ADDQ	AX, DI1150	MOVQ	-8(DI), BX1151	MOVQ	BX, (g_sched+gobuf_pc)(SI)1152	MOVQ	DI, (g_sched+gobuf_sp)(SI)11531154	// Switch back to m->g0's stack and restore m->g0->sched.sp.1155	// (Unlike m->curg, the g0 goroutine never uses sched.pc,1156	// so we do not have to restore it.)1157	MOVQ	g(CX), BX1158	MOVQ	g_m(BX), BX1159	MOVQ	m_g0(BX), SI1160	MOVQ	SI, g(CX)1161	MOVQ	(g_sched+gobuf_sp)(SI), SP1162	MOVQ	0(SP), AX1163	MOVQ	AX, (g_sched+gobuf_sp)(SI)11641165	// If the m on entry was nil, we called needm above to borrow an m,1166	// 1. for the duration of the call on non-pthread platforms,1167	// 2. or the duration of the C thread alive on pthread platforms.1168	// If the m on entry wasn't nil,1169	// 1. the thread might be a Go thread,1170	// 2. or it wasn't the first call from a C thread on pthread platforms,1171	//    since then we skip dropm to reuse the m in the first call.1172	MOVQ	savedm-8(SP), BX1173	CMPQ	BX, $01174	JNE	done11751176	// Skip dropm to reuse it in the next call, when a pthread key has been created.1177	MOVQ	_cgo_pthread_key_created(SB), AX1178	// It means cgo is disabled when _cgo_pthread_key_created is a nil pointer, need dropm.1179	CMPQ	AX, $01180	JEQ	dropm1181	CMPQ	(AX), $01182	JNE	done11831184dropm:1185	MOVQ	$runtime·dropm(SB), AX1186	CALL	AX1187#ifdef GOOS_windows1188	// We need to clear the TLS pointer in case the next1189	// thread that comes into Go tries to reuse that space1190	// but uses the same M.1191	XORQ	DI, DI1192	CALL	runtime·settls(SB)1193#endif1194done:11951196	// Done!1197	RET11981199// func setg(gg *g)1200// set g. for use by needm.1201TEXT runtime·setg(SB), NOSPLIT, $0-81202	MOVQ	gg+0(FP), BX1203	get_tls(CX)1204	MOVQ	BX, g(CX)1205	RET12061207// void setg_gcc(G*); set g called from gcc.1208TEXT setg_gcc<>(SB),NOSPLIT,$01209	get_tls(AX)1210	MOVQ	DI, g(AX)1211	MOVQ	DI, R14 // set the g register1212	RET12131214TEXT runtime·abort(SB),NOSPLIT,$0-01215	INT	$31216loop:1217	JMP	loop12181219// check that SP is in range [g->stack.lo, g->stack.hi)1220TEXT runtime·stackcheck(SB), NOSPLIT|NOFRAME, $0-01221	get_tls(CX)1222	MOVQ	g(CX), AX1223	CMPQ	(g_stack+stack_hi)(AX), SP1224	JHI	2(PC)1225	CALL	runtime·abort(SB)1226	CMPQ	SP, (g_stack+stack_lo)(AX)1227	JHI	2(PC)1228	CALL	runtime·abort(SB)1229	RET12301231// func cputicks() int641232TEXT runtime·cputicks(SB),NOSPLIT,$0-01233	CMPB	internal∕cpu·X86+const_offsetX86HasRDTSCP(SB), $11234	JNE	fences1235	// Instruction stream serializing RDTSCP is supported.1236	// RDTSCP is supported by Intel Nehalem (2008) and1237	// AMD K8 Rev. F (2006) and newer.1238	RDTSCP1239done:1240	SHLQ	$32, DX1241	ADDQ	DX, AX1242	MOVQ	AX, ret+0(FP)1243	RET1244fences:1245	// MFENCE is instruction stream serializing and flushes the1246	// store buffers on AMD. The serialization semantics of LFENCE on AMD1247	// are dependent on MSR C001_1029 and CPU generation.1248	// LFENCE on Intel does wait for all previous instructions to have executed.1249	// Intel recommends MFENCE;LFENCE in its manuals before RDTSC to have all1250	// previous instructions executed and all previous loads and stores to globally visible.1251	// Using MFENCE;LFENCE here aligns the serializing properties without1252	// runtime detection of CPU manufacturer.1253	MFENCE1254	LFENCE1255	RDTSC1256	JMP done12571258// Called from cgo wrappers, this function returns g->m->curg.stack.hi.1259// Must obey the gcc calling convention.1260TEXT _cgo_topofstack(SB),NOSPLIT,$01261	get_tls(CX)1262	MOVQ	g(CX), AX1263	MOVQ	g_m(AX), AX1264	MOVQ	m_curg(AX), AX1265	MOVQ	(g_stack+stack_hi)(AX), AX1266	RET12671268// The top-most function running on a goroutine1269// returns to goexit+PCQuantum.1270TEXT runtime·goexit(SB),NOSPLIT|TOPFRAME|NOFRAME,$0-01271	BYTE	$0x90	// NOP1272	CALL	runtime·goexit1(SB)	// does not return1273	// traceback from goexit1 must hit code range of goexit1274	BYTE	$0x90	// NOP12751276// This is called from .init_array and follows the platform, not Go, ABI.1277TEXT runtime·addmoduledata(SB),NOSPLIT,$0-01278	PUSHQ	R15 // The access to global variables below implicitly uses R15, which is callee-save1279	MOVQ	runtime·lastmoduledatap(SB), AX1280	MOVQ	DI, moduledata_next(AX)1281	MOVQ	DI, runtime·lastmoduledatap(SB)1282	POPQ	R151283	RET12841285// Initialize special registers then jump to sigpanic.1286// This function is injected from the signal handler for panicking1287// signals. It is quite painful to set X15 in the signal context,1288// so we do it here.1289TEXT ·sigpanic0(SB),NOSPLIT,$0-01290	get_tls(R14)1291	MOVQ	g(R14), R141292	XORPS	X15, X151293#ifndef GOAMD64_v31294#ifndef GOAMD64_v41295	CMPB	internal∕cpu·X86+const_offsetX86HasAVX(SB), $11296	JNE	2(PC)1297#endif1298#endif1299	VXORPS	X15, X15, X151300	JMP	·sigpanic<ABIInternal>(SB)13011302// gcWriteBarrier informs the GC about heap pointer writes.1303//1304// gcWriteBarrier returns space in a write barrier buffer which1305// should be filled in by the caller.1306// gcWriteBarrier does NOT follow the Go ABI. It accepts the1307// number of bytes of buffer needed in R11, and returns a pointer1308// to the buffer space in R11.1309// It clobbers FLAGS. It does not clobber any general-purpose registers,1310// but may clobber others (e.g., SSE registers).1311// Typical use would be, when doing *(CX+88) = AX1312//     CMPL    $0, runtime.writeBarrier(SB)1313//     JEQ     dowrite1314//     CALL    runtime.gcBatchBarrier2(SB)1315//     MOVQ    AX, (R11)1316//     MOVQ    88(CX), DX1317//     MOVQ    DX, 8(R11)1318// dowrite:1319//     MOVQ    AX, 88(CX)1320TEXT gcWriteBarrier<>(SB),NOSPLIT,$1121321	// Save the registers clobbered by the fast path. This is slightly1322	// faster than having the caller spill these.1323	MOVQ	R12, 96(SP)1324	MOVQ	R13, 104(SP)1325retry:1326	// TODO: Consider passing g.m.p in as an argument so they can be shared1327	// across a sequence of write barriers.1328	MOVQ	g_m(R14), R131329	MOVQ	m_p(R13), R131330	// Get current buffer write position.1331	MOVQ	(p_wbBuf+wbBuf_next)(R13), R12	// original next position1332	ADDQ	R11, R12			// new next position1333	// Is the buffer full?1334	CMPQ	R12, (p_wbBuf+wbBuf_end)(R13)1335	JA	flush1336	// Commit to the larger buffer.1337	MOVQ	R12, (p_wbBuf+wbBuf_next)(R13)1338	// Make return value (the original next position)1339	SUBQ	R11, R121340	MOVQ	R12, R111341	// Restore registers.1342	MOVQ	96(SP), R121343	MOVQ	104(SP), R131344	RET13451346flush:1347	// Save all general purpose registers since these could be1348	// clobbered by wbBufFlush and were not saved by the caller.1349	// It is possible for wbBufFlush to clobber other registers1350	// (e.g., SSE registers), but the compiler takes care of saving1351	// those in the caller if necessary. This strikes a balance1352	// with registers that are likely to be used.1353	//1354	// We don't have type information for these, but all code under1355	// here is NOSPLIT, so nothing will observe these.1356	//1357	// TODO: We could strike a different balance; e.g., saving X01358	// and not saving GP registers that are less likely to be used.1359	MOVQ	DI, 0(SP)1360	MOVQ	AX, 8(SP)1361	MOVQ	BX, 16(SP)1362	MOVQ	CX, 24(SP)1363	MOVQ	DX, 32(SP)1364	// DI already saved1365	MOVQ	SI, 40(SP)1366	MOVQ	BP, 48(SP)1367	MOVQ	R8, 56(SP)1368	MOVQ	R9, 64(SP)1369	MOVQ	R10, 72(SP)1370	MOVQ	R11, 80(SP)1371	// R12 already saved1372	// R13 already saved1373	// R14 is g1374	MOVQ	R15, 88(SP)13751376	CALL	runtime·wbBufFlush(SB)13771378	MOVQ	0(SP), DI1379	MOVQ	8(SP), AX1380	MOVQ	16(SP), BX1381	MOVQ	24(SP), CX1382	MOVQ	32(SP), DX1383	MOVQ	40(SP), SI1384	MOVQ	48(SP), BP1385	MOVQ	56(SP), R81386	MOVQ	64(SP), R91387	MOVQ	72(SP), R101388	MOVQ	80(SP), R111389	MOVQ	88(SP), R151390	JMP	retry13911392TEXT runtime·gcWriteBarrier1<ABIInternal>(SB),NOSPLIT|NOFRAME,$01393	MOVL   $8, R111394	JMP     gcWriteBarrier<>(SB)1395TEXT runtime·gcWriteBarrier2<ABIInternal>(SB),NOSPLIT|NOFRAME,$01396	MOVL   $16, R111397	JMP     gcWriteBarrier<>(SB)1398TEXT runtime·gcWriteBarrier3<ABIInternal>(SB),NOSPLIT|NOFRAME,$01399	MOVL   $24, R111400	JMP     gcWriteBarrier<>(SB)1401TEXT runtime·gcWriteBarrier4<ABIInternal>(SB),NOSPLIT|NOFRAME,$01402	MOVL   $32, R111403	JMP     gcWriteBarrier<>(SB)1404TEXT runtime·gcWriteBarrier5<ABIInternal>(SB),NOSPLIT|NOFRAME,$01405	MOVL   $40, R111406	JMP     gcWriteBarrier<>(SB)1407TEXT runtime·gcWriteBarrier6<ABIInternal>(SB),NOSPLIT|NOFRAME,$01408	MOVL   $48, R111409	JMP     gcWriteBarrier<>(SB)1410TEXT runtime·gcWriteBarrier7<ABIInternal>(SB),NOSPLIT|NOFRAME,$01411	MOVL   $56, R111412	JMP     gcWriteBarrier<>(SB)1413TEXT runtime·gcWriteBarrier8<ABIInternal>(SB),NOSPLIT|NOFRAME,$01414	MOVL   $64, R111415	JMP     gcWriteBarrier<>(SB)14161417DATA	debugCallFrameTooLarge<>+0x00(SB)/20, $"call frame too large"1418GLOBL	debugCallFrameTooLarge<>(SB), RODATA, $20	// Size duplicated below14191420// debugCallV2 is the entry point for debugger-injected function1421// calls on running goroutines. It informs the runtime that a1422// debug call has been injected and creates a call frame for the1423// debugger to fill in.1424//1425// To inject a function call, a debugger should:1426// 1. Check that the goroutine is in state _Grunning and that1427//    there are at least 256 bytes free on the stack.1428// 2. Push the current PC on the stack (updating SP).1429// 3. Write the desired argument frame size at SP-16 (using the SP1430//    after step 2).1431// 4. Save all machine registers (including flags and XMM registers)1432//    so they can be restored later by the debugger.1433// 5. Set the PC to debugCallV2 and resume execution.1434//1435// If the goroutine is in state _Grunnable, then it's not generally1436// safe to inject a call because it may return out via other runtime1437// operations. Instead, the debugger should unwind the stack to find1438// the return to non-runtime code, add a temporary breakpoint there,1439// and inject the call once that breakpoint is hit.1440//1441// If the goroutine is in any other state, it's not safe to inject a call.1442//1443// This function communicates back to the debugger by setting R12 and1444// invoking INT3 to raise a breakpoint signal. See the comments in the1445// implementation for the protocol the debugger is expected to1446// follow. InjectDebugCall in the runtime tests demonstrates this protocol.1447//1448// The debugger must ensure that any pointers passed to the function1449// obey escape analysis requirements. Specifically, it must not pass1450// a stack pointer to an escaping argument. debugCallV2 cannot check1451// this invariant.1452//1453// This is ABIInternal because Go code injects its PC directly into new1454// goroutine stacks.1455TEXT runtime·debugCallV2<ABIInternal>(SB),NOSPLIT,$152-01456	// Save all registers that may contain pointers so they can be1457	// conservatively scanned.1458	//1459	// We can't do anything that might clobber any of these1460	// registers before this.1461	MOVQ	R15, r15-(14*8+8)(SP)1462	MOVQ	R14, r14-(13*8+8)(SP)1463	MOVQ	R13, r13-(12*8+8)(SP)1464	MOVQ	R12, r12-(11*8+8)(SP)1465	MOVQ	R11, r11-(10*8+8)(SP)1466	MOVQ	R10, r10-(9*8+8)(SP)1467	MOVQ	R9, r9-(8*8+8)(SP)1468	MOVQ	R8, r8-(7*8+8)(SP)1469	MOVQ	DI, di-(6*8+8)(SP)1470	MOVQ	SI, si-(5*8+8)(SP)1471	MOVQ	BP, bp-(4*8+8)(SP)1472	MOVQ	BX, bx-(3*8+8)(SP)1473	MOVQ	DX, dx-(2*8+8)(SP)1474	// Save the frame size before we clobber it. Either of the last1475	// saves could clobber this depending on whether there's a saved BP.1476	MOVQ	frameSize-24(FP), DX	// aka -16(RSP) before prologue1477	MOVQ	CX, cx-(1*8+8)(SP)1478	MOVQ	AX, ax-(0*8+8)(SP)14791480	// Save the argument frame size.1481	MOVQ	DX, frameSize-128(SP)14821483	// Perform a safe-point check.1484	MOVQ	retpc-8(FP), AX	// Caller's PC1485	MOVQ	AX, 0(SP)1486	CALL	runtime·debugCallCheck(SB)1487	MOVQ	8(SP), AX1488	TESTQ	AX, AX1489	JZ	good1490	// The safety check failed. Put the reason string at the top1491	// of the stack.1492	MOVQ	AX, 0(SP)1493	MOVQ	16(SP), AX1494	MOVQ	AX, 8(SP)1495	// Set R12 to 8 and invoke INT3. The debugger should get the1496	// reason a call can't be injected from the top of the stack1497	// and resume execution.1498	MOVQ	$8, R121499	BYTE	$0xcc1500	JMP	restore15011502good:1503	// Registers are saved and it's safe to make a call.1504	// Open up a call frame, moving the stack if necessary.1505	//1506	// Once the frame is allocated, this will set R12 to 0 and1507	// invoke INT3. The debugger should write the argument1508	// frame for the call at SP, set up argument registers, push1509	// the trapping PC on the stack, set the PC to the function to1510	// call, set RDX to point to the closure (if a closure call),1511	// and resume execution.1512	//1513	// If the function returns, this will set R12 to 1 and invoke1514	// INT3. The debugger can then inspect any return value saved1515	// on the stack at SP and in registers and resume execution again.1516	//1517	// If the function panics, this will set R12 to 2 and invoke INT3.1518	// The interface{} value of the panic will be at SP. The debugger1519	// can inspect the panic value and resume execution again.1520#define DEBUG_CALL_DISPATCH(NAME,MAXSIZE)	\1521	CMPQ	AX, $MAXSIZE;			\1522	JA	5(PC);				\1523	MOVQ	$NAME(SB), AX;			\1524	MOVQ	AX, 0(SP);			\1525	CALL	runtime·debugCallWrap(SB);	\1526	JMP	restore15271528	MOVQ	frameSize-128(SP), AX1529	DEBUG_CALL_DISPATCH(debugCall32<>, 32)1530	DEBUG_CALL_DISPATCH(debugCall64<>, 64)1531	DEBUG_CALL_DISPATCH(debugCall128<>, 128)1532	DEBUG_CALL_DISPATCH(debugCall256<>, 256)1533	DEBUG_CALL_DISPATCH(debugCall512<>, 512)1534	DEBUG_CALL_DISPATCH(debugCall1024<>, 1024)1535	DEBUG_CALL_DISPATCH(debugCall2048<>, 2048)1536	DEBUG_CALL_DISPATCH(debugCall4096<>, 4096)1537	DEBUG_CALL_DISPATCH(debugCall8192<>, 8192)1538	DEBUG_CALL_DISPATCH(debugCall16384<>, 16384)1539	DEBUG_CALL_DISPATCH(debugCall32768<>, 32768)1540	DEBUG_CALL_DISPATCH(debugCall65536<>, 65536)1541	// The frame size is too large. Report the error.1542	MOVQ	$debugCallFrameTooLarge<>(SB), AX1543	MOVQ	AX, 0(SP)1544	MOVQ	$20, 8(SP) // length of debugCallFrameTooLarge string1545	MOVQ	$8, R121546	BYTE	$0xcc1547	JMP	restore15481549restore:1550	// Calls and failures resume here.1551	//1552	// Set R12 to 16 and invoke INT3. The debugger should restore1553	// all registers except RIP and RSP and resume execution.1554	MOVQ	$16, R121555	BYTE	$0xcc1556	// We must not modify flags after this point.15571558	// Restore pointer-containing registers, which may have been1559	// modified from the debugger's copy by stack copying.1560	MOVQ	ax-(0*8+8)(SP), AX1561	MOVQ	cx-(1*8+8)(SP), CX1562	MOVQ	dx-(2*8+8)(SP), DX1563	MOVQ	bx-(3*8+8)(SP), BX1564	MOVQ	bp-(4*8+8)(SP), BP1565	MOVQ	si-(5*8+8)(SP), SI1566	MOVQ	di-(6*8+8)(SP), DI1567	MOVQ	r8-(7*8+8)(SP), R81568	MOVQ	r9-(8*8+8)(SP), R91569	MOVQ	r10-(9*8+8)(SP), R101570	MOVQ	r11-(10*8+8)(SP), R111571	MOVQ	r12-(11*8+8)(SP), R121572	MOVQ	r13-(12*8+8)(SP), R131573	MOVQ	r14-(13*8+8)(SP), R141574	MOVQ	r15-(14*8+8)(SP), R1515751576	RET15771578// runtime.debugCallCheck assumes that functions defined with the1579// DEBUG_CALL_FN macro are safe points to inject calls.1580#define DEBUG_CALL_FN(NAME,MAXSIZE)		\1581TEXT NAME(SB),WRAPPER,$MAXSIZE-0;		\1582	NO_LOCAL_POINTERS;			\1583	MOVQ	$0, R12;				\1584	BYTE	$0xcc;				\1585	MOVQ	$1, R12;				\1586	BYTE	$0xcc;				\1587	RET1588DEBUG_CALL_FN(debugCall32<>, 32)1589DEBUG_CALL_FN(debugCall64<>, 64)1590DEBUG_CALL_FN(debugCall128<>, 128)1591DEBUG_CALL_FN(debugCall256<>, 256)1592DEBUG_CALL_FN(debugCall512<>, 512)1593DEBUG_CALL_FN(debugCall1024<>, 1024)1594DEBUG_CALL_FN(debugCall2048<>, 2048)1595DEBUG_CALL_FN(debugCall4096<>, 4096)1596DEBUG_CALL_FN(debugCall8192<>, 8192)1597DEBUG_CALL_FN(debugCall16384<>, 16384)1598DEBUG_CALL_FN(debugCall32768<>, 32768)1599DEBUG_CALL_FN(debugCall65536<>, 65536)16001601// func debugCallPanicked(val interface{})1602TEXT runtime·debugCallPanicked(SB),NOSPLIT,$16-161603	// Copy the panic value to the top of stack.1604	MOVQ	val_type+0(FP), AX1605	MOVQ	AX, 0(SP)1606	MOVQ	val_data+8(FP), AX1607	MOVQ	AX, 8(SP)1608	MOVQ	$2, R121609	BYTE	$0xcc1610	RET16111612TEXT runtime·panicBounds<ABIInternal>(SB),NOSPLIT,$144-01613	NO_LOCAL_POINTERS1614	// Save all 14 int registers that could have an index in them.1615	// They may be pointers, but if they are they are dead.1616	MOVQ	AX, 16(SP)1617	MOVQ	CX, 24(SP)1618	MOVQ	DX, 32(SP)1619	MOVQ	BX, 40(SP)1620	// skip SP @ 48(SP)1621	MOVQ	BP, 56(SP)1622	MOVQ	SI, 64(SP)1623	MOVQ	DI, 72(SP)1624	MOVQ	R8, 80(SP)1625	MOVQ	R9, 88(SP)1626	MOVQ	R10, 96(SP)1627	MOVQ	R11, 104(SP)1628	MOVQ	R12, 112(SP)1629	MOVQ	R13, 120(SP)1630	// skip R14 @ 128(SP) (aka G)1631	MOVQ	R15, 136(SP)16321633	MOVQ	SP, AX		// hide SP read from vet1634	MOVQ	152(AX), AX	// PC immediately after call to panicBounds1635	LEAQ	16(SP), BX1636	CALL	runtime·panicBounds64<ABIInternal>(SB)1637	RET16381639#ifdef GOOS_android1640// Use the free TLS_SLOT_APP slot #2 on Android Q.1641// Earlier androids are set up in gcc_android.c.1642DATA runtime·tls_g+0(SB)/8, $161643GLOBL runtime·tls_g+0(SB), NOPTR, $81644#endif1645#ifdef GOOS_windows1646GLOBL runtime·tls_g+0(SB), NOPTR, $81647#endif16481649// The compiler and assembler's -spectre=ret mode rewrites1650// all indirect CALL AX / JMP AX instructions to be1651// CALL retpolineAX / JMP retpolineAX.1652// See https://support.google.com/faqs/answer/7625886.1653#define RETPOLINE(reg) \1654	/*   CALL setup */     BYTE $0xE8; BYTE $(2+2); BYTE $0; BYTE $0; BYTE $0;	\1655	/* nospec: */									\1656	/*   PAUSE */           BYTE $0xF3; BYTE $0x90;					\1657	/*   JMP nospec */      BYTE $0xEB; BYTE $-(2+2);				\1658	/* setup: */									\1659	/*   MOVQ AX, 0(SP) */  BYTE $0x48|((reg&8)>>1); BYTE $0x89;			\1660	                        BYTE $0x04|((reg&7)<<3); BYTE $0x24;			\1661	/*   RET */             BYTE $0xC316621663TEXT runtime·retpolineAX(SB),NOSPLIT|NOFRAME,$0; RETPOLINE(0)1664TEXT runtime·retpolineCX(SB),NOSPLIT|NOFRAME,$0; RETPOLINE(1)1665TEXT runtime·retpolineDX(SB),NOSPLIT|NOFRAME,$0; RETPOLINE(2)1666TEXT runtime·retpolineBX(SB),NOSPLIT|NOFRAME,$0; RETPOLINE(3)1667/* SP is 4, can't happen / magic encodings */1668TEXT runtime·retpolineBP(SB),NOSPLIT|NOFRAME,$0; RETPOLINE(5)1669TEXT runtime·retpolineSI(SB),NOSPLIT|NOFRAME,$0; RETPOLINE(6)1670TEXT runtime·retpolineDI(SB),NOSPLIT|NOFRAME,$0; RETPOLINE(7)1671TEXT runtime·retpolineR8(SB),NOSPLIT|NOFRAME,$0; RETPOLINE(8)1672TEXT runtime·retpolineR9(SB),NOSPLIT|NOFRAME,$0; RETPOLINE(9)1673TEXT runtime·retpolineR10(SB),NOSPLIT|NOFRAME,$0; RETPOLINE(10)1674TEXT runtime·retpolineR11(SB),NOSPLIT|NOFRAME,$0; RETPOLINE(11)1675TEXT runtime·retpolineR12(SB),NOSPLIT|NOFRAME,$0; RETPOLINE(12)1676TEXT runtime·retpolineR13(SB),NOSPLIT|NOFRAME,$0; RETPOLINE(13)1677TEXT runtime·retpolineR14(SB),NOSPLIT|NOFRAME,$0; RETPOLINE(14)1678TEXT runtime·retpolineR15(SB),NOSPLIT|NOFRAME,$0; RETPOLINE(15)16791680TEXT ·getfp<ABIInternal>(SB),NOSPLIT|NOFRAME,$01681	MOVQ BP, AX1682	RET