1// Copyright 2018 The Go Authors. All rights reserved.2// Use of this source code is governed by a BSD-style3// license that can be found in the LICENSE file.45//go:build race67#include "go_asm.h"8#include "go_tls.h"9#include "funcdata.h"10#include "textflag.h"11#include "asm_ppc64x.h"12#include "cgo/abi_ppc64x.h"1314// The following functions allow calling the clang-compiled race runtime directly15// from Go code without going all the way through cgo.16// First, it's much faster (up to 50% speedup for real Go programs).17// Second, it eliminates race-related special cases from cgocall and scheduler.18// Third, in long-term it will allow to remove cyclic runtime/race dependency on cmd/go.1920// A brief recap of the ppc64le calling convention.21// Arguments are passed in R3, R4, R5 ...22// SP must be 16-byte aligned.2324// Note that for ppc64x, LLVM follows the standard ABI and25// expects arguments in registers, so these functions move26// the arguments from storage to the registers expected27// by the ABI.2829// When calling from Go to Clang tsan code:30// R3 is the 1st argument and is usually the ThreadState*31// R4-? are the 2nd, 3rd, 4th, etc. arguments3233// When calling racecalladdr:34// R8 is the call target address3536// The race ctx is passed in R3 and loaded in37// racecalladdr.38//39// The sequence used to get the race ctx:40//    MOVD    runtime·tls_g(SB), R10 // Address of TLS variable41//    MOVD    0(R10), g              // g = R3042//    MOVD    g_racectx(g), R3       // racectx == ThreadState4344// func runtime·RaceRead(addr uintptr)45// Called from instrumented Go code46TEXT	runtime·raceread<ABIInternal>(SB), NOSPLIT, $0-847	MOVD	R3, R4 // addr48	MOVD	LR, R5 // caller of this?49	// void __tsan_read(ThreadState *thr, void *addr, void *pc);50	MOVD	$__tsan_read(SB), R851	BR	racecalladdr<>(SB)5253TEXT    runtime·RaceRead(SB), NOSPLIT, $0-854	BR	runtime·raceread(SB)5556// void runtime·racereadpc(void *addr, void *callpc, void *pc)57TEXT	runtime·racereadpc(SB), NOSPLIT, $0-2458	MOVD	addr+0(FP), R459	MOVD	callpc+8(FP), R560	MOVD	pc+16(FP), R661	// void __tsan_read_pc(ThreadState *thr, void *addr, void *callpc, void *pc);62	MOVD	$__tsan_read_pc(SB), R863	BR	racecalladdr<>(SB)6465// func runtime·RaceWrite(addr uintptr)66// Called from instrumented Go code67TEXT	runtime·racewrite<ABIInternal>(SB), NOSPLIT, $0-868	MOVD	R3, R4 // addr69	MOVD	LR, R5 // caller has set LR via BL inst70	// void __tsan_write(ThreadState *thr, void *addr, void *pc);71	MOVD	$__tsan_write(SB), R872	BR	racecalladdr<>(SB)7374TEXT    runtime·RaceWrite(SB), NOSPLIT, $0-875	JMP	runtime·racewrite(SB)7677// void runtime·racewritepc(void *addr, void *callpc, void *pc)78TEXT	runtime·racewritepc(SB), NOSPLIT, $0-2479	MOVD	addr+0(FP), R480	MOVD	callpc+8(FP), R581	MOVD	pc+16(FP), R682	// void __tsan_write_pc(ThreadState *thr, void *addr, void *callpc, void *pc);83	MOVD	$__tsan_write_pc(SB), R884	BR	racecalladdr<>(SB)8586// func runtime·RaceReadRange(addr, size uintptr)87// Called from instrumented Go code.88TEXT	runtime·racereadrange<ABIInternal>(SB), NOSPLIT, $0-1689	MOVD	R4, R5 // size90	MOVD	R3, R4 // addr91	MOVD	LR, R692	// void __tsan_read_range(ThreadState *thr, void *addr, uintptr size, void *pc);93	MOVD	$__tsan_read_range(SB), R894	BR	racecalladdr<>(SB)9596// void runtime·racereadrangepc1(void *addr, uintptr sz, void *pc)97TEXT	runtime·racereadrangepc1(SB), NOSPLIT, $0-2498	MOVD    addr+0(FP), R499	MOVD    size+8(FP), R5100	MOVD    pc+16(FP), R6101	ADD	$4, R6		// tsan wants return addr102	// void __tsan_read_range(ThreadState *thr, void *addr, uintptr size, void *pc);103	MOVD    $__tsan_read_range(SB), R8104	BR	racecalladdr<>(SB)105106TEXT    runtime·RaceReadRange(SB), NOSPLIT, $0-16107	BR	runtime·racereadrange(SB)108109// func runtime·RaceWriteRange(addr, size uintptr)110// Called from instrumented Go code.111TEXT	runtime·racewriterange<ABIInternal>(SB), NOSPLIT, $0-16112	MOVD	R4, R5 // size113	MOVD	R3, R4 // addr114	MOVD	LR, R6115	// void __tsan_write_range(ThreadState *thr, void *addr, uintptr size, void *pc);116	MOVD	$__tsan_write_range(SB), R8117	BR	racecalladdr<>(SB)118119TEXT    runtime·RaceWriteRange(SB), NOSPLIT, $0-16120	BR	runtime·racewriterange(SB)121122// void runtime·racewriterangepc1(void *addr, uintptr sz, void *pc)123// Called from instrumented Go code124TEXT	runtime·racewriterangepc1(SB), NOSPLIT, $0-24125	MOVD	addr+0(FP), R4126	MOVD	size+8(FP), R5127	MOVD	pc+16(FP), R6128	ADD	$4, R6			// add 4 to inst offset?129	// void __tsan_write_range(ThreadState *thr, void *addr, uintptr size, void *pc);130	MOVD	$__tsan_write_range(SB), R8131	BR	racecalladdr<>(SB)132133// Call a __tsan function from Go code.134// R8 = tsan function address135// R3 = *ThreadState a.k.a. g_racectx from g136// R4 = addr passed to __tsan function137//138// Otherwise, setup goroutine context and invoke racecall. Other arguments already set.139TEXT	racecalladdr<>(SB), NOSPLIT, $0-0140	MOVD    runtime·tls_g(SB), R10141	MOVD	0(R10), g142	MOVD	g_racectx(g), R3	// goroutine context143	// Check that addr is within [arenastart, arenaend) or within [racedatastart, racedataend).144	MOVD	runtime·racearenastart(SB), R9145	CMP	R4, R9146	BLT	data147	MOVD	runtime·racearenaend(SB), R9148	CMP	R4, R9149	BLT	call150data:151	MOVD	runtime·racedatastart(SB), R9152	CMP	R4, R9153	BLT	ret154	MOVD	runtime·racedataend(SB), R9155	CMP	R4, R9156	BGE	ret157call:158	// Careful!! racecall will save LR on its159	// stack, which is OK as long as racecalladdr160	// doesn't change in a way that generates a stack.161	// racecall should return to the caller of162	// recalladdr.163	BR	racecall<>(SB)164ret:165	RET166167// func runtime·racefuncenter(pc uintptr)168// Called from instrumented Go code.169TEXT	runtime·racefuncenter(SB), NOSPLIT, $0-8170	MOVD	callpc+0(FP), R8171	BR	racefuncenter<>(SB)172173// Common code for racefuncenter174// R11 = caller's return address175TEXT	racefuncenter<>(SB), NOSPLIT, $0-0176	MOVD    runtime·tls_g(SB), R10177	MOVD    0(R10), g178	MOVD    g_racectx(g), R3        // goroutine racectx aka *ThreadState179	MOVD	R8, R4			// caller pc set by caller in R8180	// void __tsan_func_enter(ThreadState *thr, void *pc);181	MOVD	$__tsan_func_enter(SB), R8182	BR	racecall<>(SB)183	RET184185// func runtime·racefuncexit()186// Called from Go instrumented code.187TEXT	runtime·racefuncexit(SB), NOSPLIT, $0-0188	MOVD    runtime·tls_g(SB), R10189	MOVD    0(R10), g190	MOVD    g_racectx(g), R3        // goroutine racectx aka *ThreadState191	// void __tsan_func_exit(ThreadState *thr);192	MOVD	$__tsan_func_exit(SB), R8193	BR	racecall<>(SB)194195// Atomic operations for sync/atomic package.196// Some use the __tsan versions instead197// R6 = addr of arguments passed to this function198// R3, R4, R5 set in racecallatomic199200// Load atomic in tsan201TEXT	sync∕atomic·LoadInt32(SB), NOSPLIT, $0-12202	GO_ARGS203	// void __tsan_go_atomic32_load(ThreadState *thr, uptr cpc, uptr pc, u8 *a);204	MOVD	$__tsan_go_atomic32_load(SB), R8205	ADD	$32, R1, R6	// addr of caller's 1st arg206	BR	racecallatomic<>(SB)207	RET208209TEXT	sync∕atomic·LoadInt64(SB), NOSPLIT, $0-16210	GO_ARGS211	// void __tsan_go_atomic64_load(ThreadState *thr, uptr cpc, uptr pc, u8 *a);212	MOVD	$__tsan_go_atomic64_load(SB), R8213	ADD	$32, R1, R6	// addr of caller's 1st arg214	BR	racecallatomic<>(SB)215	RET216217TEXT	sync∕atomic·LoadUint32(SB), NOSPLIT, $0-12218	GO_ARGS219	BR	sync∕atomic·LoadInt32(SB)220221TEXT	sync∕atomic·LoadUint64(SB), NOSPLIT, $0-16222	GO_ARGS223	BR	sync∕atomic·LoadInt64(SB)224225TEXT	sync∕atomic·LoadUintptr(SB), NOSPLIT, $0-16226	GO_ARGS227	BR	sync∕atomic·LoadInt64(SB)228229TEXT	sync∕atomic·LoadPointer(SB), NOSPLIT, $0-16230	GO_ARGS231	BR	sync∕atomic·LoadInt64(SB)232233// Store atomic in tsan234TEXT	sync∕atomic·StoreInt32(SB), NOSPLIT, $0-12235	GO_ARGS236	// void __tsan_go_atomic32_store(ThreadState *thr, uptr cpc, uptr pc, u8 *a);237	MOVD	$__tsan_go_atomic32_store(SB), R8238	ADD	$32, R1, R6	// addr of caller's 1st arg239	BR	racecallatomic<>(SB)240241TEXT	sync∕atomic·StoreInt64(SB), NOSPLIT, $0-16242	GO_ARGS243	// void __tsan_go_atomic64_store(ThreadState *thr, uptr cpc, uptr pc, u8 *a);244	MOVD	$__tsan_go_atomic64_store(SB), R8245	ADD	$32, R1, R6	// addr of caller's 1st arg246	BR	racecallatomic<>(SB)247248TEXT	sync∕atomic·StoreUint32(SB), NOSPLIT, $0-12249	GO_ARGS250	BR	sync∕atomic·StoreInt32(SB)251252TEXT	sync∕atomic·StoreUint64(SB), NOSPLIT, $0-16253	GO_ARGS254	BR	sync∕atomic·StoreInt64(SB)255256TEXT	sync∕atomic·StoreUintptr(SB), NOSPLIT, $0-16257	GO_ARGS258	BR	sync∕atomic·StoreInt64(SB)259260// Swap in tsan261TEXT	sync∕atomic·SwapInt32(SB), NOSPLIT, $0-20262	GO_ARGS263	// void __tsan_go_atomic32_exchange(ThreadState *thr, uptr cpc, uptr pc, u8 *a);264	MOVD	$__tsan_go_atomic32_exchange(SB), R8265	ADD	$32, R1, R6	// addr of caller's 1st arg266	BR	racecallatomic<>(SB)267268TEXT	sync∕atomic·SwapInt64(SB), NOSPLIT, $0-24269	GO_ARGS270	// void __tsan_go_atomic64_exchange(ThreadState *thr, uptr cpc, uptr pc, u8 *a)271	MOVD	$__tsan_go_atomic64_exchange(SB), R8272	ADD	$32, R1, R6	// addr of caller's 1st arg273	BR	racecallatomic<>(SB)274275TEXT	sync∕atomic·SwapUint32(SB), NOSPLIT, $0-20276	GO_ARGS277	BR	sync∕atomic·SwapInt32(SB)278279TEXT	sync∕atomic·SwapUint64(SB), NOSPLIT, $0-24280	GO_ARGS281	BR	sync∕atomic·SwapInt64(SB)282283TEXT	sync∕atomic·SwapUintptr(SB), NOSPLIT, $0-24284	GO_ARGS285	BR	sync∕atomic·SwapInt64(SB)286287// Add atomic in tsan288TEXT	sync∕atomic·AddInt32(SB), NOSPLIT, $0-20289	GO_ARGS290	// void __tsan_go_atomic32_fetch_add(ThreadState *thr, uptr cpc, uptr pc, u8 *a);291	MOVD	$__tsan_go_atomic32_fetch_add(SB), R8292	ADD	$64, R1, R6	// addr of caller's 1st arg293	BL	racecallatomic<>(SB)294	// The tsan fetch_add result is not as expected by Go,295	// so the 'add' must be added to the result.296	MOVW	add+8(FP), R3	// The tsa fetch_add does not return the297	MOVW	ret+16(FP), R4	// result as expected by go, so fix it.298	ADD	R3, R4, R3299	MOVW	R3, ret+16(FP)300	RET301302TEXT	sync∕atomic·AddInt64(SB), NOSPLIT, $0-24303	GO_ARGS304	// void __tsan_go_atomic64_fetch_add(ThreadState *thr, uptr cpc, uptr pc, u8 *a);305	MOVD	$__tsan_go_atomic64_fetch_add(SB), R8306	ADD	$64, R1, R6	// addr of caller's 1st arg307	BL	racecallatomic<>(SB)308	// The tsan fetch_add result is not as expected by Go,309	// so the 'add' must be added to the result.310	MOVD	add+8(FP), R3311	MOVD	ret+16(FP), R4312	ADD	R3, R4, R3313	MOVD	R3, ret+16(FP)314	RET315316TEXT	sync∕atomic·AddUint32(SB), NOSPLIT, $0-20317	GO_ARGS318	BR	sync∕atomic·AddInt32(SB)319320TEXT	sync∕atomic·AddUint64(SB), NOSPLIT, $0-24321	GO_ARGS322	BR	sync∕atomic·AddInt64(SB)323324TEXT	sync∕atomic·AddUintptr(SB), NOSPLIT, $0-24325	GO_ARGS326	BR	sync∕atomic·AddInt64(SB)327328// And329TEXT	sync∕atomic·AndInt32(SB), NOSPLIT, $0-20330	GO_ARGS331	MOVD	$__tsan_go_atomic32_fetch_and(SB), R8332	ADD     $32, R1, R6333	BR	racecallatomic<>(SB)334335TEXT	sync∕atomic·AndInt64(SB), NOSPLIT, $0-24336	GO_ARGS337	MOVD	$__tsan_go_atomic64_fetch_and(SB), R8338	ADD     $32, R1, R6339	BR	racecallatomic<>(SB)340341TEXT	sync∕atomic·AndUint32(SB), NOSPLIT, $0-20342	GO_ARGS343	BR	sync∕atomic·AndInt32(SB)344345TEXT	sync∕atomic·AndUint64(SB), NOSPLIT, $0-24346	GO_ARGS347	BR	sync∕atomic·AndInt64(SB)348349TEXT	sync∕atomic·AndUintptr(SB), NOSPLIT, $0-24350	GO_ARGS351	BR	sync∕atomic·AndInt64(SB)352353// Or354TEXT	sync∕atomic·OrInt32(SB), NOSPLIT, $0-20355	GO_ARGS356	MOVD	$__tsan_go_atomic32_fetch_or(SB), R8357	ADD     $32, R1, R6358	BR	racecallatomic<>(SB)359360TEXT	sync∕atomic·OrInt64(SB), NOSPLIT, $0-24361	GO_ARGS362	MOVD	$__tsan_go_atomic64_fetch_or(SB), R8363	ADD     $32, R1, R6364	BR	racecallatomic<>(SB)365366TEXT	sync∕atomic·OrUint32(SB), NOSPLIT, $0-20367	GO_ARGS368	BR	sync∕atomic·OrInt32(SB)369370TEXT	sync∕atomic·OrUint64(SB), NOSPLIT, $0-24371	GO_ARGS372	BR	sync∕atomic·OrInt64(SB)373374TEXT	sync∕atomic·OrUintptr(SB), NOSPLIT, $0-24375	GO_ARGS376	BR	sync∕atomic·OrInt64(SB)377378// CompareAndSwap in tsan379TEXT	sync∕atomic·CompareAndSwapInt32(SB), NOSPLIT, $0-17380	GO_ARGS381	// void __tsan_go_atomic32_compare_exchange(382	//   ThreadState *thr, uptr cpc, uptr pc, u8 *a)383	MOVD	$__tsan_go_atomic32_compare_exchange(SB), R8384	ADD	$32, R1, R6	// addr of caller's 1st arg385	BR	racecallatomic<>(SB)386387TEXT	sync∕atomic·CompareAndSwapInt64(SB), NOSPLIT, $0-25388	GO_ARGS389	// void __tsan_go_atomic32_compare_exchange(390	//   ThreadState *thr, uptr cpc, uptr pc, u8 *a)391	MOVD	$__tsan_go_atomic64_compare_exchange(SB), R8392	ADD	$32, R1, R6	// addr of caller's 1st arg393	BR	racecallatomic<>(SB)394395TEXT	sync∕atomic·CompareAndSwapUint32(SB), NOSPLIT, $0-17396	GO_ARGS397	BR	sync∕atomic·CompareAndSwapInt32(SB)398399TEXT	sync∕atomic·CompareAndSwapUint64(SB), NOSPLIT, $0-25400	GO_ARGS401	BR	sync∕atomic·CompareAndSwapInt64(SB)402403TEXT	sync∕atomic·CompareAndSwapUintptr(SB), NOSPLIT, $0-25404	GO_ARGS405	BR	sync∕atomic·CompareAndSwapInt64(SB)406407// Common function used to call tsan's atomic functions408// R3 = *ThreadState409// R4 = TODO: What's this supposed to be?410// R5 = caller pc411// R6 = addr of incoming arg list412// R8 contains addr of target function.413TEXT	racecallatomic<>(SB), NOSPLIT, $0-0414	// Trigger SIGSEGV early if address passed to atomic function is bad.415	MOVD	(R6), R7	// 1st arg is addr416	MOVB	(R7), R9	// segv here if addr is bad417	// Check that addr is within [arenastart, arenaend) or within [racedatastart, racedataend).418	MOVD	runtime·racearenastart(SB), R9419	CMP	R7, R9420	BLT	racecallatomic_data421	MOVD	runtime·racearenaend(SB), R9422	CMP	R7, R9423	BLT	racecallatomic_ok424racecallatomic_data:425	MOVD	runtime·racedatastart(SB), R9426	CMP	R7, R9427	BLT	racecallatomic_ignore428	MOVD	runtime·racedataend(SB), R9429	CMP	R7, R9430	BGE	racecallatomic_ignore431racecallatomic_ok:432	// Addr is within the good range, call the atomic function.433	MOVD    runtime·tls_g(SB), R10434	MOVD    0(R10), g435	MOVD    g_racectx(g), R3        // goroutine racectx aka *ThreadState436	MOVD	R8, R5			// pc is the function called437	MOVD	(R1), R4		// caller pc from stack438	BL	racecall<>(SB)		// BL needed to maintain stack consistency439	RET				//440racecallatomic_ignore:441	// Addr is outside the good range.442	// Call __tsan_go_ignore_sync_begin to ignore synchronization during the atomic op.443	// An attempt to synchronize on the address would cause crash.444	MOVD	R8, R15	// save the original function445	MOVD	R6, R17 // save the original arg list addr446	MOVD	$__tsan_go_ignore_sync_begin(SB), R8 // func addr to call447	MOVD    runtime·tls_g(SB), R10448	MOVD    0(R10), g449	MOVD    g_racectx(g), R3        // goroutine context450	BL	racecall<>(SB)451	MOVD	R15, R8	// restore the original function452	MOVD	R17, R6 // restore arg list addr453	// Call the atomic function.454	// racecall will call LLVM race code which might clobber r30 (g)455	MOVD	runtime·tls_g(SB), R10456	MOVD	0(R10), g457458	MOVD	g_racectx(g), R3459	MOVD	R8, R4		// pc being called same TODO as above460	MOVD	(R1), R5	// caller pc from latest LR461	BL	racecall<>(SB)462	// Call __tsan_go_ignore_sync_end.463	MOVD	$__tsan_go_ignore_sync_end(SB), R8464	MOVD	g_racectx(g), R3	// goroutine context g should still be good?465	BL	racecall<>(SB)466	RET467468// void runtime·racecall(void(*f)(...), ...)469// Calls C function f from race runtime and passes up to 4 arguments to it.470// The arguments are never heap-object-preserving pointers, so we pretend there are no arguments.471TEXT	runtime·racecall(SB), NOSPLIT, $0-0472	MOVD	fn+0(FP), R8473	MOVD	arg0+8(FP), R3474	MOVD	arg1+16(FP), R4475	MOVD	arg2+24(FP), R5476	MOVD	arg3+32(FP), R6477	JMP	racecall<>(SB)478479// Finds g0 and sets its stack480// Arguments were loaded for call from Go to C481TEXT	racecall<>(SB), NOSPLIT, $0-0482	// Set the LR slot for the ppc64 ABI483	MOVD	LR, R10484	MOVD	R10, 0(R1)	// Go expectation485	MOVD	R10, 16(R1)	// C ABI486	// Get info from the current goroutine487	MOVD    runtime·tls_g(SB), R10	// g offset in TLS488	MOVD    0(R10), g489	MOVD	g_m(g), R7		// m for g490	MOVD	R1, R16			// callee-saved, preserved across C call491492	// Switch to g0 stack if we aren't already on g0 or gsignal.493	MOVD	m_gsignal(R7), R10494	CMP	R10, g495	BEQ	call496497	MOVD	m_g0(R7), R10498	CMP	R10, g499	BEQ	call500501	MOVD	(g_sched+gobuf_sp)(R10), R1 // switch R1502call:503	// prepare frame for C ABI504	SUB	$32, R1			// create frame for callee saving LR, CR, R2 etc.505	RLDCR   $0, R1, $~15, R1	// align SP to 16 bytes506	MOVD	R8, CTR			// R8 = caller addr507	MOVD	R8, R12			// expected by PPC64 ABI508	BL	(CTR)509	XOR     R0, R0			// clear R0 on return from Clang510	MOVD	R16, R1			// restore R1; R16 nonvol in Clang511	MOVD    runtime·tls_g(SB), R10	// find correct g512	MOVD    0(R10), g513	MOVD	16(R1), R10		// LR was saved away, restore for return514	MOVD	R10, LR515	RET516517// C->Go callback thunk that allows to call runtime·racesymbolize from C code.518// Direct Go->C race call has only switched SP, finish g->g0 switch by setting correct g.519// The overall effect of Go->C->Go call chain is similar to that of mcall.520// RARG0 contains command code. RARG1 contains command-specific context.521// See racecallback for command codes.522TEXT	runtime·racecallbackthunk(SB), NOSPLIT|NOFRAME, $0523	// Handle command raceGetProcCmd (0) here.524	// First, code below assumes that we are on curg, while raceGetProcCmd525	// can be executed on g0. Second, it is called frequently, so will526	// benefit from this fast path.527	MOVD	$0, R0		// clear R0 since we came from C code528	CMP	R3, $0529	BNE	rest530	// Inline raceGetProdCmd without clobbering callee-save registers.531	MOVD	runtime·tls_g(SB), R10532	MOVD	0(R10), R11533	MOVD	g_m(R11), R3534	MOVD	m_p(R3), R3535	MOVD	p_raceprocctx(R3), R3536	MOVD	R3, (R4)537	RET538539rest:540	// Save registers according to the host PPC64 ABI541	// and reserve 16B for argument storage.542	STACK_AND_SAVE_HOST_TO_GO_ABI(16)543544	// Load g, and switch to g0 if not already on it.545	MOVD	runtime·tls_g(SB), R10546	MOVD	0(R10), g547548	MOVD	g_m(g), R7549	MOVD	m_g0(R7), R8550	CMP	g, R8551	BEQ	noswitch552553	MOVD	R8, g // set g = m->g0554555noswitch:556	BL	runtime·racecallback<ABIInternal>(SB)557558	UNSTACK_AND_RESTORE_GO_TO_HOST_ABI(16)559	RET560561// tls_g, g value for each thread in TLS562GLOBL runtime·tls_g+0(SB), TLSBSS+DUPOK, $8