1// Copyright 2016 The Go Authors. All rights reserved.2// Use of this source code is governed by a BSD-style3// license that can be found in the LICENSE file.45package ssacompile67import (8	"slices"910	"cmd/compile/internal/ir"11	"cmd/compile/internal/ssa"12	"cmd/compile/internal/ssa/block"13	"cmd/compile/internal/ssa/ssaop"14	"cmd/compile/internal/types"15	"cmd/internal/obj"16)1718// The pair pass finds memory operations that can be paired up19// into single 2-register memory instructions.20func pair(f *ssa.Func) {21	// Only arm64 for now. This pass is fairly arch-specific.22	switch f.Config.Arch {23	case "arm64":24	default:25		return26	}27	pairLoads(f)28	pairStores(f)29}3031type pairInfo struct {32	width int64 // width of one element in the pair, in bytes33	pair  ssaop.Op34}3536// All pairableLoad ops must take 2 arguments, a pointer and a memory.37// They must also take an offset in Aux/AuxInt.38var pairableLoads = map[ssaop.Op]pairInfo{39	ssaop.OpARM64MOVDload:  {8, ssaop.OpARM64LDP},40	ssaop.OpARM64MOVWUload: {4, ssaop.OpARM64LDPW},41	ssaop.OpARM64MOVWload:  {4, ssaop.OpARM64LDPSW},42	// TODO: conceivably we could pair a signed and unsigned load43	// if we knew the upper bits of one of them weren't being used.44	ssaop.OpARM64FMOVDload: {8, ssaop.OpARM64FLDPD},45	ssaop.OpARM64FMOVSload: {4, ssaop.OpARM64FLDPS},46	// NEON loads47	ssaop.OpARM64FMOVQload: {16, ssaop.OpARM64FLDPQ},48}4950// All pairableStore keys must take 3 arguments, a pointer, a value, and a memory.51// All pairableStore values must take 4 arguments, a pointer, 2 values, and a memory.52// They must also take an offset in Aux/AuxInt.53var pairableStores = map[ssaop.Op]pairInfo{54	ssaop.OpARM64MOVDstore:  {8, ssaop.OpARM64STP},55	ssaop.OpARM64MOVWstore:  {4, ssaop.OpARM64STPW},56	ssaop.OpARM64FMOVDstore: {8, ssaop.OpARM64FSTPD},57	ssaop.OpARM64FMOVSstore: {4, ssaop.OpARM64FSTPS},58	// NEON stores59	ssaop.OpARM64FMOVQstore: {16, ssaop.OpARM64FSTPQ},60}6162// offsetOk returns true if a pair instruction should be used63// for the offset Aux+off, when the data width (of the64// unpaired instructions) is width.65// This function is best-effort. The compiled function must66// still work if offsetOk always returns true.67// TODO: this is currently arm64-specific.68func offsetOk(aux ssa.Aux, off, width int64) bool {69	if true {70		// Seems to generate slightly smaller code if we just71		// always allow this rewrite.72		//73		// Without pairing, we have 2 load instructions, like:74		//   LDR 88(R0), R175		//   LDR 96(R0), R276		// with pairing we have, best case:77		//   LDP 88(R0), R1, R278		// but maybe we need an adjuster if out of range or unaligned:79		//   ADD R0, $88, R2780		//   LDP (R27), R1, R281		// Even with the adjuster, it is at least no worse.82		//83		// A similar situation occurs when accessing globals.84		// Two loads from globals requires 4 instructions,85		// two ADRP and two LDR. With pairing, we need86		// ADRP+ADD+LDP, three instructions.87		//88		// With pairing, it looks like the critical path might89		// be a little bit longer. But it should never be more90		// instructions.91		// TODO: see if that longer critical path causes any92		// regressions.93		return true94	}95	if aux != nil {96		if _, ok := aux.(*ir.Name); !ok {97			// Offset is probably too big (globals).98			return false99		}100		// We let *ir.Names pass here, as101		// they are probably small offsets from SP.102		// There's no guarantee that we're in range103		// in that case though (we don't know the104		// stack frame size yet), so the assembler105		// might need to issue fixup instructions.106		// Assume some small frame size.107		if off >= 0 {108			off += 128 // This should be multiple of 4, 8 and 16 for later off%width check109		}110		// TODO: figure out how often this helps vs. hurts.111	}112	switch width {113	case 4, 8, 16:114		// Offset is encoded as signed 7 bit imm * width115		const simm7Min = -64116		const simm7Max = 63117118		if off >= simm7Min*width && off <= simm7Max*width && off%width == 0 {119			return true120		}121	}122	return false123}124125func pairLoads(f *ssa.Func) {126	var loads []*ssa.Value127128	// Registry of aux values for sorting.129	auxIDs := map[ssa.Aux]int{}130	auxID := func(aux ssa.Aux) int {131		id, ok := auxIDs[aux]132		if !ok {133			id = len(auxIDs)134			auxIDs[aux] = id135		}136		return id137	}138139	for _, b := range f.Blocks {140		// Find loads.141		loads = loads[:0]142		clear(auxIDs)143		for _, v := range b.Values {144			info := pairableLoads[v.Op]145			if info.width == 0 {146				continue // not pairable147			}148			if !offsetOk(v.Aux, v.AuxInt, info.width) {149				continue // not advisable150			}151			loads = append(loads, v)152		}153		if len(loads) < 2 {154			continue155		}156157		// Sort to put pairable loads together.158		slices.SortFunc(loads, func(x, y *ssa.Value) int {159			// First sort by op, ptr, and memory arg.160			if x.Op != y.Op {161				return int(x.Op - y.Op)162			}163			if x.Args[0].ID != y.Args[0].ID {164				return int(x.Args[0].ID - y.Args[0].ID)165			}166			if x.Args[1].ID != y.Args[1].ID {167				return int(x.Args[1].ID - y.Args[1].ID)168			}169			// Then sort by aux. (nil first, then by aux ID)170			if x.Aux != nil {171				if y.Aux == nil {172					return 1173				}174				a, b := auxID(x.Aux), auxID(y.Aux)175				if a != b {176					return a - b177				}178			} else if y.Aux != nil {179				return -1180			}181			// Then sort by offset, low to high.182			return int(x.AuxInt - y.AuxInt)183		})184185		// Look for pairable loads.186		for i := 0; i < len(loads)-1; i++ {187			x := loads[i]188			y := loads[i+1]189			if x.Op != y.Op || x.Args[0] != y.Args[0] || x.Args[1] != y.Args[1] {190				continue191			}192			if x.Aux != y.Aux {193				continue194			}195			if x.AuxInt+pairableLoads[x.Op].width != y.AuxInt {196				continue197			}198199			// Commit point.200201			// Make the 2-register load.202			load := b.NewValue2IA(x.Pos, pairableLoads[x.Op].pair, types.NewTuple(x.Type, y.Type), x.AuxInt, x.Aux, x.Args[0], x.Args[1])203204			// Modify x to be (Select0 load). Similar for y.205			x.Reset(ssaop.OpSelect0)206			x.SetArgs1(load)207			y.Reset(ssaop.OpSelect1)208			y.SetArgs1(load)209210			i++ // Skip y next time around the loop.211		}212	}213214	// Try to pair a load with a load from a subsequent block.215	// Note that this is always safe to do if the memory arguments match.216	// (But see the memory barrier case below.)217	type nextBlockKey struct {218		op     ssaop.Op219		ptr    ssa.ID220		mem    ssa.ID221		auxInt int64222		aux    any223	}224	nextBlock := map[nextBlockKey]*ssa.Value{}225	for _, b := range f.Blocks {226		if memoryBarrierTest(b) {227			// TODO: Do we really need to skip write barrier test blocks?228			//     type T struct {229			//         a *byte230			//         b int231			//     }232			//     func f(t *T) int {233			//         r := t.b234			//         t.a = nil235			//         return r236			//     }237			// This would issue a single LDP for both the t.a and t.b fields,238			// *before* we check the write barrier flag. (We load the t.a field239			// to put it in the write barrier buffer.) Not sure if that is ok.240			continue241		}242		// Find loads in the next block(s) that we can move to this one.243		// TODO: could maybe look further than just one successor hop.244		clear(nextBlock)245		for _, e := range b.Succs {246			if len(e.B.Preds) > 1 {247				continue248			}249			for _, v := range e.B.Values {250				info := pairableLoads[v.Op]251				if info.width == 0 {252					continue253				}254				if !offsetOk(v.Aux, v.AuxInt, info.width) {255					continue // not advisable256				}257				nextBlock[nextBlockKey{op: v.Op, ptr: v.Args[0].ID, mem: v.Args[1].ID, auxInt: v.AuxInt, aux: v.Aux}] = v258			}259		}260		if len(nextBlock) == 0 {261			continue262		}263		// don't move too many loads. Each requires a register across a basic block boundary.264		const maxMoved = 4265		nMoved := 0266		for i := len(b.Values) - 1; i >= 0 && nMoved < maxMoved; i-- {267			x := b.Values[i]268			info := pairableLoads[x.Op]269			if info.width == 0 {270				continue271			}272			if !offsetOk(x.Aux, x.AuxInt, info.width) {273				continue // not advisable274			}275			key := nextBlockKey{op: x.Op, ptr: x.Args[0].ID, mem: x.Args[1].ID, auxInt: x.AuxInt + info.width, aux: x.Aux}276			if y := nextBlock[key]; y != nil {277				delete(nextBlock, key)278279				// Make the 2-register load.280				load := b.NewValue2IA(x.Pos, info.pair, types.NewTuple(x.Type, y.Type), x.AuxInt, x.Aux, x.Args[0], x.Args[1])281282				// Modify x to be (Select0 load).283				x.Reset(ssaop.OpSelect0)284				x.SetArgs1(load)285				// Modify y to be (Copy (Select1 load)).286				// Note: the Select* needs to live in the load's block, not y's block.287				y.Reset(ssaop.OpCopy)288				y.SetArgs1(b.NewValue1(y.Pos, ssaop.OpSelect1, y.Type, load))289				nMoved++290				continue291			}292			key.auxInt = x.AuxInt - info.width293			if y := nextBlock[key]; y != nil {294				delete(nextBlock, key)295296				// Make the 2-register load.297				load := b.NewValue2IA(x.Pos, info.pair, types.NewTuple(y.Type, x.Type), y.AuxInt, x.Aux, x.Args[0], x.Args[1])298299				// Modify x to be (Select1 load).300				x.Reset(ssaop.OpSelect1)301				x.SetArgs1(load)302				// Modify y to be (Copy (Select0 load)).303				y.Reset(ssaop.OpCopy)304				y.SetArgs1(b.NewValue1(y.Pos, ssaop.OpSelect0, y.Type, load))305				nMoved++306				continue307			}308		}309	}310}311312func memoryBarrierTest(b *ssa.Block) bool {313	if b.Kind != block.BlockARM64NZW {314		return false315	}316	c := b.Controls[0]317	if c.Op != ssaop.OpARM64MOVWUload {318		return false319	}320	if globl, ok := c.Aux.(*obj.LSym); ok {321		return globl.Name == "runtime.writeBarrier"322	}323	return false324}325326// pairStores merges store instructions.327// It collects stores into a buffer where they can be freely reordered.328// When encountering an instruction that cannot be added to the buffer,329// it pairs the accumulated stores, flushes the buffer, and continues processing.330func pairStores(f *ssa.Func) {331	last := f.Cache.AllocBoolSlice(f.NumValues())332	defer f.Cache.FreeBoolSlice(last)333334	// memChain contains a list of stores with the same ptr/aux pair and335	// nonoverlapping write ranges [AuxInt:AuxInt+writeSize]. All of the336	// elements of memChain can be reordered with each other.337	memChain := []*ssa.Value{}338339	// Limit of length of memChain array.340	// This keeps us in O(n) territory.341	limit := 100342343	// flushMemChain sorts the stores in memChain and merges them when possible.344	// Then it flushes memChain.345	flushMemChain := func() {346		if len(memChain) < 2 {347			memChain = memChain[:0]348			return349		}350351		// Sort in increasing AuxInt to put pairable stores together.352		slices.SortFunc(memChain, func(x, y *ssa.Value) int {353			return int(x.AuxInt - y.AuxInt)354		})355356		lastIdx := len(memChain) - 1357		for i := 0; i < lastIdx; i++ {358			v := memChain[i]359			w := memChain[i+1]360			info := pairableStores[v.Op]361362			off := v.AuxInt363			mem := v.MemoryArg()364			aux := v.Aux365			pos := v.Pos366			wmem := w.MemoryArg()367368			if w.Op == v.Op && w.AuxInt == off+info.width {369				// Arguments for the merged store: ptr, val1, val2, mem.370				args := []*ssa.Value{v.Args[0], v.Args[1], w.Args[1], mem}371372				v.Reset(info.pair)373				v.AddArgs(args...)374				v.Aux = aux375				v.AuxInt = off376				v.Pos = pos377378				// Make w just a memory copy.379				w.Reset(ssaop.OpCopy)380				w.SetArgs1(wmem)381382				// Skip merged store (w)383				i++384			}385		}386387		memChain = memChain[:0]388	}389390	// prevStore returns the previous store in the391	// same block, or nil if there are none.392	prevStore := func(v *ssa.Value) *ssa.Value {393		if v.Op == ssaop.OpInitMem || v.Op == ssaop.OpPhi {394			return nil395		}396		m := v.MemoryArg()397		if m.Block != v.Block {398			return nil399		}400		return m401	}402403	// storeWidth returns the width of store,404	// or 0 if it is not a store this pass understands.405	storeWidth := func(op ssaop.Op) int64 {406		if info, ok := pairableStores[op]; ok {407			return info.width408		}409410		// We don't pair these stores, but returning zero here411		// would flush the memory chain.412		var width int64413		switch op {414		case ssaop.OpARM64MOVHstore:415			width = 2416		case ssaop.OpARM64MOVBstore:417			width = 1418		default:419			width = 0420		}421422		return width423	}424425	for _, b := range f.Blocks {426		memChain = memChain[:0]427428		// Find last store in block, so we can429		// walk the stores last to first.430		// Last to first helps ensure that the rewrites we431		// perform do not get in the way of subsequent rewrites.432		for _, v := range b.Values {433			if v.Type.IsMemory() {434				last[v.ID] = true435			}436		}437		for _, v := range b.Values {438			if v.Type.IsMemory() {439				if m := prevStore(v); m != nil {440					last[m.ID] = false441				}442			}443		}444		var lastMem *ssa.Value445		for _, v := range b.Values {446			if last[v.ID] {447				lastMem = v448				break449			}450		}451452		// Iterate over memory stores, accumulating them in memChain for potential merging.453		// Flush the chain when reordering is unsafe or a conflict is detected.454		for v := lastMem; v != nil; v = prevStore(v) {455			writeSize := storeWidth(v.Op)456457			if writeSize == 0 {458				// We can't reorder stores with calls or other instructions459				// with writeSize == 0.460				flushMemChain()461				continue462			}463			if v.Uses != 1 && len(memChain) > 0 ||464				len(memChain) > 0 && (v.Args[0] != memChain[0].Args[0] || v.Aux != memChain[0].Aux) ||465				len(memChain) == limit {466				// 1. If v has multiple uses and it is not the latest store in the chain,467				// we cannot merge it with other store instructions.468				//469				// 2. If v has a different base pointer or Aux value from the current chain,470				// we need to flush memChain and start a new one with v.471				//472				// 3. If memChain length limit is exceeded, we also need to flush the chain473				// and start a new one with v.474				//475				// Only look back so far.476				// This keeps us in O(n) territory, and it477				// also prevents us from keeping values478				// in registers for too long (and thus479				// needing to spill them).480				flushMemChain()481			}482483			for _, w := range memChain {484				wWriteSize := storeWidth(w.Op)485				if ssa.Overlap(w.AuxInt, wWriteSize, v.AuxInt, writeSize) {486					// Aliases with w's location.487					// Flush the chain and start a new one with v.488					flushMemChain()489					break490				}491			}492493			memChain = append(memChain, v)494		}495		flushMemChain()496	}497}