1// Copyright 2016 The Go Authors. All rights reserved.2// Use of this source code is governed by a BSD-style3// license that can be found in the LICENSE file.45package ssagen67import (8	"container/heap"9	"fmt"1011	"cmd/compile/internal/ir"12	"cmd/compile/internal/ssa"13	"cmd/compile/internal/ssa/ssaop"14	"cmd/compile/internal/types"15	"cmd/internal/src"16)1718// This file contains the algorithm to place phi nodes in a function.19// For small functions, we use Braun, Buchwald, Hack, Leißa, Mallon, and Zwinkau.20// https://pp.info.uni-karlsruhe.de/uploads/publikationen/braun13cc.pdf21// For large functions, we use Sreedhar & Gao: A Linear Time Algorithm for Placing Φ-Nodes.22// http://citeseerx.ist.psu.edu/viewdoc/download?doi=10.1.1.8.1979&rep=rep1&type=pdf2324const smallBlocks = 5002526const debugPhi = false2728// fwdRefAux wraps an arbitrary ir.Node as an ssa.Aux for use with OpFwdref.29type fwdRefAux struct {30	_ [0]func() // ensure ir.Node isn't compared for equality31	N ir.Node32}3334func (fwdRefAux) CanBeAnSSAAux() {}3536// insertPhis finds all the places in the function where a phi is37// necessary and inserts them.38// Uses FwdRef ops to find all uses of variables, and s.defvars to find39// all definitions.40// Phi values are inserted, and all FwdRefs are changed to a Copy41// of the appropriate phi or definition.42// TODO: make this part of cmd/compile/internal/ssa somehow?43func (s *state) insertPhis() {44	if len(s.f.Blocks) <= smallBlocks {45		sps := simplePhiState{s: s, f: s.f, defvars: s.defvars}46		sps.insertPhis()47		return48	}49	ps := phiState{s: s, f: s.f, defvars: s.defvars}50	ps.insertPhis()51}5253type phiState struct {54	s       *state                   // SSA state55	f       *ssa.Func                // function to work on56	defvars []map[ir.Node]*ssa.Value // defined variables at end of each block5758	varnum map[ir.Node]int32 // variable numbering5960	// properties of the dominator tree61	idom  []*ssa.Block // dominator parents62	tree  []domBlock   // dominator child+sibling63	level []int32      // level in dominator tree (0 = root or unreachable, 1 = children of root, ...)6465	// scratch locations66	priq   blockHeap    // priority queue of blocks, higher level (toward leaves) = higher priority67	q      []*ssa.Block // inner loop queue68	queued *sparseSet   // has been put in q69	hasPhi *sparseSet   // has a phi70	hasDef *sparseSet   // has a write of the variable we're processing7172	// miscellaneous73	placeholder *ssa.Value // value to use as a "not set yet" placeholder.74}7576func (s *phiState) insertPhis() {77	if debugPhi {78		fmt.Println(s.f.String())79	}8081	// Find all the variables for which we need to match up reads & writes.82	// This step prunes any basic-block-only variables from consideration.83	// Generate a numbering for these variables.84	s.varnum = map[ir.Node]int32{}85	var vars []ir.Node86	var vartypes []*types.Type87	for _, b := range s.f.Blocks {88		for _, v := range b.Values {89			if v.Op != ssaop.OpFwdRef {90				continue91			}92			var_ := v.Aux.(fwdRefAux).N9394			// Optimization: look back 1 block for the definition.95			if len(b.Preds) == 1 {96				c := b.Preds[0].Block()97				if w := s.defvars[c.ID][var_]; w != nil {98					v.Op = ssaop.OpCopy99					v.Aux = nil100					v.AddArg(w)101					continue102				}103			}104105			if _, ok := s.varnum[var_]; ok {106				continue107			}108			s.varnum[var_] = int32(len(vartypes))109			if debugPhi {110				fmt.Printf("var%d = %v\n", len(vartypes), var_)111			}112			vars = append(vars, var_)113			vartypes = append(vartypes, v.Type)114		}115	}116117	if len(vartypes) == 0 {118		return119	}120121	// Find all definitions of the variables we need to process.122	// defs[n] contains all the blocks in which variable number n is assigned.123	defs := make([][]*ssa.Block, len(vartypes))124	for _, b := range s.f.Blocks {125		for var_ := range s.defvars[b.ID] { // TODO: encode defvars some other way (explicit ops)? make defvars[n] a slice instead of a map.126			if n, ok := s.varnum[var_]; ok {127				defs[n] = append(defs[n], b)128			}129		}130	}131132	// Make dominator tree.133	s.idom = s.f.Idom()134	s.tree = make([]domBlock, s.f.NumBlocks())135	for _, b := range s.f.Blocks {136		p := s.idom[b.ID]137		if p != nil {138			s.tree[b.ID].sibling = s.tree[p.ID].firstChild139			s.tree[p.ID].firstChild = b140		}141	}142	// Compute levels in dominator tree.143	// With parent pointers we can do a depth-first walk without144	// any auxiliary storage.145	s.level = make([]int32, s.f.NumBlocks())146	b := s.f.Entry147levels:148	for {149		if p := s.idom[b.ID]; p != nil {150			s.level[b.ID] = s.level[p.ID] + 1151			if debugPhi {152				fmt.Printf("level %s = %d\n", b, s.level[b.ID])153			}154		}155		if c := s.tree[b.ID].firstChild; c != nil {156			b = c157			continue158		}159		for {160			if c := s.tree[b.ID].sibling; c != nil {161				b = c162				continue levels163			}164			b = s.idom[b.ID]165			if b == nil {166				break levels167			}168		}169	}170171	// Allocate scratch locations.172	s.priq.level = s.level173	s.q = make([]*ssa.Block, 0, s.f.NumBlocks())174	s.queued = newSparseSet(s.f.NumBlocks())175	s.hasPhi = newSparseSet(s.f.NumBlocks())176	s.hasDef = newSparseSet(s.f.NumBlocks())177	s.placeholder = s.s.entryNewValue0(ssaop.OpUnknown, types.TypeInvalid)178179	// Generate phi ops for each variable.180	for n := range vartypes {181		s.insertVarPhis(n, vars[n], defs[n], vartypes[n])182	}183184	// Resolve FwdRefs to the correct write or phi.185	s.resolveFwdRefs()186187	// Erase variable numbers stored in AuxInt fields of phi ops. They are no longer needed.188	for _, b := range s.f.Blocks {189		for _, v := range b.Values {190			if v.Op == ssaop.OpPhi {191				v.AuxInt = 0192			}193			// Any remaining FwdRefs are dead code.194			if v.Op == ssaop.OpFwdRef {195				v.Op = ssaop.OpUnknown196				v.Aux = nil197			}198		}199	}200}201202func (s *phiState) insertVarPhis(n int, var_ ir.Node, defs []*ssa.Block, typ *types.Type) {203	priq := &s.priq204	q := s.q205	queued := s.queued206	queued.clear()207	hasPhi := s.hasPhi208	hasPhi.clear()209	hasDef := s.hasDef210	hasDef.clear()211212	// Add defining blocks to priority queue.213	for _, b := range defs {214		priq.a = append(priq.a, b)215		hasDef.add(b.ID)216		if debugPhi {217			fmt.Printf("def of var%d in %s\n", n, b)218		}219	}220	heap.Init(priq)221222	// Visit blocks defining variable n, from deepest to shallowest.223	for len(priq.a) > 0 {224		currentRoot := heap.Pop(priq).(*ssa.Block)225		if debugPhi {226			fmt.Printf("currentRoot %s\n", currentRoot)227		}228		// Walk subtree below definition.229		// Skip subtrees we've done in previous iterations.230		// Find edges exiting tree dominated by definition (the dominance frontier).231		// Insert phis at target blocks.232		if queued.contains(currentRoot.ID) {233			s.s.Fatalf("root already in queue")234		}235		q = append(q, currentRoot)236		queued.add(currentRoot.ID)237		for len(q) > 0 {238			b := q[len(q)-1]239			q = q[:len(q)-1]240			if debugPhi {241				fmt.Printf("  processing %s\n", b)242			}243244			currentRootLevel := s.level[currentRoot.ID]245			for _, e := range b.Succs {246				c := e.Block()247				// TODO: if the variable is dead at c, skip it.248				if s.level[c.ID] > currentRootLevel {249					// a D-edge, or an edge whose target is in currentRoot's subtree.250					continue251				}252				if hasPhi.contains(c.ID) {253					continue254				}255				// Add a phi to block c for variable n.256				hasPhi.add(c.ID)257				v := c.NewValue0I(s.s.blockStarts[b.ID], ssaop.OpPhi, typ, int64(n))258				// Note: we store the variable number in the phi's AuxInt field. Used temporarily by phi building.259				if var_.Op() == ir.ONAME {260					s.s.addNamedValue(var_.(*ir.Name), v)261				}262				for range c.Preds {263					v.AddArg(s.placeholder) // Actual args will be filled in by resolveFwdRefs.264				}265				if debugPhi {266					fmt.Printf("new phi for var%d in %s: %s\n", n, c, v)267				}268				if !hasDef.contains(c.ID) {269					// There's now a new definition of this variable in block c.270					// Add it to the priority queue to explore.271					heap.Push(priq, c)272					hasDef.add(c.ID)273				}274			}275276			// Visit children if they have not been visited yet.277			for c := s.tree[b.ID].firstChild; c != nil; c = s.tree[c.ID].sibling {278				if !queued.contains(c.ID) {279					q = append(q, c)280					queued.add(c.ID)281				}282			}283		}284	}285}286287// resolveFwdRefs links all FwdRef uses up to their nearest dominating definition.288func (s *phiState) resolveFwdRefs() {289	// Do a depth-first walk of the dominator tree, keeping track290	// of the most-recently-seen value for each variable.291292	// Map from variable ID to SSA value at the current point of the walk.293	values := make([]*ssa.Value, len(s.varnum))294	for i := range values {295		values[i] = s.placeholder296	}297298	// Stack of work to do.299	type stackEntry struct {300		b *ssa.Block // block to explore301302		// variable/value pair to reinstate on exit303		n int32 // variable ID304		v *ssa.Value305306		// Note: only one of b or n,v will be set.307	}308	var stk []stackEntry309310	stk = append(stk, stackEntry{b: s.f.Entry})311	for len(stk) > 0 {312		work := stk[len(stk)-1]313		stk = stk[:len(stk)-1]314315		b := work.b316		if b == nil {317			// On exit from a block, this case will undo any assignments done below.318			values[work.n] = work.v319			continue320		}321322		// Process phis as new defs. They come before FwdRefs in this block.323		for _, v := range b.Values {324			if v.Op != ssaop.OpPhi {325				continue326			}327			n := int32(v.AuxInt)328			// Remember the old assignment so we can undo it when we exit b.329			stk = append(stk, stackEntry{n: n, v: values[n]})330			// Record the new assignment.331			values[n] = v332		}333334		// Replace a FwdRef op with the current incoming value for its variable.335		for _, v := range b.Values {336			if v.Op != ssaop.OpFwdRef {337				continue338			}339			n := s.varnum[v.Aux.(fwdRefAux).N]340			v.Op = ssaop.OpCopy341			v.Aux = nil342			v.AddArg(values[n])343		}344345		// Establish values for variables defined in b.346		for var_, v := range s.defvars[b.ID] {347			n, ok := s.varnum[var_]348			if !ok {349				// some variable not live across a basic block boundary.350				continue351			}352			// Remember the old assignment so we can undo it when we exit b.353			stk = append(stk, stackEntry{n: n, v: values[n]})354			// Record the new assignment.355			values[n] = v356		}357358		// Replace phi args in successors with the current incoming value.359		for _, e := range b.Succs {360			c, i := e.Block(), e.Index()361			for j := len(c.Values) - 1; j >= 0; j-- {362				v := c.Values[j]363				if v.Op != ssaop.OpPhi {364					break // All phis will be at the end of the block during phi building.365				}366				// Only set arguments that have been resolved.367				// For very wide CFGs, this significantly speeds up phi resolution.368				// See golang.org/issue/8225.369				if w := values[v.AuxInt]; w.Op != ssaop.OpUnknown {370					v.SetArg(i, w)371				}372			}373		}374375		// Walk children in dominator tree.376		for c := s.tree[b.ID].firstChild; c != nil; c = s.tree[c.ID].sibling {377			stk = append(stk, stackEntry{b: c})378		}379	}380}381382// domBlock contains extra per-block information to record the dominator tree.383type domBlock struct {384	firstChild *ssa.Block // first child of block in dominator tree385	sibling    *ssa.Block // next child of parent in dominator tree386}387388// A block heap is used as a priority queue to implement the PiggyBank389// from Sreedhar and Gao.  That paper uses an array which is better390// asymptotically but worse in the common case when the PiggyBank391// holds a sparse set of blocks.392type blockHeap struct {393	a     []*ssa.Block // block IDs in heap394	level []int32      // depth in dominator tree (static, used for determining priority)395}396397func (h *blockHeap) Len() int      { return len(h.a) }398func (h *blockHeap) Swap(i, j int) { a := h.a; a[i], a[j] = a[j], a[i] }399400func (h *blockHeap) Push(x any) {401	v := x.(*ssa.Block)402	h.a = append(h.a, v)403}404func (h *blockHeap) Pop() any {405	old := h.a406	n := len(old)407	x := old[n-1]408	h.a = old[:n-1]409	return x410}411func (h *blockHeap) Less(i, j int) bool {412	return h.level[h.a[i].ID] > h.level[h.a[j].ID]413}414415// TODO: stop walking the iterated domininance frontier when416// the variable is dead. Maybe detect that by checking if the417// node we're on is reverse dominated by all the reads?418// Reverse dominated by the highest common successor of all the reads?419420// copy of ../ssa/sparseset.go421// TODO: move this file to ../ssa, then use sparseSet there.422type sparseSet struct {423	dense  []ssa.ID424	sparse []int32425}426427// newSparseSet returns a sparseSet that can represent428// integers between 0 and n-1.429func newSparseSet(n int) *sparseSet {430	return &sparseSet{dense: nil, sparse: make([]int32, n)}431}432433func (s *sparseSet) contains(x ssa.ID) bool {434	i := s.sparse[x]435	return i < int32(len(s.dense)) && s.dense[i] == x436}437438func (s *sparseSet) add(x ssa.ID) {439	i := s.sparse[x]440	if i < int32(len(s.dense)) && s.dense[i] == x {441		return442	}443	s.dense = append(s.dense, x)444	s.sparse[x] = int32(len(s.dense)) - 1445}446447func (s *sparseSet) clear() {448	s.dense = s.dense[:0]449}450451// Variant to use for small functions.452type simplePhiState struct {453	s         *state                   // SSA state454	f         *ssa.Func                // function to work on455	fwdrefs   []*ssa.Value             // list of FwdRefs to be processed456	defvars   []map[ir.Node]*ssa.Value // defined variables at end of each block457	reachable []bool                   // which blocks are reachable458}459460func (s *simplePhiState) insertPhis() {461	s.reachable = ssa.ReachableBlocks(s.f)462463	// Find FwdRef ops.464	for _, b := range s.f.Blocks {465		for _, v := range b.Values {466			if v.Op != ssaop.OpFwdRef {467				continue468			}469			s.fwdrefs = append(s.fwdrefs, v)470			var_ := v.Aux.(fwdRefAux).N471			if _, ok := s.defvars[b.ID][var_]; !ok {472				s.defvars[b.ID][var_] = v // treat FwdDefs as definitions.473			}474		}475	}476477	var args []*ssa.Value478479loop:480	for len(s.fwdrefs) > 0 {481		v := s.fwdrefs[len(s.fwdrefs)-1]482		s.fwdrefs = s.fwdrefs[:len(s.fwdrefs)-1]483		b := v.Block484		var_ := v.Aux.(fwdRefAux).N485		if b == s.f.Entry {486			// No variable should be live at entry.487			s.s.Fatalf("value %v (%v) incorrectly live at entry", var_, v)488		}489		if !s.reachable[b.ID] {490			// This block is dead.491			// It doesn't matter what we use here as long as it is well-formed.492			v.Op = ssaop.OpUnknown493			v.Aux = nil494			continue495		}496		// Find variable value on each predecessor.497		args = args[:0]498		for _, e := range b.Preds {499			args = append(args, s.lookupVarOutgoing(e.Block(), v.Type, var_, v.Pos))500		}501502		// Decide if we need a phi or not. We need a phi if there503		// are two different args (which are both not v).504		var w *ssa.Value505		for _, a := range args {506			if a == v {507				continue // self-reference508			}509			if a == w {510				continue // already have this witness511			}512			if w != nil {513				// two witnesses, need a phi value514				v.Op = ssaop.OpPhi515				v.AddArgs(args...)516				v.Aux = nil517				v.Pos = s.s.blockStarts[b.ID]518				continue loop519			}520			w = a // save witness521		}522		if w == nil {523			s.s.Fatalf("no witness for reachable phi %s", v)524		}525		// One witness. Make v a copy of w.526		v.Op = ssaop.OpCopy527		v.Aux = nil528		v.AddArg(w)529	}530}531532// lookupVarOutgoing finds the variable's value at the end of block b.533func (s *simplePhiState) lookupVarOutgoing(b *ssa.Block, t *types.Type, var_ ir.Node, line src.XPos) *ssa.Value {534	for {535		if v := s.defvars[b.ID][var_]; v != nil {536			return v537		}538		// The variable is not defined by b and we haven't looked it up yet.539		// If b has exactly one predecessor, loop to look it up there.540		// Otherwise, give up and insert a new FwdRef and resolve it later.541		if len(b.Preds) != 1 {542			break543		}544		b = b.Preds[0].Block()545		if !s.reachable[b.ID] {546			// This is rare; it happens with oddly interleaved infinite loops in dead code.547			// See issue 19783.548			break549		}550	}551	// Generate a FwdRef for the variable and return that.552	v := b.NewValue0A(line, ssaop.OpFwdRef, t, fwdRefAux{N: var_})553	s.defvars[b.ID][var_] = v554	if var_.Op() == ir.ONAME {555		s.s.addNamedValue(var_.(*ir.Name), v)556	}557	s.fwdrefs = append(s.fwdrefs, v)558	return v559}