1// Copyright 2018 The Go Authors. All rights reserved.2// Use of this source code is governed by a BSD-style3// license that can be found in the LICENSE file.45package ssacompile67import (8	"fmt"910	"cmd/compile/internal/base"11	"cmd/compile/internal/ssa"12	"cmd/compile/internal/ssa/block"13	"cmd/compile/internal/ssa/ssaop"14	"cmd/compile/internal/types"15)1617type indVarFlags uint81819const (20	indVarMinExc indVarFlags = 1 << iota // minimum value is exclusive (default: inclusive)21	indVarMaxInc                         // maximum value is inclusive (default: exclusive)22)2324type indVar struct {25	ind   *ssa.Value // induction variable26	nxt   *ssa.Value // the incremented variable27	min   *ssa.Value // minimum value, inclusive/exclusive depends on flags28	max   *ssa.Value // maximum value, inclusive/exclusive depends on flags29	entry *ssa.Block // the block where the edge from the succeeded comparison of the induction variable goes to, means when the bound check has passed.30	step  int64      // it will always be positive.31	flags indVarFlags32	// Invariant: for all blocks dominated by entry:33	//	min <= ind <  max    [if flags == 0]34	//	min <  ind <  max    [if flags == indVarMinExc]35	//	min <= ind <= max    [if flags == indVarMaxInc]36	//	min <  ind <= max    [if flags == indVarMinExc|indVarMaxInc]37}3839// parseIndVar checks whether the SSA value passed as argument is a valid induction40// variable, and, if so, extracts:41//   - the minimum bound42//   - the increment value43//   - the "next" value (SSA value that is Phi'd into the induction variable every loop)44//   - the header's edge returning from the body45//46// Currently, we detect induction variables that match (Phi min nxt),47// with nxt being (Add inc ind).48// If it can't parse the induction variable correctly, it returns (nil, nil, nil).49func parseIndVar(ind *ssa.Value) (min, inc, nxt *ssa.Value, loopReturn ssa.Edge) {50	if ind.Op != ssaop.OpPhi {51		return52	}5354	if n := ind.Args[0]; (n.Op == ssaop.OpAdd64 || n.Op == ssaop.OpAdd32 || n.Op == ssaop.OpAdd16 || n.Op == ssaop.OpAdd8) && (n.Args[0] == ind || n.Args[1] == ind) {55		min, nxt, loopReturn = ind.Args[1], n, ind.Block.Preds[0]56	} else if n := ind.Args[1]; (n.Op == ssaop.OpAdd64 || n.Op == ssaop.OpAdd32 || n.Op == ssaop.OpAdd16 || n.Op == ssaop.OpAdd8) && (n.Args[0] == ind || n.Args[1] == ind) {57		min, nxt, loopReturn = ind.Args[0], n, ind.Block.Preds[1]58	} else {59		// Not a recognized induction variable.60		return61	}6263	if nxt.Args[0] == ind { // nxt = ind + inc64		inc = nxt.Args[1]65	} else if nxt.Args[1] == ind { // nxt = inc + ind66		inc = nxt.Args[0]67	} else {68		panic("unreachable") // one of the cases must be true from the above.69	}7071	return72}7374// findIndVar finds induction variables in a function.75//76// Look for variables and blocks that satisfy the following77//78//	 loop:79//	   ind = (Phi min nxt),80//	   if ind < max81//	     then goto enter_loop82//	     else goto exit_loop83//84//	   enter_loop:85//		do something86//	      nxt = inc + ind87//		goto loop88//89//	 exit_loop:90//91// We may have more than one induction variables, the loop in the go92// source code may looks like this:93//94//	for i >= 0 && j >= 0 {95//		// use i and j96//		i--97//		j--98//	}99//100// So, also look for variables and blocks that satisfy the following101//102//	loop:103//	  i = (Phi maxi nxti)104//	  j = (Phi maxj nxtj)105//	  if i >= mini106//	    then goto check_j107//	    else goto exit_loop108//109//	check_j:110//	  if j >= minj111//	    then goto enter_loop112//	    else goto exit_loop113//114//	enter_loop:115//	  do something116//	  nxti = i - di117//	  nxtj = j - dj118//	  goto loop119//120//	exit_loop:121func findIndVar(f *ssa.Func) []indVar {122	var iv []indVar123	sdom := f.Sdom()124125nextblock:126	for _, b := range f.Blocks {127		if b.Kind != block.BlockIf {128			continue129		}130		c := b.Controls[0]131		for idx := range 2 {132			// Check that the control if it either ind </<= limit or limit </<= ind.133			// TODO: Handle unsigned comparisons?134			inclusive := false135			switch c.Op {136			case ssaop.OpLeq64, ssaop.OpLeq32, ssaop.OpLeq16, ssaop.OpLeq8:137				inclusive = true138			case ssaop.OpLess64, ssaop.OpLess32, ssaop.OpLess16, ssaop.OpLess8:139			default:140				continue nextblock141			}142143			less := idx == 0144			// induction variable, ending value145			ind, limit := c.Args[idx], c.Args[1-idx]146			// starting value, increment value, next value, loop return edge147			init, inc, nxt, loopReturn := parseIndVar(ind)148			if init == nil {149				continue // this is not an induction variable150			}151152			// This is ind.Block.Preds, not b.Preds. That's a restriction on the loop header,153			// not the comparison block.154			if len(ind.Block.Preds) != 2 {155				continue156			}157158			// Expect the increment to be a nonzero constant.159			if !inc.IsGenericIntConst() {160				continue161			}162			step := inc.AuxInt163			if step == 0 {164				continue165			}166			// step == minInt64 cannot be safely negated below, because -step167			// overflows back to minInt64. The later underflow checks need a168			// positive magnitude, so reject this case here.169			if step == minSignedValue(ind.Type) {170				continue171			}172173			// startBody is the edge that eventually returns to the loop header.174			var startBody ssa.Edge175			switch {176			case sdom.IsAncestorEq(b.Succs[0].B, loopReturn.B):177				startBody = b.Succs[0]178			case sdom.IsAncestorEq(b.Succs[1].B, loopReturn.B):179				// if x { goto exit } else { goto entry } is identical to if !x { goto entry } else { goto exit }180				startBody = b.Succs[1]181				less = !less182				inclusive = !inclusive183			default:184				continue185			}186187			// Increment sign must match comparison direction.188			// When incrementing, the termination comparison must be ind </<= limit.189			// When decrementing, the termination comparison must be ind >/>= limit.190			// See issue 26116.191			if step > 0 && !less {192				continue193			}194			if step < 0 && less {195				continue196			}197198			// Up to now we extracted the induction variable (ind),199			// the increment delta (inc), the temporary sum (nxt),200			// the initial value (init) and the limiting value (limit).201			//202			// We also know that ind has the form (Phi init nxt) where203			// nxt is (Add inc nxt) which means: 1) inc dominates nxt204			// and 2) there is a loop starting at inc and containing nxt.205			//206			// We need to prove that the induction variable is incremented207			// only when it's smaller than the limiting value.208			// Two conditions must happen listed below to accept ind209			// as an induction variable.210211			// First condition: the entry block has a single predecessor.212			// The entry now means the in-loop edge where the induction variable213			// comparison succeeded. Its predecessor is not necessarily the header214			// block. This implies that b.Succs[0] is reached iff ind < limit.215			if len(startBody.B.Preds) != 1 {216				// the other successor must exit the loop.217				continue218			}219220			// Second condition: startBody.b dominates nxt so that221			// nxt is computed when inc < limit.222			if !sdom.IsAncestorEq(startBody.B, nxt.Block) {223				// inc+ind can only be reached through the branch that confirmed the224				// induction variable is in bounds.225				continue226			}227228			// Check for overflow/underflow. We need to make sure that inc never causes229			// the induction variable to wrap around.230			// We use a function wrapper here for easy return true / return false / keep going logic.231			// This function returns true if the increment will never overflow/underflow.232			ok := func() bool {233				if step > 0 {234					if limit.IsGenericIntConst() {235						// Figure out the actual largest value.236						v := limit.AuxInt237						if !inclusive {238							if v == minSignedValue(limit.Type) {239								return false // < minint is never satisfiable.240							}241							v--242						}243						if init.IsGenericIntConst() {244							// Use stride to compute a better lower limit.245							if init.AuxInt > v {246								return false247							}248							// TODO(1.27): investigate passing a smaller-magnitude overflow limit to addU249							// for addWillOverflow.250							v = addU(init.AuxInt, diff(v, init.AuxInt)/uint64(step)*uint64(step))251						}252						if addWillOverflow(v, step, maxSignedValue(ind.Type)) {253							return false254						}255						if inclusive && v != limit.AuxInt || !inclusive && v+1 != limit.AuxInt {256							// We know a better limit than the programmer did. Use our limit instead.257							limit = f.ConstVal(limit.Op, limit.Type, v, true)258							inclusive = true259						}260						return true261					}262					if step == 1 && !inclusive {263						// Can't overflow because maxint is never a possible value.264						return true265					}266					// If the limit is not a constant, check to see if it is a267					// negative offset from a known non-negative value.268					knn, k := findKNN(limit)269					if knn == nil || k < 0 {270						return false271					}272					// limit == (something nonnegative) - k. That subtraction can't underflow, so273					// we can trust it.274					if inclusive {275						// ind <= knn - k cannot overflow if step is at most k276						return step <= k277					}278					// ind < knn - k cannot overflow if step is at most k+1279					return step <= k+1 && k != maxSignedValue(limit.Type)280281					// TODO: other unrolling idioms282					// for i := 0; i < KNN - KNN % k ; i += k283					// for i := 0; i < KNN&^(k-1) ; i += k // k a power of 2284					// for i := 0; i < KNN&(-k) ; i += k // k a power of 2285				} else { // step < 0286					if limit.IsGenericIntConst() {287						// Figure out the actual smallest value.288						v := limit.AuxInt289						if !inclusive {290							if v == maxSignedValue(limit.Type) {291								return false // > maxint is never satisfiable.292							}293							v++294						}295						if init.IsGenericIntConst() {296							// Use stride to compute a better lower limit.297							if init.AuxInt < v {298								return false299							}300							// TODO(1.27): investigate passing a smaller-magnitude underflow limit to subU301							// for subWillUnderflow.302							v = subU(init.AuxInt, diff(init.AuxInt, v)/uint64(-step)*uint64(-step))303						}304						if subWillUnderflow(v, -step, minSignedValue(ind.Type)) {305							return false306						}307						if inclusive && v != limit.AuxInt || !inclusive && v-1 != limit.AuxInt {308							// We know a better limit than the programmer did. Use our limit instead.309							limit = f.ConstVal(limit.Op, limit.Type, v, true)310							inclusive = true311						}312						return true313					}314					if step == -1 && !inclusive {315						// Can't underflow because minint is never a possible value.316						return true317					}318				}319				return false320			}321322			if ok() {323				flags := indVarFlags(0)324				var min, max *ssa.Value325				if step > 0 {326					min = init327					max = limit328					if inclusive {329						flags |= indVarMaxInc330					}331				} else {332					min = limit333					max = init334					flags |= indVarMaxInc335					if !inclusive {336						flags |= indVarMinExc337					}338					step = -step339				}340				if f.Pass.Debug >= 1 {341					printIndVar(b, ind, min, max, step, flags)342				}343344				iv = append(iv, indVar{345					ind: ind,346					nxt: nxt,347					min: min,348					max: max,349					// This is startBody.b, where startBody is the edge from the comparison for the350					// induction variable, not necessarily the in-loop edge from the loop header.351					// Induction variable bounds are not valid in the loop before this edge.352					entry: startBody.B,353					step:  step,354					flags: flags,355				})356				b.Logf("found induction variable %v (inc = %v, min = %v, max = %v)\n", ind, inc, min, max)357			}358		}359	}360361	return iv362}363364// subWillUnderflow checks if x - y underflows the min value.365// y must be positive.366func subWillUnderflow(x, y int64, min int64) bool {367	if y < 0 {368		base.Fatalf("expecting positive value")369	}370	return x < min+y371}372373// addWillOverflow checks if x + y overflows the max value.374// y must be positive.375func addWillOverflow(x, y int64, max int64) bool {376	if y < 0 {377		base.Fatalf("expecting positive value")378	}379	return x > max-y380}381382// diff returns x-y as a uint64. Requires x>=y.383func diff(x, y int64) uint64 {384	if x < y {385		base.Fatalf("diff %d - %d underflowed", x, y)386	}387	return uint64(x - y)388}389390// addU returns x+y. Requires that x+y does not overflow an int64.391func addU(x int64, y uint64) int64 {392	if y >= 1<<63 {393		if x >= 0 {394			base.Fatalf("addU overflowed %d + %d", x, y)395		}396		x += 1<<63 - 1397		x += 1398		y -= 1 << 63399	}400	// TODO(1.27): investigate passing a smaller-magnitude overflow limit in here.401	if addWillOverflow(x, int64(y), maxSignedValue(types.Types[types.TINT64])) {402		base.Fatalf("addU overflowed %d + %d", x, y)403	}404	return x + int64(y)405}406407// subU returns x-y. Requires that x-y does not underflow an int64.408func subU(x int64, y uint64) int64 {409	if y >= 1<<63 {410		if x < 0 {411			base.Fatalf("subU underflowed %d - %d", x, y)412		}413		x -= 1<<63 - 1414		x -= 1415		y -= 1 << 63416	}417	// TODO(1.27): investigate passing a smaller-magnitude underflow limit in here.418	if subWillUnderflow(x, int64(y), minSignedValue(types.Types[types.TINT64])) {419		base.Fatalf("subU underflowed %d - %d", x, y)420	}421	return x - int64(y)422}423424// if v is known to be x - c, where x is known to be nonnegative and c is a425// constant, return x, c. Otherwise return nil, 0.426func findKNN(v *ssa.Value) (*ssa.Value, int64) {427	var x, y *ssa.Value428	x = v429	switch v.Op {430	case ssaop.OpSub64, ssaop.OpSub32, ssaop.OpSub16, ssaop.OpSub8:431		x = v.Args[0]432		y = v.Args[1]433434	case ssaop.OpAdd64, ssaop.OpAdd32, ssaop.OpAdd16, ssaop.OpAdd8:435		x = v.Args[0]436		y = v.Args[1]437		if x.IsGenericIntConst() {438			x, y = y, x439		}440	}441	switch x.Op {442	case ssaop.OpSliceLen, ssaop.OpStringLen, ssaop.OpSliceCap:443	default:444		return nil, 0445	}446	if y == nil {447		return x, 0448	}449	if !y.IsGenericIntConst() {450		return nil, 0451	}452	if v.Op == ssaop.OpAdd64 || v.Op == ssaop.OpAdd32 || v.Op == ssaop.OpAdd16 || v.Op == ssaop.OpAdd8 {453		return x, -y.AuxInt454	}455	return x, y.AuxInt456}457458func printIndVar(b *ssa.Block, i, min, max *ssa.Value, inc int64, flags indVarFlags) {459	mb1, mb2 := "[", "]"460	if flags&indVarMinExc != 0 {461		mb1 = "("462	}463	if flags&indVarMaxInc == 0 {464		mb2 = ")"465	}466467	mlim1, mlim2 := fmt.Sprint(min.AuxInt), fmt.Sprint(max.AuxInt)468	if !min.IsGenericIntConst() {469		if b.Func.Pass.Debug >= 2 {470			mlim1 = fmt.Sprint(min)471		} else {472			mlim1 = "?"473		}474	}475	if !max.IsGenericIntConst() {476		if b.Func.Pass.Debug >= 2 {477			mlim2 = fmt.Sprint(max)478		} else {479			mlim2 = "?"480		}481	}482	extra := ""483	if b.Func.Pass.Debug >= 2 {484		extra = fmt.Sprintf(" (%s)", i)485	}486	b.Func.Warnl(b.Pos, "Induction variable: limits %v%v,%v%v, increment %d%s", mb1, mlim1, mlim2, mb2, inc, extra)487}488489func minSignedValue(t *types.Type) int64 {490	return -1 << (t.Size()*8 - 1)491}492493func maxSignedValue(t *types.Type) int64 {494	return 1<<((t.Size()*8)-1) - 1495}