1// Copyright 2011 The Go Authors. All rights reserved.2// Use of this source code is governed by a BSD-style3// license that can be found in the LICENSE file.4//5// The inlining facility makes 2 passes: first CanInline determines which6// functions are suitable for inlining, and for those that are it7// saves a copy of the body. Then InlineCalls walks each function body to8// expand calls to inlinable functions.9//10// The Debug.l flag controls the aggressiveness. Note that main() swaps level 0 and 1,11// making 1 the default and -l disable. Additional levels (beyond -l) may be buggy and12// are not supported.13//      0: disabled14//      1: 80-nodes leaf functions, oneliners, panic, lazy typechecking (default)15//      2: (unassigned)16//      3: (unassigned)17//      4: allow non-leaf functions18//19// At some point this may get another default and become switch-offable with -N.20//21// The -d typcheckinl flag enables early typechecking of all imported bodies,22// which is useful to flush out bugs.23//24// The Debug.m flag enables diagnostic output.  a single -m is useful for verifying25// which calls get inlined or not, more is for debugging, and may go away at any point.2627package inline2829import (30	"fmt"31	"go/constant"32	"internal/buildcfg"33	"strconv"34	"strings"3536	"cmd/compile/internal/base"37	"cmd/compile/internal/inline/inlheur"38	"cmd/compile/internal/ir"39	"cmd/compile/internal/logopt"40	"cmd/compile/internal/pgoir"41	"cmd/compile/internal/typecheck"42	"cmd/compile/internal/types"43	"cmd/internal/obj"44	"cmd/internal/pgo"45	"cmd/internal/src"46)4748// Inlining budget parameters, gathered in one place49const (50	inlineMaxBudget       = 8051	inlineExtraAppendCost = 052	// default is to inline if there's at most one call. -l=4 overrides this by using 1 instead.53	inlineExtraCallCost  = 57              // 57 was benchmarked to provided most benefit with no bad surprises; see https://github.com/golang/go/issues/19348#issuecomment-43937074254	inlineParamCallCost  = 17              // calling a parameter only costs this much extra (inlining might expose a constant function)55	inlineExtraPanicCost = 1               // do not penalize inlining panics.56	inlineExtraThrowCost = inlineMaxBudget // with current (2018-05/1.11) code, inlining runtime.throw does not help.5758	inlineBigFunctionNodes      = 5000                 // Functions with this many nodes are considered "big".59	inlineBigFunctionMaxCost    = 20                   // Max cost of inlinee when inlining into a "big" function.60	inlineClosureCalledOnceCost = 10 * inlineMaxBudget // if a closure is just called once, inline it.61)6263var (64	// List of all hot callee nodes.65	// TODO(prattmic): Make this non-global.66	candHotCalleeMap = make(map[*pgoir.IRNode]struct{})6768	// Set of functions that contain hot call sites.69	hasHotCall = make(map[*ir.Func]struct{})7071	// List of all hot call sites. CallSiteInfo.Callee is always nil.72	// TODO(prattmic): Make this non-global.73	candHotEdgeMap = make(map[pgoir.CallSiteInfo]struct{})7475	// Threshold in percentage for hot callsite inlining.76	inlineHotCallSiteThresholdPercent float647778	// Threshold in CDF percentage for hot callsite inlining,79	// that is, for a threshold of X the hottest callsites that80	// make up the top X% of total edge weight will be81	// considered hot for inlining candidates.82	inlineCDFHotCallSiteThresholdPercent = float64(99)8384	// Budget increased due to hotness.85	inlineHotMaxBudget int32 = 200086)8788func IsPgoHotFunc(fn *ir.Func, profile *pgoir.Profile) bool {89	if profile == nil {90		return false91	}92	if n, ok := profile.WeightedCG.IRNodes[ir.LinkFuncName(fn)]; ok {93		_, ok := candHotCalleeMap[n]94		return ok95	}96	return false97}9899func HasPgoHotInline(fn *ir.Func) bool {100	_, has := hasHotCall[fn]101	return has102}103104// PGOInlinePrologue records the hot callsites from ir-graph.105func PGOInlinePrologue(p *pgoir.Profile) {106	if base.Debug.PGOInlineCDFThreshold != "" {107		if s, err := strconv.ParseFloat(base.Debug.PGOInlineCDFThreshold, 64); err == nil && s >= 0 && s <= 100 {108			inlineCDFHotCallSiteThresholdPercent = s109		} else {110			base.Fatalf("invalid PGOInlineCDFThreshold, must be between 0 and 100")111		}112	}113	var hotCallsites []pgo.NamedCallEdge114	inlineHotCallSiteThresholdPercent, hotCallsites = hotNodesFromCDF(p)115	if base.Debug.PGODebug > 0 {116		fmt.Printf("hot-callsite-thres-from-CDF=%v\n", inlineHotCallSiteThresholdPercent)117	}118119	if x := base.Debug.PGOInlineBudget; x != 0 {120		inlineHotMaxBudget = int32(x)121	}122123	for _, n := range hotCallsites {124		// mark inlineable callees from hot edges125		if callee := p.WeightedCG.IRNodes[n.CalleeName]; callee != nil {126			candHotCalleeMap[callee] = struct{}{}127		}128		// mark hot call sites129		if caller := p.WeightedCG.IRNodes[n.CallerName]; caller != nil && caller.AST != nil {130			csi := pgoir.CallSiteInfo{LineOffset: n.CallSiteOffset, Caller: caller.AST}131			candHotEdgeMap[csi] = struct{}{}132		}133	}134135	if base.Debug.PGODebug >= 3 {136		fmt.Printf("hot-cg before inline in dot format:")137		p.PrintWeightedCallGraphDOT(inlineHotCallSiteThresholdPercent)138	}139}140141// hotNodesFromCDF computes an edge weight threshold and the list of hot142// nodes that make up the given percentage of the CDF. The threshold, as143// a percent, is the lower bound of weight for nodes to be considered hot144// (currently only used in debug prints) (in case of equal weights,145// comparing with the threshold may not accurately reflect which nodes are146// considered hot).147func hotNodesFromCDF(p *pgoir.Profile) (float64, []pgo.NamedCallEdge) {148	cum := int64(0)149	for i, n := range p.NamedEdgeMap.ByWeight {150		w := p.NamedEdgeMap.Weight[n]151		cum += w152		if pgo.WeightInPercentage(cum, p.TotalWeight) > inlineCDFHotCallSiteThresholdPercent {153			// nodes[:i+1] to include the very last node that makes it to go over the threshold.154			// (Say, if the CDF threshold is 50% and one hot node takes 60% of weight, we want to155			// include that node instead of excluding it.)156			return pgo.WeightInPercentage(w, p.TotalWeight), p.NamedEdgeMap.ByWeight[:i+1]157		}158	}159	return 0, p.NamedEdgeMap.ByWeight160}161162// CanInlineFuncs computes whether a batch of functions are inlinable.163func CanInlineFuncs(funcs []*ir.Func, profile *pgoir.Profile) {164	if profile != nil {165		PGOInlinePrologue(profile)166	}167168	if base.Flag.LowerL == 0 {169		return170	}171172	ir.VisitFuncsBottomUp(funcs, func(funcs []*ir.Func, recursive bool) {173		for _, fn := range funcs {174			CanInline(fn, profile)175			if inlheur.Enabled() {176				analyzeFuncProps(fn, profile)177			}178		}179	})180}181182func simdCreditMultiplier(fn *ir.Func) int32 {183	for _, field := range fn.Type().RecvParamsResults() {184		if field.Type.IsSIMD() {185			return 3186		}187	}188	// Sometimes code uses closures, that do not take simd189	// parameters, to perform repetitive SIMD operations.190	// fn.  These really need to be inlined, or the anticipated191	// awesome SIMD performance will be missed.192	for _, v := range fn.ClosureVars {193		if v.Type().IsSIMD() {194			return 16 // <strike>11</strike> 16 ought to be enough.195		}196	}197198	return 1199}200201// inlineBudget determines the max budget for function 'fn' prior to202// analyzing the hairiness of the body of 'fn'. We pass in the pgo203// profile if available (which can change the budget), also a204// 'relaxed' flag, which expands the budget slightly to allow for the205// possibility that a call to the function might have its score206// adjusted downwards. If 'verbose' is set, then print a remark where207// we boost the budget due to PGO.208// Note that inlineCostOK has the final say on whether an inline will209// happen; changes here merely make inlines possible.210func inlineBudget(fn *ir.Func, profile *pgoir.Profile, relaxed bool, verbose bool) int32 {211	// Update the budget for profile-guided inlining.212	budget := int32(inlineMaxBudget)213214	budget *= simdCreditMultiplier(fn)215216	if strings.HasPrefix(ir.FuncName(fn), "runtime_mapaccess2") &&217		fn.Sym().Pkg.Path == "internal/runtime/maps" {218		// Increase budget for mapaccess2* functions so they could be219		// inlined to mapaccess1* wrappers220		budget = inlineHotMaxBudget221		if verbose {222			fmt.Printf("mapaccess enabled increased budget=%v for func=%v\n", budget, ir.PkgFuncName(fn))223		}224	}225226	if IsPgoHotFunc(fn, profile) {227		budget = inlineHotMaxBudget228		if verbose {229			fmt.Printf("hot-node enabled increased budget=%v for func=%v\n", budget, ir.PkgFuncName(fn))230		}231	}232	if relaxed {233		budget += inlheur.BudgetExpansion(inlineMaxBudget)234	}235	if fn.ClosureParent != nil {236		// be very liberal here, if the closure is only called once, the budget is large237		budget = max(budget, inlineClosureCalledOnceCost)238	}239240	return budget241}242243// CanInline determines whether fn is inlineable.244// If so, CanInline saves copies of fn.Body and fn.Dcl in fn.Inl.245// fn and fn.Body will already have been typechecked.246func CanInline(fn *ir.Func, profile *pgoir.Profile) {247	if fn.Nname == nil {248		base.Fatalf("CanInline no nname %+v", fn)249	}250251	var reason string // reason, if any, that the function was not inlined252	if base.Flag.LowerM > 1 || logopt.Enabled() {253		defer func() {254			if reason != "" {255				if base.Flag.LowerM > 1 {256					fmt.Printf("%v: cannot inline %v: %s\n", ir.Line(fn), fn.Nname, reason)257				}258				if logopt.Enabled() {259					logopt.LogOpt(fn.Pos(), "cannotInlineFunction", "inline", ir.FuncName(fn), reason)260				}261			}262		}()263	}264265	reason = InlineImpossible(fn)266	if reason != "" {267		return268	}269	if fn.Typecheck() == 0 {270		base.Fatalf("CanInline on non-typechecked function %v", fn)271	}272273	n := fn.Nname274	if n.Func.InlinabilityChecked() {275		return276	}277	defer n.Func.SetInlinabilityChecked(true)278279	cc := int32(inlineExtraCallCost)280	if base.Flag.LowerL == 4 {281		cc = 1 // this appears to yield better performance than 0.282	}283284	// Used a "relaxed" inline budget if the new inliner is enabled.285	relaxed := inlheur.Enabled()286287	// Compute the inline budget for this func.288	budget := inlineBudget(fn, profile, relaxed, base.Debug.PGODebug > 0)289290	// At this point in the game the function we're looking at may291	// have "stale" autos, vars that still appear in the Dcl list, but292	// which no longer have any uses in the function body (due to293	// elimination by deadcode). We'd like to exclude these dead vars294	// when creating the "Inline.Dcl" field below; to accomplish this,295	// the hairyVisitor below builds up a map of used/referenced296	// locals, and we use this map to produce a pruned Inline.Dcl297	// list. See issue 25459 for more context.298299	dbg := ir.MatchAstDump(fn, "inline")300301	visitor := hairyVisitor{302		curFunc:       fn,303		debug:         isDebugFn(fn),304		isBigFunc:     IsBigFunc(fn),305		budget:        budget,306		maxBudget:     budget,307		extraCallCost: cc,308		profile:       profile,309		dbg:           dbg, // Useful for downstream debugging310	}311312	if visitor.tooHairy(fn) {313		reason = visitor.reason314		if dbg {315			ir.AstDump(fn, "inline, too hairy because "+visitor.reason+", "+ir.FuncName(fn))316		}317		return318	} else if dbg {319		ir.AstDump(fn, "inline, OK, "+ir.FuncName(fn))320	}321322	n.Func.Inl = &ir.Inline{323		Cost:            budget - visitor.budget,324		Dcl:             pruneUnusedAutos(n.Func.Dcl, &visitor),325		HaveDcl:         true,326		CanDelayResults: canDelayResults(fn),327	}328	if base.Flag.LowerM != 0 || logopt.Enabled() {329		noteInlinableFunc(n, fn, budget-visitor.budget)330	}331}332333// noteInlinableFunc issues a message to the user that the specified334// function is inlinable.335func noteInlinableFunc(n *ir.Name, fn *ir.Func, cost int32) {336	if base.Flag.LowerM > 1 {337		fmt.Printf("%v: can inline %v with cost %d as: %v { %v }\n", ir.Line(fn), n.DiagName(), cost, fn.Type(), fn.Body)338	} else if base.Flag.LowerM != 0 {339		fmt.Printf("%v: can inline %v\n", ir.Line(fn), n.DiagName())340	}341	// JSON optimization log output.342	if logopt.Enabled() {343		logopt.LogOpt(fn.Pos(), "canInlineFunction", "inline", ir.FuncName(fn), fmt.Sprintf("cost: %d", cost))344	}345}346347// InlineImpossible returns a non-empty reason string if fn is impossible to348// inline regardless of cost or contents.349func InlineImpossible(fn *ir.Func) string {350	var reason string // reason, if any, that the function can not be inlined.351	if fn.Nname == nil {352		reason = "no name"353		return reason354	}355356	// If marked "go:noinline", don't inline.357	if fn.Pragma&ir.Noinline != 0 {358		reason = "marked go:noinline"359		return reason360	}361362	// If marked "go:norace" and -race compilation, don't inline.363	if base.Flag.Race && fn.Pragma&ir.Norace != 0 {364		reason = "marked go:norace with -race compilation"365		return reason366	}367368	// If marked "go:nocheckptr" and -d checkptr compilation, don't inline.369	if base.Debug.Checkptr != 0 && fn.Pragma&ir.NoCheckPtr != 0 {370		reason = "marked go:nocheckptr"371		return reason372	}373374	// If marked "go:cgo_unsafe_args", don't inline, since the function375	// makes assumptions about its argument frame layout.376	if fn.Pragma&ir.CgoUnsafeArgs != 0 {377		reason = "marked go:cgo_unsafe_args"378		return reason379	}380381	// If marked as "go:uintptrkeepalive", don't inline, since the keep382	// alive information is lost during inlining.383	//384	// TODO(prattmic): This is handled on calls during escape analysis,385	// which is after inlining. Move prior to inlining so the keep-alive is386	// maintained after inlining.387	if fn.Pragma&ir.UintptrKeepAlive != 0 {388		reason = "marked as having a keep-alive uintptr argument"389		return reason390	}391392	// If marked as "go:uintptrescapes", don't inline, since the escape393	// information is lost during inlining.394	if fn.Pragma&ir.UintptrEscapes != 0 {395		reason = "marked as having an escaping uintptr argument"396		return reason397	}398399	// The nowritebarrierrec checker currently works at function400	// granularity, so inlining yeswritebarrierrec functions can confuse it401	// (#22342). As a workaround, disallow inlining them for now.402	if fn.Pragma&ir.Yeswritebarrierrec != 0 {403		reason = "marked go:yeswritebarrierrec"404		return reason405	}406407	// If a local function has no fn.Body (is defined outside of Go), cannot inline it.408	// Imported functions don't have fn.Body but might have inline body in fn.Inl.409	if len(fn.Body) == 0 && !typecheck.HaveInlineBody(fn) {410		reason = "no function body"411		return reason412	}413414	return ""415}416417// canDelayResults reports whether inlined calls to fn can delay418// declaring the result parameter until the "return" statement.419func canDelayResults(fn *ir.Func) bool {420	// We can delay declaring+initializing result parameters if:421	// (1) there's exactly one "return" statement in the inlined function;422	// (2) it's not an empty return statement (#44355); and423	// (3) the result parameters aren't named.424425	nreturns := 0426	ir.VisitList(fn.Body, func(n ir.Node) {427		if n, ok := n.(*ir.ReturnStmt); ok {428			nreturns++429			if len(n.Results) == 0 {430				nreturns++ // empty return statement (case 2)431			}432		}433	})434435	if nreturns != 1 {436		return false // not exactly one return statement (case 1)437	}438439	// temporaries for return values.440	for _, param := range fn.Type().Results() {441		if sym := param.Sym; sym != nil && !sym.IsBlank() {442			return false // found a named result parameter (case 3)443		}444	}445446	return true447}448449// hairyVisitor visits a function body to determine its inlining450// hairiness and whether or not it can be inlined.451type hairyVisitor struct {452	// This is needed to access the current caller in the doNode function.453	curFunc       *ir.Func454	isBigFunc     bool455	debug         bool456	budget        int32457	maxBudget     int32458	reason        string459	extraCallCost int32460	usedLocals    ir.NameSet461	do            func(ir.Node) bool462	profile       *pgoir.Profile463	dbg           bool464}465466func isDebugFn(fn *ir.Func) bool {467	// if n := fn.Nname; n != nil {468	// 	if n.Sym().Name == "Int32x8.Transpose8" && n.Sym().Pkg.Path == "simd/archsimd" {469	// 		fmt.Printf("isDebugFn '%s' DOT '%s'\n", n.Sym().Pkg.Path, n.Sym().Name)470	// 		return true471	// 	}472	// }473	return false474}475476func (v *hairyVisitor) tooHairy(fn *ir.Func) bool {477	v.do = v.doNode // cache closure478	if ir.DoChildren(fn, v.do) {479		return true480	}481	if v.budget < 0 {482		v.reason = fmt.Sprintf("function too complex: cost %d exceeds budget %d", v.maxBudget-v.budget, v.maxBudget)483		return true484	}485	return false486}487488// doNode visits n and its children, updates the state in v, and returns true if489// n makes the current function too hairy for inlining.490func (v *hairyVisitor) doNode(n ir.Node) bool {491	if n == nil {492		return false493	}494	if v.debug {495		fmt.Printf("%v: doNode %v budget is %d\n", ir.Line(n), n.Op(), v.budget)496	}497opSwitch:498	switch n.Op() {499	// Call is okay if inlinable and we have the budget for the body.500	case ir.OCALLFUNC:501		n := n.(*ir.CallExpr)502		var cheap bool503		if n.Fun.Op() == ir.ONAME {504			name := n.Fun.(*ir.Name)505			if name.Class == ir.PFUNC {506				s := name.Sym()507				fn := s.Name508				switch s.Pkg.Path {509				case "internal/abi":510					switch fn {511					case "NoEscape":512						// Special case for internal/abi.NoEscape. It does just type513						// conversions to appease the escape analysis, and doesn't514						// generate code.515						cheap = true516					}517					if strings.HasPrefix(fn, "EscapeNonString[") {518						// internal/abi.EscapeNonString[T] is a compiler intrinsic519						// implemented in the escape analysis phase.520						cheap = true521					}522				case "internal/runtime/sys":523					switch fn {524					case "GetCallerPC", "GetCallerSP":525						// Functions that call GetCallerPC/SP can not be inlined526						// because users expect the PC/SP of the logical caller,527						// but GetCallerPC/SP returns the physical caller.528						v.reason = "call to " + fn529						return true530					}531				case "go.runtime":532					switch fn {533					case "throw":534						// runtime.throw is a "cheap call" like panic in normal code.535						v.budget -= inlineExtraThrowCost536						break opSwitch537					case "panicrangestate":538						cheap = true539					case "deferrangefunc":540						v.reason = "defer call in range func"541						return true542					}543				}544			}545			// Special case for coverage counter updates; although546			// these correspond to real operations, we treat them as547			// zero cost for the moment. This is due to the existence548			// of tests that are sensitive to inlining-- if the549			// insertion of coverage instrumentation happens to tip a550			// given function over the threshold and move it from551			// "inlinable" to "not-inlinable", this can cause changes552			// in allocation behavior, which can then result in test553			// failures (a good example is the TestAllocations in554			// crypto/ed25519).555			if isAtomicCoverageCounterUpdate(n) {556				return false557			}558		}559		if n.Fun.Op() == ir.OMETHEXPR {560			if meth := ir.MethodExprName(n.Fun); meth != nil {561				if fn := meth.Func; fn != nil {562					s := fn.Sym()563					if types.RuntimeSymName(s) == "heapBits.nextArena" {564						// Special case: explicitly allow mid-stack inlining of565						// runtime.heapBits.next even though it calls slow-path566						// runtime.heapBits.nextArena.567						cheap = true568					}569					// Special case: on architectures that can do unaligned loads,570					// explicitly mark encoding/binary methods as cheap,571					// because in practice they are, even though our inlining572					// budgeting system does not see that. See issue 42958.573					if base.Ctxt.Arch.CanMergeLoads && s.Pkg.Path == "encoding/binary" {574						switch s.Name {575						case "littleEndian.Uint64", "littleEndian.Uint32", "littleEndian.Uint16",576							"bigEndian.Uint64", "bigEndian.Uint32", "bigEndian.Uint16",577							"littleEndian.PutUint64", "littleEndian.PutUint32", "littleEndian.PutUint16",578							"bigEndian.PutUint64", "bigEndian.PutUint32", "bigEndian.PutUint16",579							"littleEndian.AppendUint64", "littleEndian.AppendUint32", "littleEndian.AppendUint16",580							"bigEndian.AppendUint64", "bigEndian.AppendUint32", "bigEndian.AppendUint16":581							cheap = true582						}583					}584				}585			}586		}587588		// A call to a parameter is optimistically a cheap call, if it's a constant function589		// perhaps it will inline, it also can simplify escape analysis.590		extraCost := v.extraCallCost591592		if n.Fun.Op() == ir.ONAME {593			name := n.Fun.(*ir.Name)594			if name.Class == ir.PFUNC {595				// Special case: on architectures that can do unaligned loads,596				// explicitly mark internal/byteorder methods as cheap,597				// because in practice they are, even though our inlining598				// budgeting system does not see that. See issue 42958.599				if base.Ctxt.Arch.CanMergeLoads && name.Sym().Pkg.Path == "internal/byteorder" {600					switch name.Sym().Name {601					case "LEUint64", "LEUint32", "LEUint16",602						"BEUint64", "BEUint32", "BEUint16",603						"LEPutUint64", "LEPutUint32", "LEPutUint16",604						"BEPutUint64", "BEPutUint32", "BEPutUint16",605						"LEAppendUint64", "LEAppendUint32", "LEAppendUint16",606						"BEAppendUint64", "BEAppendUint32", "BEAppendUint16":607						cheap = true608					}609				}610			}611			if name.Class == ir.PPARAM || name.Class == ir.PAUTOHEAP && name.IsClosureVar() {612				extraCost = min(extraCost, inlineParamCallCost)613			}614		}615616		if cheap {617			if v.debug {618				if ir.IsIntrinsicCall(n) {619					fmt.Printf("%v: cheap call is also intrinsic, %v\n", ir.Line(n), n)620				}621			}622			break // treat like any other node, that is, cost of 1623		}624625		if ir.IsIntrinsicCall(n) {626			if v.debug {627				fmt.Printf("%v: intrinsic call, %v\n", ir.Line(n), n)628			}629			break // Treat like any other node.630		}631632		if callee := inlCallee(v.curFunc, n.Fun, v.profile, false); callee != nil && typecheck.HaveInlineBody(callee) {633			// Check whether we'd actually inline this call. Set634			// log == false since we aren't actually doing inlining635			// yet.636			if ok, _, _ := canInlineCallExpr(v.curFunc, n, callee, v.isBigFunc, false, false); ok {637				// mkinlcall would inline this call [1], so use638				// the cost of the inline body as the cost of639				// the call, as that is what will actually640				// appear in the code.641				//642				// [1] This is almost a perfect match to the643				// mkinlcall logic, except that644				// canInlineCallExpr considers inlining cycles645				// by looking at what has already been inlined.646				// Since we haven't done any inlining yet we647				// will miss those.648				//649				// TODO: in the case of a single-call closure, the inlining budget here is potentially much, much larger.650				//651				v.budget -= callee.Inl.Cost652				break653			}654		}655656		if v.debug {657			fmt.Printf("%v: costly OCALLFUNC %v\n", ir.Line(n), n)658		}659660		// Call cost for non-leaf inlining.661		v.budget -= extraCost662663	case ir.OCALLMETH:664		base.FatalfAt(n.Pos(), "OCALLMETH missed by typecheck")665666	// Things that are too hairy, irrespective of the budget667	case ir.OCALL, ir.OCALLINTER:668		// Call cost for non-leaf inlining.669		if v.debug {670			fmt.Printf("%v: costly OCALL %v\n", ir.Line(n), n)671		}672		v.budget -= v.extraCallCost673674	case ir.OPANIC:675		n := n.(*ir.UnaryExpr)676		if n.X.Op() == ir.OCONVIFACE && n.X.(*ir.ConvExpr).Implicit() {677			// Hack to keep reflect.flag.mustBe inlinable for TestIntendedInlining.678			// Before CL 284412, these conversions were introduced later in the679			// compiler, so they didn't count against inlining budget.680			v.budget++681		}682		v.budget -= inlineExtraPanicCost683684	case ir.ORECOVER:685		// TODO: maybe we could allow inlining of recover() now?686		v.reason = "call to recover"687		return true688689	case ir.OCLOSURE:690		if base.Debug.InlFuncsWithClosures == 0 {691			v.reason = "not inlining functions with closures"692			return true693		}694695		// TODO(danscales): Maybe make budget proportional to number of closure696		// variables, e.g.:697		//v.budget -= int32(len(n.(*ir.ClosureExpr).Func.ClosureVars) * 3)698		// TODO(austin): However, if we're able to inline this closure into699		// v.curFunc, then we actually pay nothing for the closure captures. We700		// should try to account for that if we're going to account for captures.701		v.budget -= 15702703	case ir.OGO, ir.ODEFER, ir.OTAILCALL:704		v.reason = "unhandled op " + n.Op().String()705		return true706707	case ir.OAPPEND:708		v.budget -= inlineExtraAppendCost709710	case ir.OADDR:711		n := n.(*ir.AddrExpr)712		// Make "&s.f" cost 0 when f's offset is zero.713		if dot, ok := n.X.(*ir.SelectorExpr); ok && (dot.Op() == ir.ODOT || dot.Op() == ir.ODOTPTR) {714			if _, ok := dot.X.(*ir.Name); ok && dot.Selection.Offset == 0 {715				v.budget += 2 // undo ir.OADDR+ir.ODOT/ir.ODOTPTR716			}717		}718719	case ir.ODEREF:720		// *(*X)(unsafe.Pointer(&x)) is low-cost721		n := n.(*ir.StarExpr)722723		ptr := n.X724		for ptr.Op() == ir.OCONVNOP {725			ptr = ptr.(*ir.ConvExpr).X726		}727		if ptr.Op() == ir.OADDR {728			v.budget += 1 // undo half of default cost of ir.ODEREF+ir.OADDR729		}730731	case ir.OCONVNOP:732		// This doesn't produce code, but the children might.733		v.budget++ // undo default cost734735	case ir.OFALL, ir.OTYPE:736		// These nodes don't produce code; omit from inlining budget.737		return false738739	case ir.OIF:740		n := n.(*ir.IfStmt)741		if ir.IsConst(n.Cond, constant.Bool) {742			// This if and the condition cost nothing.743			if doList(n.Init(), v.do) {744				return true745			}746			if ir.BoolVal(n.Cond) {747				return doList(n.Body, v.do)748			} else {749				return doList(n.Else, v.do)750			}751		}752753	case ir.ONAME:754		n := n.(*ir.Name)755		if n.Class == ir.PAUTO {756			v.usedLocals.Add(n)757		}758759	case ir.OBLOCK:760		// The only OBLOCK we should see at this point is an empty one.761		// In any event, let the visitList(n.List()) below take care of the statements,762		// and don't charge for the OBLOCK itself. The ++ undoes the -- below.763		v.budget++764765	case ir.OMETHVALUE, ir.OSLICELIT:766		v.budget-- // Hack for toolstash -cmp.767768	case ir.OMETHEXPR:769		v.budget++ // Hack for toolstash -cmp.770771	case ir.OAS2:772		n := n.(*ir.AssignListStmt)773774		// Unified IR unconditionally rewrites:775		//776		//	a, b = f()777		//778		// into:779		//780		//	DCL tmp1781		//	DCL tmp2782		//	tmp1, tmp2 = f()783		//	a, b = tmp1, tmp2784		//785		// so that it can insert implicit conversions as necessary. To786		// minimize impact to the existing inlining heuristics (in787		// particular, to avoid breaking the existing inlinability regress788		// tests), we need to compensate for this here.789		//790		// See also identical logic in IsBigFunc.791		if len(n.Rhs) > 0 {792			if init := n.Rhs[0].Init(); len(init) == 1 {793				if _, ok := init[0].(*ir.AssignListStmt); ok {794					// 4 for each value, because each temporary variable now795					// appears 3 times (DCL, LHS, RHS), plus an extra DCL node.796					//797					// 1 for the extra "tmp1, tmp2 = f()" assignment statement.798					v.budget += 4*int32(len(n.Lhs)) + 1799				}800			}801		}802803	case ir.OAS:804		// Special case for coverage counter updates and coverage805		// function registrations. Although these correspond to real806		// operations, we treat them as zero cost for the moment. This807		// is primarily due to the existence of tests that are808		// sensitive to inlining-- if the insertion of coverage809		// instrumentation happens to tip a given function over the810		// threshold and move it from "inlinable" to "not-inlinable",811		// this can cause changes in allocation behavior, which can812		// then result in test failures (a good example is the813		// TestAllocations in crypto/ed25519).814		n := n.(*ir.AssignStmt)815		if n.X.Op() == ir.OINDEX && isIndexingCoverageCounter(n.X) {816			return false817		}818819	case ir.OSLICE, ir.OSLICEARR, ir.OSLICESTR, ir.OSLICE3, ir.OSLICE3ARR:820		n := n.(*ir.SliceExpr)821822		// Ignore superfluous slicing.823		if n.Low != nil && n.Low.Op() == ir.OLITERAL && ir.Int64Val(n.Low) == 0 {824			v.budget++825		}826		if n.High != nil && n.High.Op() == ir.OLEN && n.High.(*ir.UnaryExpr).X == n.X {827			v.budget += 2828		}829	}830831	v.budget--832833	// When debugging, don't stop early, to get full cost of inlining this function834	if v.budget < 0 && base.Flag.LowerM < 2 && !logopt.Enabled() && !v.debug {835		v.reason = "too expensive"836		return true837	}838839	return ir.DoChildren(n, v.do)840}841842// IsBigFunc reports whether fn is a "big" function.843//844// Note: The criteria for "big" is heuristic and subject to change.845func IsBigFunc(fn *ir.Func) bool {846	budget := inlineBigFunctionNodes847	return ir.Any(fn, func(n ir.Node) bool {848		// See logic in hairyVisitor.doNode, explaining unified IR's849		// handling of "a, b = f()" assignments.850		if n, ok := n.(*ir.AssignListStmt); ok && n.Op() == ir.OAS2 && len(n.Rhs) > 0 {851			if init := n.Rhs[0].Init(); len(init) == 1 {852				if _, ok := init[0].(*ir.AssignListStmt); ok {853					budget += 4*len(n.Lhs) + 1854				}855			}856		}857858		budget--859		return budget <= 0860	})861}862863// inlineCallCheck returns whether a call will never be inlineable864// for basic reasons, and whether the call is an intrinisic call.865// The intrinsic result singles out intrinsic calls for debug logging.866func inlineCallCheck(callerfn *ir.Func, call *ir.CallExpr) (bool, bool) {867	if base.Flag.LowerL == 0 {868		return false, false869	}870	if call.Op() != ir.OCALLFUNC {871		return false, false872	}873	if call.GoDefer || call.NoInline {874		return false, false875	}876877	// Prevent inlining some reflect.Value methods when using checkptr,878	// even when package reflect was compiled without it (#35073).879	if base.Debug.Checkptr != 0 && call.Fun.Op() == ir.OMETHEXPR {880		if method := ir.MethodExprName(call.Fun); method != nil {881			switch types.ReflectSymName(method.Sym()) {882			case "Value.UnsafeAddr", "Value.Pointer":883				return false, false884			}885		}886	}887888	// internal/abi.EscapeNonString[T] is a compiler intrinsic implemented889	// in the escape analysis phase.890	if fn := ir.StaticCalleeName(call.Fun); fn != nil && fn.Sym().Pkg.Path == "internal/abi" &&891		strings.HasPrefix(fn.Sym().Name, "EscapeNonString[") {892		return false, true893	}894895	if ir.IsIntrinsicCall(call) {896		return false, true897	}898	return true, false899}900901// InlineCallTarget returns the resolved-for-inlining target of a call.902// It does not necessarily guarantee that the target can be inlined, though903// obvious exclusions are applied.904func InlineCallTarget(callerfn *ir.Func, call *ir.CallExpr, profile *pgoir.Profile) *ir.Func {905	if mightInline, _ := inlineCallCheck(callerfn, call); !mightInline {906		return nil907	}908	return inlCallee(callerfn, call.Fun, profile, true)909}910911// TryInlineCall returns an inlined call expression for call, or nil912// if inlining is not possible.913func TryInlineCall(callerfn *ir.Func, call *ir.CallExpr, bigCaller bool, profile *pgoir.Profile, closureCalledOnce bool) *ir.InlinedCallExpr {914	mightInline, isIntrinsic := inlineCallCheck(callerfn, call)915916	// Preserve old logging behavior917	if (mightInline || isIntrinsic) && base.Flag.LowerM > 3 {918		fmt.Printf("%v:call to func %+v\n", ir.Line(call), call.Fun)919	}920	if !mightInline {921		return nil922	}923924	if fn := inlCallee(callerfn, call.Fun, profile, false); fn != nil && typecheck.HaveInlineBody(fn) {925		return mkinlcall(callerfn, call, fn, bigCaller, closureCalledOnce, profile)926	}927	return nil928}929930// inlCallee takes a function-typed expression and returns the underlying function ONAME931// that it refers to if statically known. Otherwise, it returns nil.932// resolveOnly skips cost-based inlineability checks for closures; the result may not actually be inlineable.933func inlCallee(caller *ir.Func, fn ir.Node, profile *pgoir.Profile, resolveOnly bool) (res *ir.Func) {934	fn = ir.StaticValue(fn)935	switch fn.Op() {936	case ir.OMETHEXPR:937		fn := fn.(*ir.SelectorExpr)938		n := ir.MethodExprName(fn)939		// Check that receiver type matches fn.X.940		// TODO(mdempsky): Handle implicit dereference941		// of pointer receiver argument?942		if n == nil || !types.Identical(n.Type().Recv().Type, fn.X.Type()) {943			return nil944		}945		return n.Func946	case ir.ONAME:947		fn := fn.(*ir.Name)948		if fn.Class == ir.PFUNC {949			return fn.Func950		}951	case ir.OCLOSURE:952		fn := fn.(*ir.ClosureExpr)953		c := fn.Func954		if len(c.ClosureVars) != 0 && c.ClosureVars[0].Outer.Curfn != caller {955			return nil // inliner doesn't support inlining across closure frames956		}957		if !resolveOnly {958			CanInline(c, profile)959		}960		return c961	}962	return nil963}964965var inlgen int966967// SSADumpInline gives the SSA back end a chance to dump the function968// when producing output for debugging the compiler itself.969var SSADumpInline = func(*ir.Func) {}970971// InlineCall allows the inliner implementation to be overridden.972// If it returns nil, the function will not be inlined.973var InlineCall = func(callerfn *ir.Func, call *ir.CallExpr, fn *ir.Func, inlIndex int, profile *pgoir.Profile) *ir.InlinedCallExpr {974	base.Fatalf("inline.InlineCall not overridden")975	panic("unreachable")976}977978// inlineCostOK returns true if call n from caller to callee is cheap enough to979// inline. bigCaller indicates that caller is a big function.980//981// In addition to the "cost OK" boolean, it also returns982//   - the "max cost" limit used to make the decision (which may differ depending on func size)983//   - the score assigned to this specific callsite984//   - whether the inlined function is "hot" according to PGO.985func inlineCostOK(n *ir.CallExpr, caller, callee *ir.Func, bigCaller, closureCalledOnce bool) (bool, int32, int32, bool) {986	maxCost := int32(inlineMaxBudget)987988	if strings.HasPrefix(ir.FuncName(caller), "runtime_mapaccess1") && caller.Sym().Pkg.Path == "internal/runtime/maps" &&989		strings.HasPrefix(ir.FuncName(callee), "runtime_mapaccess2") && callee.Sym().Pkg.Path == "internal/runtime/maps" {990		// Raise cost to allow inlining of mapaccess2* functions to mapaccess1* wrappers991		maxCost = inlineHotMaxBudget992	}993994	if bigCaller {995		// We use this to restrict inlining into very big functions.996		// See issue 26546 and 17566.997		maxCost = inlineBigFunctionMaxCost998	}9991000	simdMaxCost := simdCreditMultiplier(callee) * maxCost10011002	if callee.ClosureParent != nil {1003		maxCost *= 2           // favor inlining closures1004		if closureCalledOnce { // really favor inlining the one call to this closure1005			maxCost = max(maxCost, inlineClosureCalledOnceCost)1006		}1007	}10081009	maxCost = max(maxCost, simdMaxCost)10101011	metric := callee.Inl.Cost1012	if inlheur.Enabled() {1013		score, ok := inlheur.GetCallSiteScore(caller, n)1014		if ok {1015			metric = int32(score)1016		}1017	}10181019	lineOffset := pgoir.NodeLineOffset(n, caller)1020	csi := pgoir.CallSiteInfo{LineOffset: lineOffset, Caller: caller}1021	_, hot := candHotEdgeMap[csi]10221023	if metric <= maxCost {1024		// Simple case. Function is already cheap enough.1025		return true, 0, metric, hot1026	}10271028	// We'll also allow inlining of hot functions below inlineHotMaxBudget,1029	// but only in small functions.10301031	if !hot {1032		// Cold1033		return false, maxCost, metric, false1034	}10351036	// Hot10371038	if bigCaller {1039		if base.Debug.PGODebug > 0 {1040			fmt.Printf("hot-big check disallows inlining for call %s (cost %d) at %v in big function %s\n", ir.PkgFuncName(callee), callee.Inl.Cost, ir.Line(n), ir.PkgFuncName(caller))1041		}1042		return false, maxCost, metric, false1043	}10441045	if metric > inlineHotMaxBudget {1046		return false, inlineHotMaxBudget, metric, false1047	}10481049	if !base.PGOHash.MatchPosWithInfo(n.Pos(), "inline", nil) {1050		// De-selected by PGO Hash.1051		return false, maxCost, metric, false1052	}10531054	if base.Debug.PGODebug > 0 {1055		fmt.Printf("hot-budget check allows inlining for call %s (cost %d) at %v in function %s\n", ir.PkgFuncName(callee), callee.Inl.Cost, ir.Line(n), ir.PkgFuncName(caller))1056	}10571058	return true, 0, metric, hot1059}10601061// parsePos returns all the inlining positions and the innermost position.1062func parsePos(pos src.XPos, posTmp []src.Pos) ([]src.Pos, src.Pos) {1063	ctxt := base.Ctxt1064	ctxt.AllPos(pos, func(p src.Pos) {1065		posTmp = append(posTmp, p)1066	})1067	l := len(posTmp) - 11068	return posTmp[:l], posTmp[l]1069}10701071// canInlineCallExpr returns true if the call n from caller to callee1072// can be inlined, plus the score computed for the call expr in question,1073// and whether the callee is hot according to PGO.1074// bigCaller indicates that caller is a big function. log1075// indicates that the 'cannot inline' reason should be logged.1076//1077// Preconditions: CanInline(callee) has already been called.1078func canInlineCallExpr(callerfn *ir.Func, n *ir.CallExpr, callee *ir.Func, bigCaller, closureCalledOnce bool, log bool) (bool, int32, bool) {1079	if callee.Inl == nil {1080		// callee is never inlinable.1081		if log && logopt.Enabled() {1082			logopt.LogOpt(n.Pos(), "cannotInlineCall", "inline", ir.FuncName(callerfn),1083				fmt.Sprintf("%s cannot be inlined", ir.PkgFuncName(callee)))1084		}1085		return false, 0, false1086	}10871088	ok, maxCost, callSiteScore, hot := inlineCostOK(n, callerfn, callee, bigCaller, closureCalledOnce)1089	if !ok {1090		// callee cost too high for this call site.1091		if log && logopt.Enabled() {1092			logopt.LogOpt(n.Pos(), "cannotInlineCall", "inline", ir.FuncName(callerfn),1093				fmt.Sprintf("cost %d of %s exceeds max caller cost %d", callee.Inl.Cost, ir.PkgFuncName(callee), maxCost))1094		}1095		return false, 0, false1096	}10971098	callees, calleeInner := parsePos(n.Pos(), make([]src.Pos, 0, 10))10991100	for _, p := range callees {1101		if p.Line() == calleeInner.Line() && p.Col() == calleeInner.Col() && p.AbsFilename() == calleeInner.AbsFilename() {1102			if log && logopt.Enabled() {1103				logopt.LogOpt(n.Pos(), "cannotInlineCall", "inline", fmt.Sprintf("recursive call to %s", ir.FuncName(callerfn)))1104			}1105			return false, 0, false1106		}1107	}11081109	if base.Flag.Cfg.Instrumenting && types.IsNoInstrumentPkg(callee.Sym().Pkg) {1110		// Runtime package must not be instrumented.1111		// Instrument skips runtime package. However, some runtime code can be1112		// inlined into other packages and instrumented there. To avoid this,1113		// we disable inlining of runtime functions when instrumenting.1114		// The example that we observed is inlining of LockOSThread,1115		// which lead to false race reports on m contents.1116		if log && logopt.Enabled() {1117			logopt.LogOpt(n.Pos(), "cannotInlineCall", "inline", ir.FuncName(callerfn),1118				fmt.Sprintf("call to runtime function %s in instrumented build", ir.PkgFuncName(callee)))1119		}1120		return false, 0, false1121	}11221123	if base.Flag.Race && types.IsNoRacePkg(callee.Sym().Pkg) {1124		if log && logopt.Enabled() {1125			logopt.LogOpt(n.Pos(), "cannotInlineCall", "inline", ir.FuncName(callerfn),1126				fmt.Sprintf(`call to into "no-race" package function %s in race build`, ir.PkgFuncName(callee)))1127		}1128		return false, 0, false1129	}11301131	if base.Debug.Checkptr != 0 && types.IsRuntimePkg(callee.Sym().Pkg) {1132		// We don't instrument runtime packages for checkptr (see base/flag.go).1133		if log && logopt.Enabled() {1134			logopt.LogOpt(n.Pos(), "cannotInlineCall", "inline", ir.FuncName(callerfn),1135				fmt.Sprintf(`call to into runtime package function %s in -d=checkptr build`, ir.PkgFuncName(callee)))1136		}1137		return false, 0, false1138	}11391140	// Check if we've already inlined this function at this particular1141	// call site, in order to stop inlining when we reach the beginning1142	// of a recursion cycle again. We don't inline immediately recursive1143	// functions, but allow inlining if there is a recursion cycle of1144	// many functions. Most likely, the inlining will stop before we1145	// even hit the beginning of the cycle again, but this catches the1146	// unusual case.1147	parent := base.Ctxt.PosTable.Pos(n.Pos()).Base().InliningIndex()1148	sym := callee.Linksym()1149	for inlIndex := parent; inlIndex >= 0; inlIndex = base.Ctxt.InlTree.Parent(inlIndex) {1150		if base.Ctxt.InlTree.InlinedFunction(inlIndex) == sym {1151			if log {1152				if base.Flag.LowerM > 1 {1153					fmt.Printf("%v: cannot inline %v into %v: repeated recursive cycle\n", ir.Line(n), callee, ir.FuncName(callerfn))1154				}1155				if logopt.Enabled() {1156					logopt.LogOpt(n.Pos(), "cannotInlineCall", "inline", ir.FuncName(callerfn),1157						fmt.Sprintf("repeated recursive cycle to %s", ir.PkgFuncName(callee)))1158				}1159			}1160			return false, 0, false1161		}1162	}11631164	return true, callSiteScore, hot1165}11661167// mkinlcall returns an OINLCALL node that can replace OCALLFUNC n, or1168// nil if it cannot be inlined. callerfn is the function that contains1169// n, and fn is the function being called.1170//1171// The result of mkinlcall MUST be assigned back to n, e.g.1172//1173//	n.Left = mkinlcall(n.Left, fn, isddd)1174func mkinlcall(callerfn *ir.Func, n *ir.CallExpr, fn *ir.Func, bigCaller, closureCalledOnce bool, profile *pgoir.Profile) *ir.InlinedCallExpr {1175	ok, score, hot := canInlineCallExpr(callerfn, n, fn, bigCaller, closureCalledOnce, true)1176	if !ok {1177		return nil1178	}1179	if hot {1180		hasHotCall[callerfn] = struct{}{}1181	}1182	typecheck.AssertFixedCall(n)11831184	parent := base.Ctxt.PosTable.Pos(n.Pos()).Base().InliningIndex()1185	sym := fn.Linksym()1186	inlIndex := base.Ctxt.InlTree.Add(parent, n.Pos(), sym, ir.FuncName(fn))11871188	closureInitLSym := func(n *ir.CallExpr, fn *ir.Func) {1189		// The linker needs FuncInfo metadata for all inlined1190		// functions. This is typically handled by gc.enqueueFunc1191		// calling ir.InitLSym for all function declarations in1192		// typecheck.Target.Decls (ir.UseClosure adds all closures to1193		// Decls).1194		//1195		// However, closures in Decls are ignored, and are1196		// instead enqueued when walk of the calling function1197		// discovers them.1198		//1199		// This presents a problem for direct calls to closures.1200		// Inlining will replace the entire closure definition with its1201		// body, which hides the closure from walk and thus suppresses1202		// symbol creation.1203		//1204		// Explicitly create a symbol early in this edge case to ensure1205		// we keep this metadata.1206		//1207		// TODO: Refactor to keep a reference so this can all be done1208		// by enqueueFunc.12091210		if n.Op() != ir.OCALLFUNC {1211			// Not a standard call.1212			return1213		}12141215		var nf = n.Fun1216		// Skips ir.OCONVNOPs, see issue #73716.1217		for nf.Op() == ir.OCONVNOP {1218			nf = nf.(*ir.ConvExpr).X1219		}1220		if nf.Op() != ir.OCLOSURE {1221			// Not a direct closure call or one with type conversion.1222			return1223		}12241225		clo := nf.(*ir.ClosureExpr)1226		if !clo.Func.IsClosure() {1227			// enqueueFunc will handle non closures anyways.1228			return1229		}12301231		ir.InitLSym(fn, true)1232	}12331234	closureInitLSym(n, fn)12351236	if base.Flag.GenDwarfInl > 0 {1237		if !sym.WasInlined() {1238			base.Ctxt.DwFixups.SetPrecursorFunc(sym, fn)1239			sym.Set(obj.AttrWasInlined, true)1240		}1241	}12421243	if base.Flag.LowerM != 0 {1244		if buildcfg.Experiment.NewInliner {1245			fmt.Printf("%v: inlining call to %v with score %d\n",1246				ir.Line(n), fn.Nname.DiagName(), score)1247		} else {1248			fmt.Printf("%v: inlining call to %v\n", ir.Line(n), fn.Nname.DiagName())1249		}1250	}1251	if base.Flag.LowerM > 2 {1252		fmt.Printf("%v: Before inlining: %+v\n", ir.Line(n), n)1253	}12541255	res := InlineCall(callerfn, n, fn, inlIndex, profile)12561257	if res == nil {1258		base.FatalfAt(n.Pos(), "inlining call to %v failed", fn.Nname.DiagName())1259	}12601261	if base.Flag.LowerM > 2 {1262		fmt.Printf("%v: After inlining %+v\n\n", ir.Line(res), res)1263	}12641265	if inlheur.Enabled() {1266		inlheur.UpdateCallsiteTable(callerfn, n, res)1267	}12681269	return res1270}12711272// CalleeEffects appends any side effects from evaluating callee to init.1273func CalleeEffects(init *ir.Nodes, callee ir.Node) {1274	for {1275		init.Append(ir.TakeInit(callee)...)12761277		switch callee.Op() {1278		case ir.ONAME, ir.OCLOSURE, ir.OMETHEXPR:1279			return // done12801281		case ir.OCONVNOP:1282			conv := callee.(*ir.ConvExpr)1283			callee = conv.X12841285		case ir.OINLCALL:1286			ic := callee.(*ir.InlinedCallExpr)1287			init.Append(ic.Body.Take()...)1288			callee = ic.SingleResult()12891290		default:1291			base.FatalfAt(callee.Pos(), "unexpected callee expression: %v", callee)1292		}1293	}1294}12951296func pruneUnusedAutos(ll []*ir.Name, vis *hairyVisitor) []*ir.Name {1297	s := make([]*ir.Name, 0, len(ll))1298	for _, n := range ll {1299		if n.Class == ir.PAUTO {1300			if !vis.usedLocals.Has(n) {1301				// TODO(mdempsky): Simplify code after confident that this1302				// never happens anymore.1303				base.FatalfAt(n.Pos(), "unused auto: %v", n)1304				continue1305			}1306		}1307		s = append(s, n)1308	}1309	return s1310}13111312func doList(list []ir.Node, do func(ir.Node) bool) bool {1313	for _, x := range list {1314		if x != nil {1315			if do(x) {1316				return true1317			}1318		}1319	}1320	return false1321}13221323// isIndexingCoverageCounter returns true if the specified node 'n' is indexing1324// into a coverage counter array.1325func isIndexingCoverageCounter(n ir.Node) bool {1326	if n.Op() != ir.OINDEX {1327		return false1328	}1329	ixn := n.(*ir.IndexExpr)1330	if ixn.X.Op() != ir.ONAME || !ixn.X.Type().IsArray() {1331		return false1332	}1333	nn := ixn.X.(*ir.Name)1334	// CoverageAuxVar implies either a coverage counter or a package1335	// ID; since the cover tool never emits code to index into ID vars1336	// this is effectively testing whether nn is a coverage counter.1337	return nn.CoverageAuxVar()1338}13391340// isAtomicCoverageCounterUpdate examines the specified node to1341// determine whether it represents a call to sync/atomic.AddUint32 to1342// increment a coverage counter.1343func isAtomicCoverageCounterUpdate(cn *ir.CallExpr) bool {1344	if cn.Fun.Op() != ir.ONAME {1345		return false1346	}1347	name := cn.Fun.(*ir.Name)1348	if name.Class != ir.PFUNC {1349		return false1350	}1351	fn := name.Sym().Name1352	if name.Sym().Pkg.Path != "sync/atomic" ||1353		(fn != "AddUint32" && fn != "StoreUint32") {1354		return false1355	}1356	if len(cn.Args) != 2 || cn.Args[0].Op() != ir.OADDR {1357		return false1358	}1359	adn := cn.Args[0].(*ir.AddrExpr)1360	v := isIndexingCoverageCounter(adn.X)1361	return v1362}13631364func PostProcessCallSites(profile *pgoir.Profile) {1365	if base.Debug.DumpInlCallSiteScores != 0 {1366		budgetCallback := func(fn *ir.Func, prof *pgoir.Profile) (int32, bool) {1367			v := inlineBudget(fn, prof, false, false)1368			return v, v == inlineHotMaxBudget1369		}1370		inlheur.DumpInlCallSiteScores(profile, budgetCallback)1371	}1372}13731374func analyzeFuncProps(fn *ir.Func, p *pgoir.Profile) {1375	canInline := func(fn *ir.Func) { CanInline(fn, p) }1376	budgetForFunc := func(fn *ir.Func) int32 {1377		return inlineBudget(fn, p, true, false)1378	}1379	inlheur.AnalyzeFunc(fn, canInline, budgetForFunc, inlineMaxBudget)1380}