1// Copyright 2021 The Go Authors. All rights reserved.2// Use of this source code is governed by a BSD-style3// license that can be found in the LICENSE file.45package noder67import (8	"cmp"9	"fmt"10	"internal/pkgbits"11	"internal/types/errors"12	"io"13	"runtime"14	"slices"15	"strings"1617	"cmd/compile/internal/base"18	"cmd/compile/internal/inline"19	"cmd/compile/internal/ir"20	"cmd/compile/internal/pgoir"21	"cmd/compile/internal/typecheck"22	"cmd/compile/internal/types"23	"cmd/compile/internal/types2"24	"cmd/internal/src"25)2627// uirVersion is the unified IR version to use for encoding/decoding.28// Use V4 for generic methods.29const uirVersion = pkgbits.V43031// localPkgReader holds the package reader used for reading the local32// package. It exists so the unified IR linker can refer back to it33// later.34var localPkgReader *pkgReader3536// LookupFunc returns the ir.Func for an arbitrary full symbol name if37// that function exists in the set of available export data.38//39// This allows lookup of arbitrary functions and methods that aren't otherwise40// referenced by the local package and thus haven't been read yet.41//42// TODO(prattmic): Does not handle instantiation of generic types. Currently43// profiles don't contain the original type arguments, so we won't be able to44// create the runtime dictionaries.45//46// TODO(prattmic): Hit rate of this function is usually fairly low, and errors47// are only used when debug logging is enabled. Consider constructing cheaper48// errors by default.49func LookupFunc(fullName string) (*ir.Func, error) {50	pkgPath, symName, err := ir.ParseLinkFuncName(fullName)51	if err != nil {52		return nil, fmt.Errorf("error parsing symbol name %q: %v", fullName, err)53	}5455	pkg, ok := types.PkgMap()[pkgPath]56	if !ok {57		return nil, fmt.Errorf("pkg %s doesn't exist in %v", pkgPath, types.PkgMap())58	}5960	// Symbol naming is ambiguous. We can't necessarily distinguish between61	// a method and a closure. e.g., is foo.Bar.func1 a closure defined in62	// function Bar, or a method on type Bar? Thus we must simply attempt63	// to lookup both.6465	fn, err := lookupFunction(pkg, symName)66	if err == nil {67		return fn, nil68	}6970	fn, mErr := lookupMethod(pkg, symName)71	if mErr == nil {72		return fn, nil73	}7475	return nil, fmt.Errorf("%s is not a function (%v) or method (%v)", fullName, err, mErr)76}7778// PostLookupCleanup performs cleanup operations needed79// after a series of calls to LookupFunc, specifically invoking80// readBodies to post-process any funcs on the "todoBodies" list81// that were added as a result of the lookup operations.82func PostLookupCleanup() {83	readBodies(typecheck.Target, false, nil)84}8586func lookupFunction(pkg *types.Pkg, symName string) (*ir.Func, error) {87	sym := pkg.Lookup(symName)8889	// TODO(prattmic): Enclosed functions (e.g., foo.Bar.func1) are not90	// present in objReader, only as OCLOSURE nodes in the enclosing91	// function.92	pri, ok := objReader[sym]93	if !ok {94		return nil, fmt.Errorf("func sym %v missing objReader", sym)95	}9697	node, err := pri.pr.objIdxMayFail(pri.idx, nil, nil, false)98	if err != nil {99		return nil, fmt.Errorf("func sym %v lookup error: %w", sym, err)100	}101	name := node.(*ir.Name)102	if name.Op() != ir.ONAME || name.Class != ir.PFUNC {103		return nil, fmt.Errorf("func sym %v refers to non-function name: %v", sym, name)104	}105	return name.Func, nil106}107108func lookupMethod(pkg *types.Pkg, symName string) (*ir.Func, error) {109	// N.B. readPackage creates a Sym for every object in the package to110	// initialize objReader and importBodyReader, even if the object isn't111	// read.112	//113	// However, objReader is only initialized for top-level objects, so we114	// must first lookup the type and use that to find the method rather115	// than looking for the method directly.116	typ, meth, err := ir.LookupMethodSelector(pkg, symName)117	if err != nil {118		return nil, fmt.Errorf("error looking up method symbol %q: %v", symName, err)119	}120121	pri, ok := objReader[typ]122	if !ok {123		return nil, fmt.Errorf("type sym %v missing objReader", typ)124	}125126	node, err := pri.pr.objIdxMayFail(pri.idx, nil, nil, false)127	if err != nil {128		return nil, fmt.Errorf("func sym %v lookup error: %w", typ, err)129	}130	name := node.(*ir.Name)131	if name.Op() != ir.OTYPE {132		return nil, fmt.Errorf("type sym %v refers to non-type name: %v", typ, name)133	}134	if name.Alias() {135		return nil, fmt.Errorf("type sym %v refers to alias", typ)136	}137	if name.Type().IsInterface() {138		return nil, fmt.Errorf("type sym %v refers to interface type", typ)139	}140141	for _, m := range name.Type().Methods() {142		if m.Sym == meth {143			fn := m.Nname.(*ir.Name).Func144			return fn, nil145		}146	}147148	return nil, fmt.Errorf("method %s missing from method set of %v", symName, typ)149}150151// unified constructs the local package's Internal Representation (IR)152// from its syntax tree (AST).153//154// The pipeline contains 2 steps:155//156//  1. Generate the export data "stub".157//158//  2. Generate the IR from the export data above.159//160// The package data "stub" at step (1) contains everything from the local package,161// but nothing that has been imported. When we're actually writing out export data162// to the output files (see writeNewExport), we run the "linker", which:163//164//   - Updates compiler extensions data (e.g. inlining cost, escape analysis results).165//166//   - Handles re-exporting any transitive dependencies.167//168//   - Prunes out any unnecessary details (e.g. non-inlineable functions, because any169//     downstream importers only care about inlinable functions).170//171// The source files are typechecked twice: once before writing the export data172// using types2, and again after reading the export data using gc/typecheck.173// The duplication of work will go away once we only use the types2 type checker,174// removing the gc/typecheck step. For now, it is kept because:175//176//   - It reduces the engineering costs in maintaining a fork of typecheck177//     (e.g. no need to backport fixes like CL 327651).178//179//   - It makes it easier to pass toolstash -cmp.180//181//   - Historically, we would always re-run the typechecker after importing a package,182//     even though we know the imported data is valid. It's not ideal, but it's183//     not causing any problems either.184//185//   - gc/typecheck is still in charge of some transformations, such as rewriting186//     multi-valued function calls or transforming ir.OINDEX to ir.OINDEXMAP.187//188// Using the syntax tree with types2, which has a complete representation of generics,189// the unified IR has the full typed AST needed for introspection during step (1).190// In other words, we have all the necessary information to build the generic IR form191// (see writer.captureVars for an example).192func unified(m posMap, noders []*noder) {193	inline.InlineCall = unifiedInlineCall194	typecheck.HaveInlineBody = unifiedHaveInlineBody195	pgoir.LookupFunc = LookupFunc196	pgoir.PostLookupCleanup = PostLookupCleanup197198	data := writePkgStub(m, noders)199200	target := typecheck.Target201202	localPkgReader = newPkgReader(pkgbits.NewPkgDecoder(types.LocalPkg.Path, data))203	readPackage(localPkgReader, types.LocalPkg, true)204205	r := localPkgReader.newReader(pkgbits.SectionMeta, pkgbits.PrivateRootIdx, pkgbits.SyncPrivate)206	r.pkgInit(types.LocalPkg, target)207208	readBodies(target, false, nil)209210	// Check that nothing snuck past typechecking.211	for _, fn := range target.Funcs {212		if fn.Typecheck() == 0 {213			base.FatalfAt(fn.Pos(), "missed typecheck: %v", fn)214		}215216		// For functions, check that at least their first statement (if217		// any) was typechecked too.218		if len(fn.Body) != 0 {219			if stmt := fn.Body[0]; stmt.Typecheck() == 0 {220				base.FatalfAt(stmt.Pos(), "missed typecheck: %v", stmt)221			}222		}223	}224225	// For functions originally came from package runtime,226	// mark as norace to prevent instrumenting, see issue #60439.227	for _, fn := range target.Funcs {228		if !base.Flag.CompilingRuntime && types.RuntimeSymName(fn.Sym()) != "" {229			fn.Pragma |= ir.Norace230		}231	}232233	base.ExitIfErrors() // just in case234}235236// readBodies iteratively expands all pending dictionaries and237// function bodies.238//239// If duringInlining is true, then the inline.InlineDecls is called as240// necessary on instantiations of imported generic functions, so their241// inlining costs can be computed.242func readBodies(target *ir.Package, duringInlining bool, profile *pgoir.Profile) {243	var inlDecls []*ir.Func244245	// Don't use range--bodyIdx can add closures to todoBodies.246	for {247		// The order we expand dictionaries and bodies doesn't matter, so248		// pop from the end to reduce todoBodies reallocations if it grows249		// further.250		//251		// However, we do at least need to flush any pending dictionaries252		// before reading bodies, because bodies might reference the253		// dictionaries.254255		if len(todoDicts) > 0 {256			fn := todoDicts[len(todoDicts)-1]257			todoDicts = todoDicts[:len(todoDicts)-1]258			fn()259			continue260		}261262		if len(todoBodies) > 0 {263			fn := todoBodies[len(todoBodies)-1]264			todoBodies = todoBodies[:len(todoBodies)-1]265266			pri, ok := bodyReader[fn]267			assert(ok)268			pri.funcBody(fn)269270			// Instantiated generic function: add to Decls for typechecking271			// and compilation.272			if fn.OClosure == nil && len(pri.dict.targs) != 0 {273				// cmd/link does not support a type symbol referencing a method symbol274				// across DSO boundary, so force re-compiling methods on a generic type275				// even it was seen from imported package in linkshared mode, see #58966.276				canSkipNonGenericMethod := !(base.Ctxt.Flag_linkshared && ir.IsMethod(fn))277				if duringInlining && canSkipNonGenericMethod {278					inlDecls = append(inlDecls, fn)279				} else {280					target.Funcs = append(target.Funcs, fn)281				}282			}283284			continue285		}286287		break288	}289290	todoDicts = nil291	todoBodies = nil292293	if len(inlDecls) != 0 {294		// If we instantiated any generic functions during inlining, we need295		// to call CanInline on them so they'll be transitively inlined296		// correctly (#56280).297		//298		// We know these functions were already compiled in an imported299		// package though, so we don't need to actually apply InlineCalls or300		// save the function bodies any further than this.301		//302		// We can also lower the -m flag to 0, to suppress duplicate "can303		// inline" diagnostics reported against the imported package. Again,304		// we already reported those diagnostics in the original package, so305		// it's pointless repeating them here.306307		oldLowerM := base.Flag.LowerM308		base.Flag.LowerM = 0309		inline.CanInlineFuncs(inlDecls, profile)310		base.Flag.LowerM = oldLowerM311312		for _, fn := range inlDecls {313			fn.Body = nil // free memory314		}315	}316}317318// writePkgStub type checks the given parsed source files,319// writes an export data package stub representing them,320// and returns the result.321func writePkgStub(m posMap, noders []*noder) string {322	pkg, info, otherInfo := checkFiles(m, noders)323324	pw := newPkgWriter(m, pkg, info, otherInfo)325326	pw.collectDecls(noders)327328	publicRootWriter := pw.newWriter(pkgbits.SectionMeta, pkgbits.SyncPublic)329	privateRootWriter := pw.newWriter(pkgbits.SectionMeta, pkgbits.SyncPrivate)330331	assert(publicRootWriter.Idx == pkgbits.PublicRootIdx)332	assert(privateRootWriter.Idx == pkgbits.PrivateRootIdx)333334	{335		w := publicRootWriter336		w.pkg(pkg)337338		if w.Version().Has(pkgbits.HasInit) {339			w.Bool(false)340		}341342		scope := pkg.Scope()343		names := scope.Names()344		w.Len(len(names))345		for _, name := range names {346			w.obj(scope.Lookup(name), nil)347		}348349		w.Sync(pkgbits.SyncEOF)350		w.Flush()351	}352353	{354		w := privateRootWriter355		w.pkgInit(noders)356		w.Flush()357	}358359	var sb strings.Builder360	pw.DumpTo(&sb)361362	// At this point, we're done with types2. Make sure the package is363	// garbage collected.364	freePackage(pkg)365366	return sb.String()367}368369// freePackage ensures the given package is garbage collected.370func freePackage(pkg *types2.Package) {371	// The GC test below relies on a precise GC that runs finalizers as372	// soon as objects are unreachable. Our implementation provides373	// this, but other/older implementations may not (e.g., Go 1.4 does374	// not because of #22350). To avoid imposing unnecessary375	// restrictions on the GOROOT_BOOTSTRAP toolchain, we skip the test376	// during bootstrapping.377	if base.CompilerBootstrap || base.Debug.GCCheck == 0 {378		*pkg = types2.Package{}379		return380	}381382	// Set a finalizer on pkg so we can detect if/when it's collected.383	done := make(chan struct{})384	runtime.SetFinalizer(pkg, func(*types2.Package) { close(done) })385386	// Important: objects involved in cycles are not finalized, so zero387	// out pkg to break its cycles and allow the finalizer to run.388	*pkg = types2.Package{}389390	// It typically takes just 1 or 2 cycles to release pkg, but it391	// doesn't hurt to try a few more times.392	for i := 0; i < 10; i++ {393		select {394		case <-done:395			return396		default:397			runtime.GC()398		}399	}400401	base.Fatalf("package never finalized")402}403404// readPackage reads package export data from pr to populate405// importpkg.406//407// localStub indicates whether pr is reading the stub export data for408// the local package, as opposed to relocated export data for an409// import.410func readPackage(pr *pkgReader, importpkg *types.Pkg, localStub bool) {411	{412		r := pr.newReader(pkgbits.SectionMeta, pkgbits.PublicRootIdx, pkgbits.SyncPublic)413414		pkg := r.pkg()415		// This error can happen if "go tool compile" is called with wrong "-p" flag, see issue #54542.416		if pkg != importpkg {417			base.ErrorfAt(base.AutogeneratedPos, errors.BadImportPath, "mismatched import path, have %q (%p), want %q (%p)", pkg.Path, pkg, importpkg.Path, importpkg)418			base.ErrorExit()419		}420421		if r.Version().Has(pkgbits.HasInit) {422			r.Bool()423		}424425		for i, n := 0, r.Len(); i < n; i++ {426			r.Sync(pkgbits.SyncObject)427			if r.Version().Has(pkgbits.DerivedFuncInstance) {428				assert(!r.Bool())429			}430			idx := r.Reloc(pkgbits.SectionObj)431			assert(r.Len() == 0)432433			path, name, code := r.p.PeekObj(idx)434			if code != pkgbits.ObjStub {435				objReader[types.NewPkg(path, "").Lookup(name)] = pkgReaderIndex{pr, idx, nil, nil, nil}436			}437		}438439		r.Sync(pkgbits.SyncEOF)440	}441442	if !localStub {443		r := pr.newReader(pkgbits.SectionMeta, pkgbits.PrivateRootIdx, pkgbits.SyncPrivate)444445		if r.Bool() {446			sym := importpkg.Lookup(".inittask")447			task := ir.NewNameAt(src.NoXPos, sym, nil)448			task.Class = ir.PEXTERN449			sym.Def = task450		}451452		for i, n := 0, r.Len(); i < n; i++ {453			path := r.String()454			name := r.String()455			idx := r.Reloc(pkgbits.SectionBody)456457			sym := types.NewPkg(path, "").Lookup(name)458			if _, ok := importBodyReader[sym]; !ok {459				importBodyReader[sym] = pkgReaderIndex{pr, idx, nil, nil, nil}460			}461		}462463		r.Sync(pkgbits.SyncEOF)464	}465}466467// writeUnifiedExport writes to `out` the finalized, self-contained468// Unified IR export data file for the current compilation unit.469func writeUnifiedExport(out io.Writer) {470	l := linker{471		pw: pkgbits.NewPkgEncoder(uirVersion, base.Debug.SyncFrames),472473		pkgs:   make(map[string]index),474		decls:  make(map[*types.Sym]index),475		bodies: make(map[*types.Sym]index),476	}477478	publicRootWriter := l.pw.NewEncoder(pkgbits.SectionMeta, pkgbits.SyncPublic)479	privateRootWriter := l.pw.NewEncoder(pkgbits.SectionMeta, pkgbits.SyncPrivate)480	assert(publicRootWriter.Idx == pkgbits.PublicRootIdx)481	assert(privateRootWriter.Idx == pkgbits.PrivateRootIdx)482483	var selfPkgIdx index484485	{486		pr := localPkgReader487		r := pr.NewDecoder(pkgbits.SectionMeta, pkgbits.PublicRootIdx, pkgbits.SyncPublic)488489		r.Sync(pkgbits.SyncPkg)490		selfPkgIdx = l.relocIdx(pr, pkgbits.SectionPkg, r.Reloc(pkgbits.SectionPkg))491492		// Versions must match.493		// TODO: It seems that we should be able to use r.Version() for NewPkgEncoder494		// instead of passing uirVersion, but NewPkgEncoder is created before r.495		// If that is correct, we should make that happen.496		assert(r.Version() == uirVersion)497498		if r.Version().Has(pkgbits.HasInit) {499			r.Bool()500		}501502		for i, n := 0, r.Len(); i < n; i++ {503			r.Sync(pkgbits.SyncObject)504			if r.Version().Has(pkgbits.DerivedFuncInstance) {505				assert(!r.Bool())506			}507			idx := r.Reloc(pkgbits.SectionObj)508			assert(r.Len() == 0)509510			xpath, xname, xtag := pr.PeekObj(idx)511			assert(xpath == pr.PkgPath())512			assert(xtag != pkgbits.ObjStub)513514			if types.IsExported(xname) {515				l.relocIdx(pr, pkgbits.SectionObj, idx)516			}517		}518519		r.Sync(pkgbits.SyncEOF)520	}521522	{523		var idxs []index524		for _, idx := range l.decls {525			idxs = append(idxs, idx)526		}527		slices.Sort(idxs)528529		w := publicRootWriter530531		w.Sync(pkgbits.SyncPkg)532		w.Reloc(pkgbits.SectionPkg, selfPkgIdx)533534		if w.Version().Has(pkgbits.HasInit) {535			w.Bool(false)536		}537538		w.Len(len(idxs))539		for _, idx := range idxs {540			w.Sync(pkgbits.SyncObject)541			if w.Version().Has(pkgbits.DerivedFuncInstance) {542				w.Bool(false)543			}544			w.Reloc(pkgbits.SectionObj, idx)545			w.Len(0)546		}547548		w.Sync(pkgbits.SyncEOF)549		w.Flush()550	}551552	{553		type symIdx struct {554			sym *types.Sym555			idx index556		}557		var bodies []symIdx558		for sym, idx := range l.bodies {559			bodies = append(bodies, symIdx{sym, idx})560		}561		slices.SortFunc(bodies, func(a, b symIdx) int { return cmp.Compare(a.idx, b.idx) })562563		w := privateRootWriter564565		w.Bool(typecheck.Lookup(".inittask").Def != nil)566567		w.Len(len(bodies))568		for _, body := range bodies {569			w.String(body.sym.Pkg.Path)570			w.String(body.sym.Name)571			w.Reloc(pkgbits.SectionBody, body.idx)572		}573574		w.Sync(pkgbits.SyncEOF)575		w.Flush()576	}577578	base.Ctxt.Fingerprint = l.pw.DumpTo(out)579}