1// Copyright 2025 The Go Authors. All rights reserved.2// Use of this source code is governed by a BSD-style3// license that can be found in the LICENSE file.45package main67import (8	"cmp"9	"fmt"10	"log"11	"math/rand/v2"12	"regexp"13	"slices"14	"strconv"15	"strings"16	"unicode"1718	"simd/archsimd/_gen/gentools"19	"simd/archsimd/_gen/simdgen/types"20	"simd/archsimd/_gen/unify"21)2223type rawOperation = types.RawOperation2425type Operation struct {26	rawOperation2728	// Go is the Go method name of this operation.29	//30	// It is derived from the raw Go method name by adding optional suffixes.31	// Currently, "Masked" is the only suffix.32	Go string3334	// Documentation is the doc string for this API.35	//36	// It is computed from the raw documentation:37	//38	// - "NAME" is replaced by the Go method name.39	//40	// - For masked operation, a sentence about masking is added.41	Documentation string4243	// In is the sequence of parameters to the Go method.44	//45	// For masked operations, this will have the mask operand appended.46	In []types.Operand47}4849func (o *Operation) IsMasked() bool {50	if len(o.InVariant) == 0 {51		return false52	}53	if len(o.InVariant) == 1 && o.InVariant[0].Class == "mask" {54		return true55	}56	panic(fmt.Errorf("unknown inVariant"))57}5859func (o *Operation) SkipMaskedMethod() bool {60	if o.HideMaskMethods == nil {61		return false62	}63	if *o.HideMaskMethods && o.IsMasked() {64		return true65	}66	return false67}6869// hiHalfKind returns "narrow" or "long" based on whether the operation narrows or widens its elements.70// Returns "" if HiHalfAsm is nil or classification is ambiguous.71func (o *Operation) hiHalfKind() string {72	if o.HiHalfAsm == nil {73		return ""74	}75	// Find the first vreg input and the first vreg output to compare elemBits.76	var inElemBits, outElemBits *int77	for i := range o.In {78		if o.In[i].Class == "vreg" && o.In[i].ElemBits != nil {79			inElemBits = o.In[i].ElemBits80			break81		}82	}83	for i := range o.Out {84		if o.Out[i].Class == "vreg" && o.Out[i].ElemBits != nil {85			outElemBits = o.Out[i].ElemBits86			break87		}88	}89	if inElemBits == nil || outElemBits == nil {90		return ""91	}92	if *outElemBits < *inElemBits {93		return "narrow"94	}95	if *outElemBits > *inElemBits {96		return "long"97	}98	return ""99}100101var reForName = regexp.MustCompile(`\bNAME\b`)102103func (o *Operation) DecodeUnified(v *unify.Value) error {104	if err := v.Decode(&o.rawOperation); err != nil {105		return err106	}107108	isMasked := o.IsMasked()109110	// Compute full Go method name.111	o.Go = o.rawOperation.Go112	if isMasked {113		o.Go += "Masked"114	}115116	// Compute doc string.117	if o.rawOperation.Documentation != nil {118		o.Documentation = *o.rawOperation.Documentation119	} else {120		o.Documentation = "// UNDOCUMENTED"121	}122	o.Documentation = reForName.ReplaceAllString(o.Documentation, o.Go)123	if isMasked {124		o.Documentation += "\n//\n// This operation is applied selectively under a write mask."125		// Suppress generic op and method declaration for exported methods, if a mask is present.126		if unicode.IsUpper([]rune(o.Go)[0]) {127			trueVal := "true"128			o.NoGenericOps = &trueVal129			o.NoTypes = &trueVal130		}131	}132	if o.rawOperation.AddDoc != nil {133		o.Documentation += "\n" + reForName.ReplaceAllString(*o.rawOperation.AddDoc, o.Go)134	}135136	o.In = append(o.rawOperation.In, o.rawOperation.InVariant...)137138	// For operations that read only the lower half of input registers (indicated by hiHalfAsm),139	// add a doc note showing the compositional pattern for the upper half.140	if o.rawOperation.HiHalfAsm != nil && o.hiHalfKind() == "long" {141		// Count vector-register inputs (exclude immediates/scalars).142		vregIns := 0143		for _, in := range o.In {144			if in.Class == "vreg" {145				vregIns++146			}147		}148		// note this is arm64-specific149		switch vregIns {150		case 2:151			// Binary: MulLong, AddLong, SubLong, etc.152			o.Documentation += "\n// For the high-indexed elements, use HiToLo:\n//\n//\tx.HiToLo()." + o.Go + "(y.HiToLo())"153		case 1:154			// Unary: ShiftLeftLongConst, etc.155			o.Documentation += "\n// For the high-indexed elements, use HiToLo:\n//\n//\tx.HiToLo()." + o.Go + "(...)"156		}157	}158159	// For down conversions, the high elements are zeroed if the result has more elements.160	// TODO: we should encode this logic in the YAML file, instead of hardcoding it here.161	if len(o.In) > 0 && len(o.Out) > 0 {162		inLanes := o.In[0].Lanes163		outLanes := o.Out[0].Lanes164		if inLanes != nil && outLanes != nil && *inLanes < *outLanes {165			if (strings.Contains(o.Go, "Saturate") || strings.Contains(o.Go, "TruncTo")) &&166				!strings.Contains(o.Go, "Concat") {167				o.Documentation += "\n// Results are packed to low elements in the returned vector, its upper elements are zeroed."168			}169		}170	}171172	return nil173}174175func (o *Operation) VectorWidth() int {176	out := o.Out[0]177	if out.Class == "vreg" {178		return out.Bits.N()179	} else if out.Class == "greg" || out.Class == "mask" {180		for i := range o.In {181			if o.In[i].Class == "vreg" {182				return o.In[i].Bits.N()183			}184		}185	}186	panic(fmt.Errorf("Figure out what the vector width is for %v and implement it", *o))187}188189// Right now simdgen computes the machine op name for most instructions190// as $Name$OutputSize, by this denotation, these instructions are "overloaded".191// for example:192// (Uint16x8) ConvertToInt8193// (Uint16x16) ConvertToInt8194// are both VPMOVWB128.195// To make them distinguishable we need to append the input size to them as well.196// TODO: document them well in the generated code.197var demotingConvertOps = map[string]bool{198	"VPMOVQD128": true, "VPMOVSQD128": true, "VPMOVUSQD128": true, "VPMOVQW128": true, "VPMOVSQW128": true,199	"VPMOVUSQW128": true, "VPMOVDW128": true, "VPMOVSDW128": true, "VPMOVUSDW128": true, "VPMOVQB128": true,200	"VPMOVSQB128": true, "VPMOVUSQB128": true, "VPMOVDB128": true, "VPMOVSDB128": true, "VPMOVUSDB128": true,201	"VPMOVWB128": true, "VPMOVSWB128": true, "VPMOVUSWB128": true,202	"VPMOVQDMasked128": true, "VPMOVSQDMasked128": true, "VPMOVUSQDMasked128": true, "VPMOVQWMasked128": true, "VPMOVSQWMasked128": true,203	"VPMOVUSQWMasked128": true, "VPMOVDWMasked128": true, "VPMOVSDWMasked128": true, "VPMOVUSDWMasked128": true, "VPMOVQBMasked128": true,204	"VPMOVSQBMasked128": true, "VPMOVUSQBMasked128": true, "VPMOVDBMasked128": true, "VPMOVSDBMasked128": true, "VPMOVUSDBMasked128": true,205	"VPMOVWBMasked128": true, "VPMOVSWBMasked128": true, "VPMOVUSWBMasked128": true,206}207208// sveArrangementLetter returns the SVE element-size arrangement letter209// (B=8, H=16, S=32, D=64) that names an SVE machine op, or "" when the target210// is not SVE. The letter comes from the operation's governing element width:211// the output vreg's elemBits, else the first vreg/mask operand's elemBits.212func sveArrangementLetter(gOp Operation) string {213	if !CurrentArch().isSVE() {214		return ""215	}216	elemBits := 0217	pick := func(ops []types.Operand) {218		if elemBits != 0 {219			return220		}221		for i := range ops {222			if c := ops[i].Class; (c == "vreg" || c == "mask") && ops[i].ElemBits != nil {223				elemBits = *ops[i].ElemBits224				return225			}226		}227	}228	pick(gOp.Out)229	pick(gOp.In)230	switch elemBits {231	case 8:232		return "B"233	case 16:234		return "H"235	case 32:236		return "S"237	case 64:238		return "D"239	}240	panic(fmt.Errorf("SVE op %s has no B/H/S/D element width (elemBits=%d)", gOp.Asm, elemBits))241}242243func machineOpName(maskType maskShape, gOp Operation) string {244	asm := gOp.Asm245	if maskType == OneMask {246		asm += "Masked"247	}248	// For ARM64, use arrangement to create distinct SSA op names249	if letter := sveArrangementLetter(gOp); letter != "" {250		// SVE: scalable vectors have no fixed width, so distinguish machine ops251		// by element-size arrangement letter (B/H/S/D), e.g. ZADD -> ZADDB.252		asm += letter253	} else if gOp.Arrangement != nil && *gOp.Arrangement != "" {254		asm = fmt.Sprintf("%s%s", asm, *gOp.Arrangement)255	} else {256		asm = fmt.Sprintf("%s%d", asm, gOp.VectorWidth())257	}258	if gOp.SSAVariant != nil {259		asm += *gOp.SSAVariant260	}261	if demotingConvertOps[asm] {262		// Need to append the size of the source as well.263		// TODO: should be "%sto%d".264		asm = fmt.Sprintf("%s_%d", asm, gOp.In[0].Bits.N())265	}266	return asm267}268269func compareStringPointers(x, y *string) int {270	if x != nil && y != nil {271		return compareNatural(*x, *y)272	}273	if x == nil && y == nil {274		return 0275	}276	if x == nil {277		return -1278	}279	return 1280}281282func compareIntPointers(x, y *int) int {283	if x != nil && y != nil {284		return *x - *y285	}286	if x == nil && y == nil {287		return 0288	}289	if x == nil {290		return -1291	}292	return 1293}294295func compareVectorSizes(x, y types.VectorSize) int {296	if x.Scalable != y.Scalable {297		if !x.Scalable {298			return -1299		}300		return 1301	}302	if !x.Scalable {303		return cmp.Compare(x.NRaw, y.NRaw)304	}305	return 0306}307308func compareOperations(x, y Operation) int {309	if c := compareNatural(x.Go, y.Go); c != 0 {310		return c311	}312	xIn, yIn := x.In, y.In313314	if len(xIn) > len(yIn) && xIn[len(xIn)-1].Class == "mask" {315		xIn = xIn[:len(xIn)-1]316	} else if len(xIn) < len(yIn) && yIn[len(yIn)-1].Class == "mask" {317		yIn = yIn[:len(yIn)-1]318	}319320	if len(xIn) < len(yIn) {321		return -1322	}323	if len(xIn) > len(yIn) {324		return 1325	}326	if len(x.Out) < len(y.Out) {327		return -1328	}329	if len(x.Out) > len(y.Out) {330		return 1331	}332	for i := range xIn {333		ox, oy := &xIn[i], &yIn[i]334		if c := compareOperands(ox, oy); c != 0 {335			return c336		}337	}338	return 0339}340341func compareOperands(x, y *types.Operand) int {342	if c := compareNatural(x.Class, y.Class); c != 0 {343		return c344	}345	if x.Class == "immediate" {346		return compareStringPointers(x.ImmOffset, y.ImmOffset)347	} else {348		if c := compareStringPointers(x.Base, y.Base); c != 0 {349			return c350		}351		if c := compareIntPointers(x.ElemBits, y.ElemBits); c != 0 {352			return c353		}354		if c := compareVectorSizes(x.Bits, y.Bits); c != 0 {355			return c356		}357		if c := compareIntPointers(x.ListNumber, y.ListNumber); c != 0 {358			return c359		}360		return 0361	}362}363364// implicitPredCount reports whether the op has an implicit-all-true governing365// predicate input, as a count (0 or 1). An instruction has at most one governing366// predicate — the single mask input carrying a /Z or /M qualifier (see the367// role=="mask" operand in sve.buildOperandList) — which is a real machine-op368// input the lowering synthesizes as all-true but which is invisible in the Go369// API. So the generic op, intrinsic and stub size themselves by len(In) minus370// this. Source predicates (e.g. Pn, Pm in a predicate-logical op) are ordinary371// numbered inputs, not governing predicates, and are never counted.372func (op Operation) implicitPredCount() int {373	n := 0374	for i := range op.In {375		if op.In[i].IsImplicitAllTrue() {376			n++377		}378	}379	return n380}381382// isDigit returns true if the byte is an ASCII digit.383func isDigit(b byte) bool {384	return b >= '0' && b <= '9'385}386387// compareNatural performs a "natural sort" comparison of two strings.388// It compares non-digit sections lexicographically and digit sections389// numerically.  In the case of string-unequal "equal" strings like390// "a01b" and "a1b", strings.Compare breaks the tie.391//392// It returns:393//394//	-1 if s1 < s2395//	 0 if s1 == s2396//	+1 if s1 > s2397func compareNatural(s1, s2 string) int {398	i, j := 0, 0399	len1, len2 := len(s1), len(s2)400401	for i < len1 && j < len2 {402		// Find a non-digit segment or a number segment in both strings.403		if isDigit(s1[i]) && isDigit(s2[j]) {404			// Number segment comparison.405			numStart1 := i406			for i < len1 && isDigit(s1[i]) {407				i++408			}409			num1, _ := strconv.Atoi(s1[numStart1:i])410411			numStart2 := j412			for j < len2 && isDigit(s2[j]) {413				j++414			}415			num2, _ := strconv.Atoi(s2[numStart2:j])416417			if num1 < num2 {418				return -1419			}420			if num1 > num2 {421				return 1422			}423			// "1" < "01".  Don't expect it in simdgen, but just in case.424			if ln1, ln2 := i-numStart1, j-numStart2; ln1 != ln2 {425				return ln1 - ln2426			}427			// If numbers are equal, continue to the next segment.428		} else {429			// Non-digit comparison.430			if s1[i] < s2[j] {431				return -1432			}433			if s1[i] > s2[j] {434				return 1435			}436			i++437			j++438		}439	}440441	// deal with a01b vs a1b; there needs to be an order.442	return strings.Compare(s1, s2)443}444445// generatedHeader returns the architecture-specific header for generated files.446func generatedHeader() string {447	return CurrentArch().GeneratedHeader448}449450func writeGoDefs(cl unify.Closure) error {451	// TODO: Merge operations with the same signature but multiple452	// implementations (e.g., SSE vs AVX)453	var ops []Operation454	for def := range cl.All() {455		var op Operation456		if !def.Exact() {457			continue458		}459		if err := def.Decode(&op); err != nil {460			log.Println(err.Error())461			log.Println(def)462			continue463		}464		op.adjustAsm()465		ops = append(ops, op)466	}467468	rand.Shuffle(len(ops), func(i, j int) {469		ops[i], ops[j] = ops[j], ops[i]470	})471472	slices.SortFunc(ops, compareOperations)473	// The parsed XED data might contain duplicates, like474	// 512 bits VPADDP.475	deduped := dedup(ops)476	slices.SortFunc(deduped, compareOperations)477478	if *Verbose {479		log.Printf("dedup len: %d, ops len: %d\n", len(deduped), len(ops))480	}481	var err error482	if err = overwrite(deduped); err != nil {483		return err484	}485	if *Verbose {486		log.Printf("dedup len: %d\n", len(deduped))487	}488	if !*FlagNoDedup {489		// TODO: This can hide mistakes in the API definitions, especially when490		// multiple patterns result in the same API unintentionally. Make it stricter.491		if deduped, err = dedupGodef(deduped); err != nil {492			return err493		}494	}495	if *Verbose {496		log.Printf("dedup len: %d\n", len(deduped))497	}498	if !*FlagNoConstImmPorting {499		if err = copyConstImm(deduped); err != nil {500			return err501		}502	}503	if *Verbose {504		log.Printf("dedup len: %d\n", len(deduped))505	}506	reportXEDInconsistency(deduped)507508	// Sorting again, just in case.509	slices.SortFunc(deduped, compareOperations)510511	typeMap := parseSIMDTypes(deduped)512513	archInfo := CurrentArch()514	// Generated files are named by GoTypeArch: the Go API files directly, the515	// backend files by SIMDTag. For amd64/arm64 these match the516	// GOARCH, so those filenames are unchanged; only SVE diverges (sve/SVE) so its517	// output sits alongside the NEON arm64 files instead of overwriting them.518	simdTag := archInfo.SIMDTag519	goTypeArch := archInfo.GoTypeArch520	archLower := archInfo.Arch521522	var files gentools.Files523	defer files.FlushOrExit()524525	writeSIMDTypes(files.NewGoFile(simdPackage+"/types_"+goTypeArch+".go"), typeMap)526	// TODO: Enable CPU feature generation for non-x86 architectures.527	if archLower == "amd64" {528		writeSIMDFeatures(files.NewGoFile(simdPackage+"/cpu.go"), deduped)529	}530	writeSIMDStubs(531		files.NewGoFile(simdPackage+"/ops_"+goTypeArch+".go"),532		files.NewGoFile(simdPackage+"/ops_internal_"+goTypeArch+".go"),533		deduped, typeMap, archLower == "amd64",534	)535	writeSIMDIntrinsics(files.NewGoFile("cmd/compile/internal/ssagen/simd"+simdTag+"intrinsics.go"), deduped, typeMap)536	const simdGenericOpsFile = "cmd/compile/internal/ssa/_gen/simdgenericOps.go"537	writeSIMDGenericOps(files.NewGoFile(simdGenericOpsFile), deduped, genFlags.InputPath(simdGenericOpsFile))538	writeSIMDMachineOps(files.NewGoFile("cmd/compile/internal/ssa/_gen/simd"+simdTag+"ops.go"), deduped)539	writeSIMDSSA(files.NewGoFile("cmd/compile/internal/"+archLower+"/"+archInfo.ssaGenFile()), deduped)540	writeSIMDRules(files.NewRawFile("cmd/compile/internal/ssa/_gen/simd"+simdTag+".rules"), deduped)541542	return nil543}