1// Copyright 2016 The Go Authors. All rights reserved.2// Use of this source code is governed by a BSD-style3// license that can be found in the LICENSE file.45package amd6467import (8	"fmt"9	"math"1011	"cmd/compile/internal/base"12	"cmd/compile/internal/ir"13	"cmd/compile/internal/logopt"14	"cmd/compile/internal/objw"15	"cmd/compile/internal/ssa"16	"cmd/compile/internal/ssa/block"17	"cmd/compile/internal/ssa/ssaop"18	"cmd/compile/internal/ssagen"19	"cmd/compile/internal/types"20	"cmd/internal/obj"21	"cmd/internal/obj/x86"22	"internal/abi"23	"internal/buildcfg"24)2526// ssaMarkMoves marks any MOVXconst ops that need to avoid clobbering flags.27func ssaMarkMoves(s *ssagen.State, b *ssa.Block) {28	flive := b.FlagsLiveAtEnd29	for _, c := range b.ControlValues() {30		flive = c.Type.IsFlags() || flive31	}32	for i := len(b.Values) - 1; i >= 0; i-- {33		v := b.Values[i]34		if flive && (v.Op == ssaop.OpAMD64MOVLconst || v.Op == ssaop.OpAMD64MOVQconst) {35			// The "mark" is any non-nil Aux value.36			v.Aux = ssa.AuxMark37		}38		if v.Type.IsFlags() {39			flive = false40		}41		for _, a := range v.Args {42			if a.Type.IsFlags() {43				flive = true44			}45		}46	}47}4849func isGPReg(r int16) bool {50	return x86.REG_AL <= r && r <= x86.REG_R1551}5253func isFPReg(r int16) bool {54	return x86.REG_X0 <= r && r <= x86.REG_Z3155}5657func isKReg(r int16) bool {58	return x86.REG_K0 <= r && r <= x86.REG_K759}6061func isLowFPReg(r int16) bool {62	return x86.REG_X0 <= r && r <= x86.REG_X1563}6465func isHighFPReg(r int16) bool {66	return x86.REG_X16 <= r && r <= x86.REG_X31 || x86.REG_Y16 <= r && r <= x86.REG_Y31 || x86.REG_Z16 <= r && r <= x86.REG_Z3167}6869// loadByRegWidth returns the load instruction of the given register of a given width.70func loadByRegWidth(r int16, width int64) obj.As {71	// Avoid partial register write for GPR72	if !isFPReg(r) && !isKReg(r) {73		switch width {74		case 1:75			return x86.AMOVBLZX76		case 2:77			return x86.AMOVWLZX78		}79	}80	// Otherwise, there's no difference between load and store opcodes.81	return storeByRegWidth(r, width)82}8384// storeByRegWidth returns the store instruction of the given register of a given width.85// It's also used for loading const to a reg.86func storeByRegWidth(r int16, width int64) obj.As {87	if isHighFPReg(r) {88		// High registers require AVX512 instruction89		return x86.AVMOVDQU6490	}91	if isFPReg(r) {92		switch width {93		case 4:94			return x86.AMOVSS95		case 8:96			return x86.AMOVSD97		case 16:98			// int128s are in SSE registers99			return x86.AMOVUPS100		case 32:101			return x86.AVMOVDQU102		case 64:103			return x86.AVMOVDQU64104		}105	}106	if isKReg(r) {107		return x86.AKMOVQ108	}109	// gp110	switch width {111	case 1:112		return x86.AMOVB113	case 2:114		return x86.AMOVW115	case 4:116		return x86.AMOVL117	case 8:118		return x86.AMOVQ119	}120	panic(fmt.Sprintf("bad store reg=%v, width=%d", r, width))121}122123// moveByRegsWidth returns the reg->reg move instruction of the given dest/src registers of a given width.124func moveByRegsWidth(dest, src int16, width int64) obj.As {125	// fp -> fp126	if isFPReg(dest) && isFPReg(src) {127		if isHighFPReg(src) || isHighFPReg(dest) {128			// High registers require AVX512 instruction129			return x86.AVMOVDQU64130		}131		// Moving the whole sse2 register is faster132		// than moving just the correct low portion of it.133		// There is no xmm->xmm move with 1 byte opcode,134		// so use movups, which has 2 byte opcode.135		if width <= 16 {136			return x86.AMOVUPS137		}138		if width <= 32 {139			return x86.AVMOVDQU140		}141		return x86.AVMOVDQU64142	}143	// k -> gp, gp -> k, k -> k144	if isKReg(dest) || isKReg(src) {145		if isFPReg(dest) || isFPReg(src) {146			panic(fmt.Sprintf("bad move, src=%v, dest=%v, width=%d", src, dest, width))147		}148		return x86.AKMOVQ149	}150	// gp -> fp, fp -> gp, gp -> gp151	switch width {152	case 1:153		// Avoids partial register write154		return x86.AMOVL155	case 2:156		return x86.AMOVL157	case 4:158		return x86.AMOVL159	case 8:160		return x86.AMOVQ161	case 16:162		// int128s are in SSE registers163		return x86.AMOVUPS164	case 32:165		return x86.AVMOVDQU166	case 64:167		return x86.AVMOVDQU64168	}169	panic(fmt.Sprintf("bad move, src=%v, dest=%v, width=%d", src, dest, width))170}171172// opregreg emits instructions for173//174//	dest := dest(To) op src(From)175//176// and also returns the created obj.Prog so it177// may be further adjusted (offset, scale, etc).178func opregreg(s *ssagen.State, op obj.As, dest, src int16) *obj.Prog {179	p := s.Prog(op)180	p.From.Type = obj.TYPE_REG181	p.To.Type = obj.TYPE_REG182	p.To.Reg = dest183	p.From.Reg = src184	return p185}186187// memIdx fills out a as an indexed memory reference for v.188// It assumes that the base register and the index register189// are v.Args[0].Reg() and v.Args[1].Reg(), respectively.190// The caller must still use gc.AddAux/gc.AddAux2 to handle v.Aux as necessary.191func memIdx(a *obj.Addr, v *ssa.Value) {192	r, i := v.Args[0].Reg(), v.Args[1].Reg()193	a.Type = obj.TYPE_MEM194	a.Scale = v.Op.Scale()195	if a.Scale == 1 && i == x86.REG_SP {196		r, i = i, r197	}198	a.Reg = r199	a.Index = i200}201202func getgFromTLS(s *ssagen.State, r int16) {203	// See the comments in cmd/internal/obj/x86/obj6.go204	// near CanUse1InsnTLS for a detailed explanation of these instructions.205	if x86.CanUse1InsnTLS(base.Ctxt) {206		// MOVQ (TLS), r207		p := s.Prog(x86.AMOVQ)208		p.From.Type = obj.TYPE_MEM209		p.From.Reg = x86.REG_TLS210		p.To.Type = obj.TYPE_REG211		p.To.Reg = r212	} else {213		// MOVQ TLS, r214		// MOVQ (r)(TLS*1), r215		p := s.Prog(x86.AMOVQ)216		p.From.Type = obj.TYPE_REG217		p.From.Reg = x86.REG_TLS218		p.To.Type = obj.TYPE_REG219		p.To.Reg = r220		q := s.Prog(x86.AMOVQ)221		q.From.Type = obj.TYPE_MEM222		q.From.Reg = r223		q.From.Index = x86.REG_TLS224		q.From.Scale = 1225		q.To.Type = obj.TYPE_REG226		q.To.Reg = r227	}228}229230func ssaGenValue(s *ssagen.State, v *ssa.Value) {231	switch v.Op {232	case ssaop.OpAMD64VFMADD231SD, ssaop.OpAMD64VFMADD231SS, ssaop.OpAMD64VFMSUB231SD, ssaop.OpAMD64VFMSUB231SS, ssaop.OpAMD64VFNMADD231SD, ssaop.OpAMD64VFNMADD231SS:233		p := s.Prog(v.Op.Asm())234		p.From = obj.Addr{Type: obj.TYPE_REG, Reg: v.Args[2].Reg()}235		p.To = obj.Addr{Type: obj.TYPE_REG, Reg: v.Reg()}236		p.AddRestSourceReg(v.Args[1].Reg())237	case ssaop.OpAMD64ADDQ, ssaop.OpAMD64ADDL:238		r := v.Reg()239		r1 := v.Args[0].Reg()240		r2 := v.Args[1].Reg()241		switch {242		case r == r1:243			p := s.Prog(v.Op.Asm())244			p.From.Type = obj.TYPE_REG245			p.From.Reg = r2246			p.To.Type = obj.TYPE_REG247			p.To.Reg = r248		case r == r2:249			p := s.Prog(v.Op.Asm())250			p.From.Type = obj.TYPE_REG251			p.From.Reg = r1252			p.To.Type = obj.TYPE_REG253			p.To.Reg = r254		default:255			var asm obj.As256			if v.Op == ssaop.OpAMD64ADDQ {257				asm = x86.ALEAQ258			} else {259				asm = x86.ALEAL260			}261			p := s.Prog(asm)262			p.From.Type = obj.TYPE_MEM263			p.From.Reg = r1264			p.From.Scale = 1265			p.From.Index = r2266			p.To.Type = obj.TYPE_REG267			p.To.Reg = r268		}269	// 2-address opcode arithmetic270	case ssaop.OpAMD64SUBQ, ssaop.OpAMD64SUBL,271		ssaop.OpAMD64MULQ, ssaop.OpAMD64MULL,272		ssaop.OpAMD64ANDQ, ssaop.OpAMD64ANDL,273		ssaop.OpAMD64ORQ, ssaop.OpAMD64ORL,274		ssaop.OpAMD64XORQ, ssaop.OpAMD64XORL,275		ssaop.OpAMD64SHLQ, ssaop.OpAMD64SHLL,276		ssaop.OpAMD64SHRQ, ssaop.OpAMD64SHRL, ssaop.OpAMD64SHRW, ssaop.OpAMD64SHRB,277		ssaop.OpAMD64SARQ, ssaop.OpAMD64SARL, ssaop.OpAMD64SARW, ssaop.OpAMD64SARB,278		ssaop.OpAMD64ROLQ, ssaop.OpAMD64ROLL, ssaop.OpAMD64ROLW, ssaop.OpAMD64ROLB,279		ssaop.OpAMD64RORQ, ssaop.OpAMD64RORL, ssaop.OpAMD64RORW, ssaop.OpAMD64RORB,280		ssaop.OpAMD64ADDSS, ssaop.OpAMD64ADDSD, ssaop.OpAMD64SUBSS, ssaop.OpAMD64SUBSD,281		ssaop.OpAMD64MULSS, ssaop.OpAMD64MULSD, ssaop.OpAMD64DIVSS, ssaop.OpAMD64DIVSD,282		ssaop.OpAMD64MINSS, ssaop.OpAMD64MINSD,283		ssaop.OpAMD64MAXSS, ssaop.OpAMD64MAXSD,284		ssaop.OpAMD64POR, ssaop.OpAMD64PXOR,285		ssaop.OpAMD64BTSL, ssaop.OpAMD64BTSQ,286		ssaop.OpAMD64BTCL, ssaop.OpAMD64BTCQ,287		ssaop.OpAMD64BTRL, ssaop.OpAMD64BTRQ,288		ssaop.OpAMD64PCMPEQB, ssaop.OpAMD64PSIGNB,289		ssaop.OpAMD64PUNPCKLBW:290		opregreg(s, v.Op.Asm(), v.Reg(), v.Args[1].Reg())291292	case ssaop.OpAMD64PSHUFLW:293		p := s.Prog(v.Op.Asm())294		imm := v.AuxInt295		if imm < 0 || imm > 255 {296			v.Fatalf("Invalid source selection immediate")297		}298		p.From.Offset = imm299		p.From.Type = obj.TYPE_CONST300		p.AddRestSourceReg(v.Args[0].Reg())301		p.To.Type = obj.TYPE_REG302		p.To.Reg = v.Reg()303304	case ssaop.OpAMD64PSHUFBbroadcast:305		// PSHUFB with a control mask of zero copies byte 0 to all306		// bytes in the register.307		//308		// X15 is always zero with ABIInternal.309		if s.ABI != obj.ABIInternal {310			// zero X15 manually311			opregreg(s, x86.AXORPS, x86.REG_X15, x86.REG_X15)312		}313314		p := s.Prog(v.Op.Asm())315		p.From.Type = obj.TYPE_REG316		p.To.Type = obj.TYPE_REG317		p.To.Reg = v.Reg()318		p.From.Reg = x86.REG_X15319320	case ssaop.OpAMD64BLSIQ, ssaop.OpAMD64BLSIL,321		ssaop.OpAMD64BLSMSKQ, ssaop.OpAMD64BLSMSKL,322		ssaop.OpAMD64BLSRQ, ssaop.OpAMD64BLSRL:323		p := s.Prog(v.Op.Asm())324		p.From.Type = obj.TYPE_REG325		p.From.Reg = v.Args[0].Reg()326		p.To.Type = obj.TYPE_REG327		switch v.Op {328		case ssaop.OpAMD64BLSRQ, ssaop.OpAMD64BLSRL:329			p.To.Reg = v.Reg0()330		default:331			p.To.Reg = v.Reg()332		}333334	case ssaop.OpAMD64ANDNQ, ssaop.OpAMD64ANDNL:335		p := s.Prog(v.Op.Asm())336		p.From.Type = obj.TYPE_REG337		p.From.Reg = v.Args[0].Reg()338		p.To.Type = obj.TYPE_REG339		p.To.Reg = v.Reg()340		p.AddRestSourceReg(v.Args[1].Reg())341342	case ssaop.OpAMD64SARXL, ssaop.OpAMD64SARXQ,343		ssaop.OpAMD64SHLXL, ssaop.OpAMD64SHLXQ,344		ssaop.OpAMD64SHRXL, ssaop.OpAMD64SHRXQ:345		p := opregreg(s, v.Op.Asm(), v.Reg(), v.Args[1].Reg())346		p.AddRestSourceReg(v.Args[0].Reg())347348	case ssaop.OpAMD64SHLXLload, ssaop.OpAMD64SHLXQload,349		ssaop.OpAMD64SHRXLload, ssaop.OpAMD64SHRXQload,350		ssaop.OpAMD64SARXLload, ssaop.OpAMD64SARXQload:351		p := opregreg(s, v.Op.Asm(), v.Reg(), v.Args[1].Reg())352		m := obj.Addr{Type: obj.TYPE_MEM, Reg: v.Args[0].Reg()}353		ssagen.AddAux(&m, v)354		p.AddRestSource(m)355356	case ssaop.OpAMD64SHLXLloadidx1, ssaop.OpAMD64SHLXLloadidx4, ssaop.OpAMD64SHLXLloadidx8,357		ssaop.OpAMD64SHRXLloadidx1, ssaop.OpAMD64SHRXLloadidx4, ssaop.OpAMD64SHRXLloadidx8,358		ssaop.OpAMD64SARXLloadidx1, ssaop.OpAMD64SARXLloadidx4, ssaop.OpAMD64SARXLloadidx8,359		ssaop.OpAMD64SHLXQloadidx1, ssaop.OpAMD64SHLXQloadidx8,360		ssaop.OpAMD64SHRXQloadidx1, ssaop.OpAMD64SHRXQloadidx8,361		ssaop.OpAMD64SARXQloadidx1, ssaop.OpAMD64SARXQloadidx8:362		p := opregreg(s, v.Op.Asm(), v.Reg(), v.Args[2].Reg())363		m := obj.Addr{Type: obj.TYPE_MEM}364		memIdx(&m, v)365		ssagen.AddAux(&m, v)366		p.AddRestSource(m)367368	case ssaop.OpAMD64DIVQU, ssaop.OpAMD64DIVLU, ssaop.OpAMD64DIVWU:369		// Arg[0] (the dividend) is in AX.370		// Arg[1] (the divisor) can be in any other register.371		// Result[0] (the quotient) is in AX.372		// Result[1] (the remainder) is in DX.373		r := v.Args[1].Reg()374375		// Zero extend dividend.376		opregreg(s, x86.AXORL, x86.REG_DX, x86.REG_DX)377378		// Issue divide.379		p := s.Prog(v.Op.Asm())380		p.From.Type = obj.TYPE_REG381		p.From.Reg = r382383	case ssaop.OpAMD64DIVQ, ssaop.OpAMD64DIVL, ssaop.OpAMD64DIVW:384		// Arg[0] (the dividend) is in AX.385		// Arg[1] (the divisor) can be in any other register.386		// Result[0] (the quotient) is in AX.387		// Result[1] (the remainder) is in DX.388		r := v.Args[1].Reg()389390		var opCMP, opNEG, opSXD obj.As391		switch v.Op {392		case ssaop.OpAMD64DIVQ:393			opCMP, opNEG, opSXD = x86.ACMPQ, x86.ANEGQ, x86.ACQO394		case ssaop.OpAMD64DIVL:395			opCMP, opNEG, opSXD = x86.ACMPL, x86.ANEGL, x86.ACDQ396		case ssaop.OpAMD64DIVW:397			opCMP, opNEG, opSXD = x86.ACMPW, x86.ANEGW, x86.ACWD398		}399400		// CPU faults upon signed overflow, which occurs when the most401		// negative int is divided by -1. Handle divide by -1 as a special case.402		var j1, j2 *obj.Prog403		if ssa.DivisionNeedsFixUp(v) {404			c := s.Prog(opCMP)405			c.From.Type = obj.TYPE_REG406			c.From.Reg = r407			c.To.Type = obj.TYPE_CONST408			c.To.Offset = -1409410			// Divisor is not -1, proceed with normal division.411			j1 = s.Prog(x86.AJNE)412			j1.To.Type = obj.TYPE_BRANCH413414			// Divisor is -1, manually compute quotient and remainder via fixup code.415			// n / -1 = -n416			n1 := s.Prog(opNEG)417			n1.To.Type = obj.TYPE_REG418			n1.To.Reg = x86.REG_AX419420			// n % -1 == 0421			opregreg(s, x86.AXORL, x86.REG_DX, x86.REG_DX)422423			// TODO(khr): issue only the -1 fixup code we need.424			// For instance, if only the quotient is used, no point in zeroing the remainder.425426			// Skip over normal division.427			j2 = s.Prog(obj.AJMP)428			j2.To.Type = obj.TYPE_BRANCH429		}430431		// Sign extend dividend and perform division.432		p := s.Prog(opSXD)433		if j1 != nil {434			j1.To.SetTarget(p)435		}436		p = s.Prog(v.Op.Asm())437		p.From.Type = obj.TYPE_REG438		p.From.Reg = r439440		if j2 != nil {441			j2.To.SetTarget(s.Pc())442		}443444	case ssaop.OpAMD64HMULQ, ssaop.OpAMD64HMULL, ssaop.OpAMD64HMULQU, ssaop.OpAMD64HMULLU:445		// the frontend rewrites constant division by 8/16/32 bit integers into446		// HMUL by a constant447		// SSA rewrites generate the 64 bit versions448449		// Arg[0] is already in AX as it's the only register we allow450		// and DX is the only output we care about (the high bits)451		p := s.Prog(v.Op.Asm())452		p.From.Type = obj.TYPE_REG453		p.From.Reg = v.Args[1].Reg()454455		// IMULB puts the high portion in AH instead of DL,456		// so move it to DL for consistency457		if v.Type.Size() == 1 {458			m := s.Prog(x86.AMOVB)459			m.From.Type = obj.TYPE_REG460			m.From.Reg = x86.REG_AH461			m.To.Type = obj.TYPE_REG462			m.To.Reg = x86.REG_DX463		}464465	case ssaop.OpAMD64MULQU, ssaop.OpAMD64MULLU:466		// Arg[0] is already in AX as it's the only register we allow467		// results lo in AX468		p := s.Prog(v.Op.Asm())469		p.From.Type = obj.TYPE_REG470		p.From.Reg = v.Args[1].Reg()471472	case ssaop.OpAMD64MULQU2:473		// Arg[0] is already in AX as it's the only register we allow474		// results hi in DX, lo in AX475		p := s.Prog(v.Op.Asm())476		p.From.Type = obj.TYPE_REG477		p.From.Reg = v.Args[1].Reg()478479	case ssaop.OpAMD64MULXQ:480		// Arg[0] is already in DX (the implicit operand); Arg[1] is any GP/mem.481		// SSA outputs are (hi, lo) -> Reg0()=hi, Reg1()=lo.482		// Go assembler syntax: MULXQ src, lo, hi (encodes vvvv=lo, reg=hi).483		p := s.Prog(v.Op.Asm())484		p.From.Type = obj.TYPE_REG485		p.From.Reg = v.Args[1].Reg()486		p.AddRestSourceReg(v.Reg1())487		p.To.Type = obj.TYPE_REG488		p.To.Reg = v.Reg0()489490	case ssaop.OpAMD64DIVQU2:491		// Arg[0], Arg[1] are already in Dx, AX, as they're the only registers we allow492		// results q in AX, r in DX493		p := s.Prog(v.Op.Asm())494		p.From.Type = obj.TYPE_REG495		p.From.Reg = v.Args[2].Reg()496497	case ssaop.OpAMD64AVGQU:498		// compute (x+y)/2 unsigned.499		// Do a 64-bit add, the overflow goes into the carry.500		// Shift right once and pull the carry back into the 63rd bit.501		p := s.Prog(x86.AADDQ)502		p.From.Type = obj.TYPE_REG503		p.To.Type = obj.TYPE_REG504		p.To.Reg = v.Reg()505		p.From.Reg = v.Args[1].Reg()506		p = s.Prog(x86.ARCRQ)507		p.From.Type = obj.TYPE_CONST508		p.From.Offset = 1509		p.To.Type = obj.TYPE_REG510		p.To.Reg = v.Reg()511512	case ssaop.OpAMD64ADDQcarry, ssaop.OpAMD64ADCQ:513		r := v.Reg0()514		r0 := v.Args[0].Reg()515		r1 := v.Args[1].Reg()516		switch r {517		case r0:518			p := s.Prog(v.Op.Asm())519			p.From.Type = obj.TYPE_REG520			p.From.Reg = r1521			p.To.Type = obj.TYPE_REG522			p.To.Reg = r523		case r1:524			p := s.Prog(v.Op.Asm())525			p.From.Type = obj.TYPE_REG526			p.From.Reg = r0527			p.To.Type = obj.TYPE_REG528			p.To.Reg = r529		default:530			v.Fatalf("output not in same register as an input %s", v.LongString())531		}532533	case ssaop.OpAMD64SUBQborrow, ssaop.OpAMD64SBBQ:534		p := s.Prog(v.Op.Asm())535		p.From.Type = obj.TYPE_REG536		p.From.Reg = v.Args[1].Reg()537		p.To.Type = obj.TYPE_REG538		p.To.Reg = v.Reg0()539540	case ssaop.OpAMD64ADDQconstcarry, ssaop.OpAMD64ADCQconst, ssaop.OpAMD64SUBQconstborrow, ssaop.OpAMD64SBBQconst:541		p := s.Prog(v.Op.Asm())542		p.From.Type = obj.TYPE_CONST543		p.From.Offset = v.AuxInt544		p.To.Type = obj.TYPE_REG545		p.To.Reg = v.Reg0()546547	case ssaop.OpAMD64ADDQconst, ssaop.OpAMD64ADDLconst:548		r := v.Reg()549		a := v.Args[0].Reg()550		if r == a {551			switch v.AuxInt {552			case 1:553				var asm obj.As554				// Software optimization manual recommends add $1,reg.555				// But inc/dec is 1 byte smaller. ICC always uses inc556				// Clang/GCC choose depending on flags, but prefer add.557				// Experiments show that inc/dec is both a little faster558				// and make a binary a little smaller.559				if v.Op == ssaop.OpAMD64ADDQconst {560					asm = x86.AINCQ561				} else {562					asm = x86.AINCL563				}564				p := s.Prog(asm)565				p.To.Type = obj.TYPE_REG566				p.To.Reg = r567				return568			case -1:569				var asm obj.As570				if v.Op == ssaop.OpAMD64ADDQconst {571					asm = x86.ADECQ572				} else {573					asm = x86.ADECL574				}575				p := s.Prog(asm)576				p.To.Type = obj.TYPE_REG577				p.To.Reg = r578				return579			case 0x80:580				// 'SUBQ $-0x80, r' is shorter to encode than581				// and functionally equivalent to 'ADDQ $0x80, r'.582				asm := x86.ASUBL583				if v.Op == ssaop.OpAMD64ADDQconst {584					asm = x86.ASUBQ585				}586				p := s.Prog(asm)587				p.From.Type = obj.TYPE_CONST588				p.From.Offset = -0x80589				p.To.Type = obj.TYPE_REG590				p.To.Reg = r591				return592593			}594			p := s.Prog(v.Op.Asm())595			p.From.Type = obj.TYPE_CONST596			p.From.Offset = v.AuxInt597			p.To.Type = obj.TYPE_REG598			p.To.Reg = r599			return600		}601		var asm obj.As602		if v.Op == ssaop.OpAMD64ADDQconst {603			asm = x86.ALEAQ604		} else {605			asm = x86.ALEAL606		}607		p := s.Prog(asm)608		p.From.Type = obj.TYPE_MEM609		p.From.Reg = a610		p.From.Offset = v.AuxInt611		p.To.Type = obj.TYPE_REG612		p.To.Reg = r613614	case ssaop.OpAMD64CMOVQEQ, ssaop.OpAMD64CMOVLEQ, ssaop.OpAMD64CMOVWEQ,615		ssaop.OpAMD64CMOVQLT, ssaop.OpAMD64CMOVLLT, ssaop.OpAMD64CMOVWLT,616		ssaop.OpAMD64CMOVQNE, ssaop.OpAMD64CMOVLNE, ssaop.OpAMD64CMOVWNE,617		ssaop.OpAMD64CMOVQGT, ssaop.OpAMD64CMOVLGT, ssaop.OpAMD64CMOVWGT,618		ssaop.OpAMD64CMOVQLE, ssaop.OpAMD64CMOVLLE, ssaop.OpAMD64CMOVWLE,619		ssaop.OpAMD64CMOVQGE, ssaop.OpAMD64CMOVLGE, ssaop.OpAMD64CMOVWGE,620		ssaop.OpAMD64CMOVQHI, ssaop.OpAMD64CMOVLHI, ssaop.OpAMD64CMOVWHI,621		ssaop.OpAMD64CMOVQLS, ssaop.OpAMD64CMOVLLS, ssaop.OpAMD64CMOVWLS,622		ssaop.OpAMD64CMOVQCC, ssaop.OpAMD64CMOVLCC, ssaop.OpAMD64CMOVWCC,623		ssaop.OpAMD64CMOVQCS, ssaop.OpAMD64CMOVLCS, ssaop.OpAMD64CMOVWCS,624		ssaop.OpAMD64CMOVQGTF, ssaop.OpAMD64CMOVLGTF, ssaop.OpAMD64CMOVWGTF,625		ssaop.OpAMD64CMOVQGEF, ssaop.OpAMD64CMOVLGEF, ssaop.OpAMD64CMOVWGEF:626		p := s.Prog(v.Op.Asm())627		p.From.Type = obj.TYPE_REG628		p.From.Reg = v.Args[1].Reg()629		p.To.Type = obj.TYPE_REG630		p.To.Reg = v.Reg()631632	case ssaop.OpAMD64CMOVQNEF, ssaop.OpAMD64CMOVLNEF, ssaop.OpAMD64CMOVWNEF:633		// Flag condition: ^ZERO || PARITY634		// Generate:635		//   CMOV*NE  SRC,DST636		//   CMOV*PS  SRC,DST637		p := s.Prog(v.Op.Asm())638		p.From.Type = obj.TYPE_REG639		p.From.Reg = v.Args[1].Reg()640		p.To.Type = obj.TYPE_REG641		p.To.Reg = v.Reg()642		var q *obj.Prog643		if v.Op == ssaop.OpAMD64CMOVQNEF {644			q = s.Prog(x86.ACMOVQPS)645		} else if v.Op == ssaop.OpAMD64CMOVLNEF {646			q = s.Prog(x86.ACMOVLPS)647		} else {648			q = s.Prog(x86.ACMOVWPS)649		}650		q.From.Type = obj.TYPE_REG651		q.From.Reg = v.Args[1].Reg()652		q.To.Type = obj.TYPE_REG653		q.To.Reg = v.Reg()654655	case ssaop.OpAMD64CMOVQEQF, ssaop.OpAMD64CMOVLEQF, ssaop.OpAMD64CMOVWEQF:656		// Flag condition: ZERO && !PARITY657		// Generate:658		//   MOV      SRC,TMP659		//   CMOV*NE  DST,TMP660		//   CMOV*PC  TMP,DST661		//662		// TODO(rasky): we could generate:663		//   CMOV*NE  DST,SRC664		//   CMOV*PC  SRC,DST665		// But this requires a way for regalloc to know that SRC might be666		// clobbered by this instruction.667		t := v.RegTmp()668		opregreg(s, moveByRegsWidth(t, v.Args[1].Reg(), v.Type.Size()), t, v.Args[1].Reg())669670		p := s.Prog(v.Op.Asm())671		p.From.Type = obj.TYPE_REG672		p.From.Reg = v.Reg()673		p.To.Type = obj.TYPE_REG674		p.To.Reg = t675		var q *obj.Prog676		if v.Op == ssaop.OpAMD64CMOVQEQF {677			q = s.Prog(x86.ACMOVQPC)678		} else if v.Op == ssaop.OpAMD64CMOVLEQF {679			q = s.Prog(x86.ACMOVLPC)680		} else {681			q = s.Prog(x86.ACMOVWPC)682		}683		q.From.Type = obj.TYPE_REG684		q.From.Reg = t685		q.To.Type = obj.TYPE_REG686		q.To.Reg = v.Reg()687688	case ssaop.OpAMD64MULQconst, ssaop.OpAMD64MULLconst:689		r := v.Reg()690		p := s.Prog(v.Op.Asm())691		p.From.Type = obj.TYPE_CONST692		p.From.Offset = v.AuxInt693		p.To.Type = obj.TYPE_REG694		p.To.Reg = r695		p.AddRestSourceReg(v.Args[0].Reg())696697	case ssaop.OpAMD64ANDQconst:698		asm := v.Op.Asm()699		// If the constant is positive and fits into 32 bits, use ANDL.700		// This saves a few bytes of encoding.701		if 0 <= v.AuxInt && v.AuxInt <= (1<<32-1) {702			asm = x86.AANDL703		}704		p := s.Prog(asm)705		p.From.Type = obj.TYPE_CONST706		p.From.Offset = v.AuxInt707		p.To.Type = obj.TYPE_REG708		p.To.Reg = v.Reg()709710	case ssaop.OpAMD64SUBQconst, ssaop.OpAMD64SUBLconst,711		ssaop.OpAMD64ANDLconst,712		ssaop.OpAMD64ORQconst, ssaop.OpAMD64ORLconst,713		ssaop.OpAMD64XORQconst, ssaop.OpAMD64XORLconst:714		p := s.Prog(v.Op.Asm())715		p.From.Type = obj.TYPE_CONST716		p.From.Offset = v.AuxInt717		p.To.Type = obj.TYPE_REG718		p.To.Reg = v.Reg()719720	case ssaop.OpAMD64SHLQconst, ssaop.OpAMD64SHLLconst,721		ssaop.OpAMD64SHRQconst, ssaop.OpAMD64SHRLconst, ssaop.OpAMD64SHRWconst, ssaop.OpAMD64SHRBconst,722		ssaop.OpAMD64SARQconst, ssaop.OpAMD64SARLconst, ssaop.OpAMD64SARWconst, ssaop.OpAMD64SARBconst,723		ssaop.OpAMD64ROLQconst, ssaop.OpAMD64ROLLconst, ssaop.OpAMD64ROLWconst, ssaop.OpAMD64ROLBconst:724		var maxShift int64725		switch v.Op {726		case ssaop.OpAMD64SHLQconst, ssaop.OpAMD64SHRQconst, ssaop.OpAMD64SARQconst, ssaop.OpAMD64ROLQconst:727			maxShift = 63728		case ssaop.OpAMD64SHLLconst, ssaop.OpAMD64SHRLconst, ssaop.OpAMD64SARLconst, ssaop.OpAMD64ROLLconst:729			maxShift = 31730		case ssaop.OpAMD64SHRWconst, ssaop.OpAMD64SARWconst, ssaop.OpAMD64ROLWconst:731			maxShift = 15732		case ssaop.OpAMD64SHRBconst, ssaop.OpAMD64SARBconst, ssaop.OpAMD64ROLBconst:733			maxShift = 7734		default:735			panic("unreachable")736		}737		if v.AuxInt < 0 || v.AuxInt > maxShift {738			v.Fatalf("shift amount out of range [0,%d]: %d", maxShift, v.AuxInt)739		}740		p := s.Prog(v.Op.Asm())741		p.From.Type = obj.TYPE_CONST742		p.From.Offset = v.AuxInt743		p.To.Type = obj.TYPE_REG744		p.To.Reg = v.Reg()745	case ssaop.OpAMD64SBBQcarrymask, ssaop.OpAMD64SBBLcarrymask:746		r := v.Reg()747		p := s.Prog(v.Op.Asm())748		p.From.Type = obj.TYPE_REG749		p.From.Reg = r750		p.To.Type = obj.TYPE_REG751		p.To.Reg = r752	case ssaop.OpAMD64LEAQ1, ssaop.OpAMD64LEAQ2, ssaop.OpAMD64LEAQ4, ssaop.OpAMD64LEAQ8,753		ssaop.OpAMD64LEAL1, ssaop.OpAMD64LEAL2, ssaop.OpAMD64LEAL4, ssaop.OpAMD64LEAL8,754		ssaop.OpAMD64LEAW1, ssaop.OpAMD64LEAW2, ssaop.OpAMD64LEAW4, ssaop.OpAMD64LEAW8:755		p := s.Prog(v.Op.Asm())756		memIdx(&p.From, v)757		ssagen.AddAux(&p.From, v)758		p.To.Type = obj.TYPE_REG759		p.To.Reg = v.Reg()760	case ssaop.OpAMD64LEAQ, ssaop.OpAMD64LEAL, ssaop.OpAMD64LEAW:761		p := s.Prog(v.Op.Asm())762		p.From.Type = obj.TYPE_MEM763		p.From.Reg = v.Args[0].Reg()764		ssagen.AddAux(&p.From, v)765		p.To.Type = obj.TYPE_REG766		p.To.Reg = v.Reg()767	case ssaop.OpAMD64CMPQ, ssaop.OpAMD64CMPL, ssaop.OpAMD64CMPW, ssaop.OpAMD64CMPB,768		ssaop.OpAMD64TESTQ, ssaop.OpAMD64TESTL, ssaop.OpAMD64TESTW, ssaop.OpAMD64TESTB,769		ssaop.OpAMD64BTL, ssaop.OpAMD64BTQ:770		opregreg(s, v.Op.Asm(), v.Args[1].Reg(), v.Args[0].Reg())771	case ssaop.OpAMD64UCOMISS, ssaop.OpAMD64UCOMISD:772		// Go assembler has swapped operands for UCOMISx relative to CMP,773		// must account for that right here.774		opregreg(s, v.Op.Asm(), v.Args[0].Reg(), v.Args[1].Reg())775	case ssaop.OpAMD64CMPQconst, ssaop.OpAMD64CMPLconst, ssaop.OpAMD64CMPWconst, ssaop.OpAMD64CMPBconst:776		p := s.Prog(v.Op.Asm())777		p.From.Type = obj.TYPE_REG778		p.From.Reg = v.Args[0].Reg()779		p.To.Type = obj.TYPE_CONST780		p.To.Offset = v.AuxInt781	case ssaop.OpAMD64BTLconst, ssaop.OpAMD64BTQconst,782		ssaop.OpAMD64TESTQconst, ssaop.OpAMD64TESTLconst, ssaop.OpAMD64TESTWconst, ssaop.OpAMD64TESTBconst,783		ssaop.OpAMD64BTSQconst,784		ssaop.OpAMD64BTCQconst,785		ssaop.OpAMD64BTRQconst:786		op := v.Op787		if op == ssaop.OpAMD64BTQconst && v.AuxInt < 32 {788			// Emit 32-bit version because it's shorter789			op = ssaop.OpAMD64BTLconst790		}791		p := s.Prog(op.Asm())792		p.From.Type = obj.TYPE_CONST793		p.From.Offset = v.AuxInt794		p.To.Type = obj.TYPE_REG795		p.To.Reg = v.Args[0].Reg()796	case ssaop.OpAMD64CMPQload, ssaop.OpAMD64CMPLload, ssaop.OpAMD64CMPWload, ssaop.OpAMD64CMPBload:797		p := s.Prog(v.Op.Asm())798		p.From.Type = obj.TYPE_MEM799		p.From.Reg = v.Args[0].Reg()800		ssagen.AddAux(&p.From, v)801		p.To.Type = obj.TYPE_REG802		p.To.Reg = v.Args[1].Reg()803	case ssaop.OpAMD64CMPQconstload, ssaop.OpAMD64CMPLconstload, ssaop.OpAMD64CMPWconstload, ssaop.OpAMD64CMPBconstload:804		sc := v.AuxValAndOff()805		p := s.Prog(v.Op.Asm())806		p.From.Type = obj.TYPE_MEM807		p.From.Reg = v.Args[0].Reg()808		ssagen.AddAux2(&p.From, v, sc.Off64())809		p.To.Type = obj.TYPE_CONST810		p.To.Offset = sc.Val64()811	case ssaop.OpAMD64CMPQloadidx8, ssaop.OpAMD64CMPQloadidx1, ssaop.OpAMD64CMPLloadidx4, ssaop.OpAMD64CMPLloadidx1, ssaop.OpAMD64CMPWloadidx2, ssaop.OpAMD64CMPWloadidx1, ssaop.OpAMD64CMPBloadidx1:812		p := s.Prog(v.Op.Asm())813		memIdx(&p.From, v)814		ssagen.AddAux(&p.From, v)815		p.To.Type = obj.TYPE_REG816		p.To.Reg = v.Args[2].Reg()817	case ssaop.OpAMD64CMPQconstloadidx8, ssaop.OpAMD64CMPQconstloadidx1, ssaop.OpAMD64CMPLconstloadidx4, ssaop.OpAMD64CMPLconstloadidx1, ssaop.OpAMD64CMPWconstloadidx2, ssaop.OpAMD64CMPWconstloadidx1, ssaop.OpAMD64CMPBconstloadidx1:818		sc := v.AuxValAndOff()819		p := s.Prog(v.Op.Asm())820		memIdx(&p.From, v)821		ssagen.AddAux2(&p.From, v, sc.Off64())822		p.To.Type = obj.TYPE_CONST823		p.To.Offset = sc.Val64()824	case ssaop.OpAMD64MOVLconst, ssaop.OpAMD64MOVQconst:825		x := v.Reg()826827		// If flags aren't live (indicated by v.Aux == nil),828		// then we can rewrite MOV $0, AX into XOR AX, AX.829		if v.AuxInt == 0 && v.Aux == nil {830			opregreg(s, x86.AXORL, x, x)831			break832		}833834		asm := v.Op.Asm()835		// Use MOVL to move a small constant into a register836		// when the constant is positive and fits into 32 bits.837		if 0 <= v.AuxInt && v.AuxInt <= (1<<32-1) {838			// The upper 32bit are zeroed automatically when using MOVL.839			asm = x86.AMOVL840		}841		p := s.Prog(asm)842		p.From.Type = obj.TYPE_CONST843		p.From.Offset = v.AuxInt844		p.To.Type = obj.TYPE_REG845		p.To.Reg = x846847	case ssaop.OpAMD64MOVSSconst, ssaop.OpAMD64MOVSDconst:848		x := v.Reg()849		if !isFPReg(x) && v.AuxInt == 0 && v.Aux == nil {850			opregreg(s, x86.AXORL, x, x)851			break852		}853		p := s.Prog(storeByRegWidth(x, v.Type.Size()))854		p.From.Type = obj.TYPE_FCONST855		p.From.Val = math.Float64frombits(uint64(v.AuxInt))856		p.To.Type = obj.TYPE_REG857		p.To.Reg = x858	case ssaop.OpAMD64MOVQload, ssaop.OpAMD64MOVLload, ssaop.OpAMD64MOVWload, ssaop.OpAMD64MOVBload, ssaop.OpAMD64MOVOload,859		ssaop.OpAMD64MOVSSload, ssaop.OpAMD64MOVSDload, ssaop.OpAMD64MOVBQSXload, ssaop.OpAMD64MOVWQSXload, ssaop.OpAMD64MOVLQSXload,860		ssaop.OpAMD64MOVBEQload, ssaop.OpAMD64MOVBELload:861		p := s.Prog(v.Op.Asm())862		p.From.Type = obj.TYPE_MEM863		p.From.Reg = v.Args[0].Reg()864		ssagen.AddAux(&p.From, v)865		p.To.Type = obj.TYPE_REG866		p.To.Reg = v.Reg()867	case ssaop.OpAMD64MOVBloadidx1, ssaop.OpAMD64MOVWloadidx1, ssaop.OpAMD64MOVLloadidx1, ssaop.OpAMD64MOVQloadidx1, ssaop.OpAMD64MOVSSloadidx1, ssaop.OpAMD64MOVSDloadidx1,868		ssaop.OpAMD64MOVQloadidx8, ssaop.OpAMD64MOVSDloadidx8, ssaop.OpAMD64MOVLloadidx8, ssaop.OpAMD64MOVLloadidx4, ssaop.OpAMD64MOVSSloadidx4, ssaop.OpAMD64MOVWloadidx2,869		ssaop.OpAMD64MOVBELloadidx1, ssaop.OpAMD64MOVBELloadidx4, ssaop.OpAMD64MOVBELloadidx8, ssaop.OpAMD64MOVBEQloadidx1, ssaop.OpAMD64MOVBEQloadidx8:870		p := s.Prog(v.Op.Asm())871		memIdx(&p.From, v)872		ssagen.AddAux(&p.From, v)873		p.To.Type = obj.TYPE_REG874		p.To.Reg = v.Reg()875	case ssaop.OpAMD64MOVQstore, ssaop.OpAMD64MOVSSstore, ssaop.OpAMD64MOVSDstore, ssaop.OpAMD64MOVLstore, ssaop.OpAMD64MOVWstore, ssaop.OpAMD64MOVBstore, ssaop.OpAMD64MOVOstore,876		ssaop.OpAMD64ADDQmodify, ssaop.OpAMD64SUBQmodify, ssaop.OpAMD64ANDQmodify, ssaop.OpAMD64ORQmodify, ssaop.OpAMD64XORQmodify,877		ssaop.OpAMD64ADDLmodify, ssaop.OpAMD64SUBLmodify, ssaop.OpAMD64ANDLmodify, ssaop.OpAMD64ORLmodify, ssaop.OpAMD64XORLmodify,878		ssaop.OpAMD64MOVBEQstore, ssaop.OpAMD64MOVBELstore, ssaop.OpAMD64MOVBEWstore:879		p := s.Prog(v.Op.Asm())880		p.From.Type = obj.TYPE_REG881		p.From.Reg = v.Args[1].Reg()882		p.To.Type = obj.TYPE_MEM883		p.To.Reg = v.Args[0].Reg()884		ssagen.AddAux(&p.To, v)885	case ssaop.OpAMD64MOVBstoreidx1, ssaop.OpAMD64MOVWstoreidx1, ssaop.OpAMD64MOVLstoreidx1, ssaop.OpAMD64MOVQstoreidx1, ssaop.OpAMD64MOVSSstoreidx1, ssaop.OpAMD64MOVSDstoreidx1,886		ssaop.OpAMD64MOVQstoreidx8, ssaop.OpAMD64MOVSDstoreidx8, ssaop.OpAMD64MOVLstoreidx8, ssaop.OpAMD64MOVSSstoreidx4, ssaop.OpAMD64MOVLstoreidx4, ssaop.OpAMD64MOVWstoreidx2,887		ssaop.OpAMD64ADDLmodifyidx1, ssaop.OpAMD64ADDLmodifyidx4, ssaop.OpAMD64ADDLmodifyidx8, ssaop.OpAMD64ADDQmodifyidx1, ssaop.OpAMD64ADDQmodifyidx8,888		ssaop.OpAMD64SUBLmodifyidx1, ssaop.OpAMD64SUBLmodifyidx4, ssaop.OpAMD64SUBLmodifyidx8, ssaop.OpAMD64SUBQmodifyidx1, ssaop.OpAMD64SUBQmodifyidx8,889		ssaop.OpAMD64ANDLmodifyidx1, ssaop.OpAMD64ANDLmodifyidx4, ssaop.OpAMD64ANDLmodifyidx8, ssaop.OpAMD64ANDQmodifyidx1, ssaop.OpAMD64ANDQmodifyidx8,890		ssaop.OpAMD64ORLmodifyidx1, ssaop.OpAMD64ORLmodifyidx4, ssaop.OpAMD64ORLmodifyidx8, ssaop.OpAMD64ORQmodifyidx1, ssaop.OpAMD64ORQmodifyidx8,891		ssaop.OpAMD64XORLmodifyidx1, ssaop.OpAMD64XORLmodifyidx4, ssaop.OpAMD64XORLmodifyidx8, ssaop.OpAMD64XORQmodifyidx1, ssaop.OpAMD64XORQmodifyidx8,892		ssaop.OpAMD64MOVBEWstoreidx1, ssaop.OpAMD64MOVBEWstoreidx2, ssaop.OpAMD64MOVBELstoreidx1, ssaop.OpAMD64MOVBELstoreidx4, ssaop.OpAMD64MOVBELstoreidx8, ssaop.OpAMD64MOVBEQstoreidx1, ssaop.OpAMD64MOVBEQstoreidx8:893		p := s.Prog(v.Op.Asm())894		p.From.Type = obj.TYPE_REG895		p.From.Reg = v.Args[2].Reg()896		memIdx(&p.To, v)897		ssagen.AddAux(&p.To, v)898	case ssaop.OpAMD64ADDQconstmodify, ssaop.OpAMD64ADDLconstmodify,899		ssaop.OpAMD64ADDWconstmodify, ssaop.OpAMD64ADDBconstmodify:900		sc := v.AuxValAndOff()901		off := sc.Off64()902		val := sc.Val()903		if val == 1 || val == -1 {904			var asm obj.As905			switch v.Op {906			case ssaop.OpAMD64ADDQconstmodify:907				asm = x86.AINCQ908				if val == -1 {909					asm = x86.ADECQ910				}911			case ssaop.OpAMD64ADDLconstmodify:912				asm = x86.AINCL913				if val == -1 {914					asm = x86.ADECL915				}916			case ssaop.OpAMD64ADDWconstmodify:917				asm = x86.AINCW918				if val == -1 {919					asm = x86.ADECW920				}921			default:922				asm = x86.AINCB923				if val == -1 {924					asm = x86.ADECB925				}926			}927			p := s.Prog(asm)928			p.To.Type = obj.TYPE_MEM929			p.To.Reg = v.Args[0].Reg()930			ssagen.AddAux2(&p.To, v, off)931			break932		}933		fallthrough934	case ssaop.OpAMD64ANDQconstmodify, ssaop.OpAMD64ANDLconstmodify, ssaop.OpAMD64ORQconstmodify, ssaop.OpAMD64ORLconstmodify,935		ssaop.OpAMD64XORQconstmodify, ssaop.OpAMD64XORLconstmodify,936		ssaop.OpAMD64ANDWconstmodify, ssaop.OpAMD64ANDBconstmodify, ssaop.OpAMD64ORWconstmodify, ssaop.OpAMD64ORBconstmodify,937		ssaop.OpAMD64XORWconstmodify, ssaop.OpAMD64XORBconstmodify,938		ssaop.OpAMD64BTSQconstmodify, ssaop.OpAMD64BTRQconstmodify, ssaop.OpAMD64BTCQconstmodify:939		sc := v.AuxValAndOff()940		off := sc.Off64()941		val := sc.Val64()942		p := s.Prog(v.Op.Asm())943		p.From.Type = obj.TYPE_CONST944		p.From.Offset = val945		p.To.Type = obj.TYPE_MEM946		p.To.Reg = v.Args[0].Reg()947		ssagen.AddAux2(&p.To, v, off)948949	case ssaop.OpAMD64MOVQstoreconst, ssaop.OpAMD64MOVLstoreconst, ssaop.OpAMD64MOVWstoreconst, ssaop.OpAMD64MOVBstoreconst:950		sc := v.AuxValAndOff()951		p := s.Prog(v.Op.Asm())952		if sc.Val() == 0 && s.ABI == obj.ABIInternal && buildcfg.GOOS != "plan9" && (v.Op == ssaop.OpAMD64MOVQstoreconst || v.Op == ssaop.OpAMD64MOVLstoreconst) {953			p.From.Type = obj.TYPE_REG954			p.From.Reg = x86.REG_X15955		} else {956			p.From.Type = obj.TYPE_CONST957			p.From.Offset = sc.Val64()958		}959		p.To.Type = obj.TYPE_MEM960		p.To.Reg = v.Args[0].Reg()961		ssagen.AddAux2(&p.To, v, sc.Off64())962	case ssaop.OpAMD64MOVOstoreconst:963		sc := v.AuxValAndOff()964		if sc.Val() != 0 {965			v.Fatalf("MOVO for non zero constants not implemented: %s", v.LongString())966		}967968		if s.ABI != obj.ABIInternal {969			// zero X15 manually970			opregreg(s, x86.AXORPS, x86.REG_X15, x86.REG_X15)971		}972		p := s.Prog(v.Op.Asm())973		p.From.Type = obj.TYPE_REG974		p.From.Reg = x86.REG_X15975		p.To.Type = obj.TYPE_MEM976		p.To.Reg = v.Args[0].Reg()977		ssagen.AddAux2(&p.To, v, sc.Off64())978979	case ssaop.OpAMD64MOVQstoreconstidx1, ssaop.OpAMD64MOVQstoreconstidx8, ssaop.OpAMD64MOVLstoreconstidx1, ssaop.OpAMD64MOVLstoreconstidx4, ssaop.OpAMD64MOVWstoreconstidx1, ssaop.OpAMD64MOVWstoreconstidx2, ssaop.OpAMD64MOVBstoreconstidx1,980		ssaop.OpAMD64ADDLconstmodifyidx1, ssaop.OpAMD64ADDLconstmodifyidx4, ssaop.OpAMD64ADDLconstmodifyidx8, ssaop.OpAMD64ADDQconstmodifyidx1, ssaop.OpAMD64ADDQconstmodifyidx8,981		ssaop.OpAMD64ANDLconstmodifyidx1, ssaop.OpAMD64ANDLconstmodifyidx4, ssaop.OpAMD64ANDLconstmodifyidx8, ssaop.OpAMD64ANDQconstmodifyidx1, ssaop.OpAMD64ANDQconstmodifyidx8,982		ssaop.OpAMD64ORLconstmodifyidx1, ssaop.OpAMD64ORLconstmodifyidx4, ssaop.OpAMD64ORLconstmodifyidx8, ssaop.OpAMD64ORQconstmodifyidx1, ssaop.OpAMD64ORQconstmodifyidx8,983		ssaop.OpAMD64XORLconstmodifyidx1, ssaop.OpAMD64XORLconstmodifyidx4, ssaop.OpAMD64XORLconstmodifyidx8, ssaop.OpAMD64XORQconstmodifyidx1, ssaop.OpAMD64XORQconstmodifyidx8,984		ssaop.OpAMD64ADDWconstmodifyidx1, ssaop.OpAMD64ADDWconstmodifyidx2, ssaop.OpAMD64ADDBconstmodifyidx1,985		ssaop.OpAMD64ANDWconstmodifyidx1, ssaop.OpAMD64ANDWconstmodifyidx2, ssaop.OpAMD64ANDBconstmodifyidx1,986		ssaop.OpAMD64ORWconstmodifyidx1, ssaop.OpAMD64ORWconstmodifyidx2, ssaop.OpAMD64ORBconstmodifyidx1,987		ssaop.OpAMD64XORWconstmodifyidx1, ssaop.OpAMD64XORWconstmodifyidx2, ssaop.OpAMD64XORBconstmodifyidx1:988		p := s.Prog(v.Op.Asm())989		p.From.Type = obj.TYPE_CONST990		sc := v.AuxValAndOff()991		p.From.Offset = sc.Val64()992		if sc.Val() == 0 && s.ABI == obj.ABIInternal && buildcfg.GOOS != "plan9" {993			switch v.Op {994			case ssaop.OpAMD64MOVQstoreconstidx1, ssaop.OpAMD64MOVQstoreconstidx8,995				ssaop.OpAMD64MOVLstoreconstidx1, ssaop.OpAMD64MOVLstoreconstidx4:996				p.From.Type = obj.TYPE_REG997				p.From.Reg = x86.REG_X15998			}999		}1000		switch {1001		case p.As == x86.AADDQ && p.From.Offset == 1:1002			p.As = x86.AINCQ1003			p.From.Type = obj.TYPE_NONE1004		case p.As == x86.AADDQ && p.From.Offset == -1:1005			p.As = x86.ADECQ1006			p.From.Type = obj.TYPE_NONE1007		case p.As == x86.AADDL && p.From.Offset == 1:1008			p.As = x86.AINCL1009			p.From.Type = obj.TYPE_NONE1010		case p.As == x86.AADDL && p.From.Offset == -1:1011			p.As = x86.ADECL1012			p.From.Type = obj.TYPE_NONE1013		case p.As == x86.AADDW && p.From.Offset == 1:1014			p.As = x86.AINCW1015			p.From.Type = obj.TYPE_NONE1016		case p.As == x86.AADDW && p.From.Offset == -1:1017			p.As = x86.ADECW1018			p.From.Type = obj.TYPE_NONE1019		case p.As == x86.AADDB && p.From.Offset == 1:1020			p.As = x86.AINCB1021			p.From.Type = obj.TYPE_NONE1022		case p.As == x86.AADDB && p.From.Offset == -1:1023			p.As = x86.ADECB1024			p.From.Type = obj.TYPE_NONE1025		}1026		memIdx(&p.To, v)1027		ssagen.AddAux2(&p.To, v, sc.Off64())1028	case ssaop.OpAMD64MOVLQSX, ssaop.OpAMD64MOVWQSX, ssaop.OpAMD64MOVBQSX, ssaop.OpAMD64MOVLQZX, ssaop.OpAMD64MOVWQZX, ssaop.OpAMD64MOVBQZX,1029		ssaop.OpAMD64CVTTSS2SL, ssaop.OpAMD64CVTTSD2SL, ssaop.OpAMD64CVTTSS2SQ, ssaop.OpAMD64CVTTSD2SQ,1030		ssaop.OpAMD64CVTSS2SD, ssaop.OpAMD64CVTSD2SS, ssaop.OpAMD64VPBROADCASTB, ssaop.OpAMD64PMOVMSKB:1031		opregreg(s, v.Op.Asm(), v.Reg(), v.Args[0].Reg())1032	case ssaop.OpAMD64CVTSL2SD, ssaop.OpAMD64CVTSQ2SD, ssaop.OpAMD64CVTSQ2SS, ssaop.OpAMD64CVTSL2SS:1033		r := v.Reg()1034		// Break false dependency on destination register.1035		opregreg(s, x86.AXORPS, r, r)1036		opregreg(s, v.Op.Asm(), r, v.Args[0].Reg())1037	case ssaop.OpAMD64MOVQi2f, ssaop.OpAMD64MOVQf2i, ssaop.OpAMD64MOVLi2f, ssaop.OpAMD64MOVLf2i:1038		var p *obj.Prog1039		switch v.Op {1040		case ssaop.OpAMD64MOVQi2f, ssaop.OpAMD64MOVQf2i:1041			p = s.Prog(x86.AMOVQ)1042		case ssaop.OpAMD64MOVLi2f, ssaop.OpAMD64MOVLf2i:1043			p = s.Prog(x86.AMOVL)1044		}1045		p.From.Type = obj.TYPE_REG1046		p.From.Reg = v.Args[0].Reg()1047		p.To.Type = obj.TYPE_REG1048		p.To.Reg = v.Reg()1049	case ssaop.OpAMD64ADDQload, ssaop.OpAMD64ADDLload, ssaop.OpAMD64SUBQload, ssaop.OpAMD64SUBLload,1050		ssaop.OpAMD64ANDQload, ssaop.OpAMD64ANDLload, ssaop.OpAMD64ORQload, ssaop.OpAMD64ORLload,1051		ssaop.OpAMD64XORQload, ssaop.OpAMD64XORLload, ssaop.OpAMD64ADDSDload, ssaop.OpAMD64ADDSSload,1052		ssaop.OpAMD64SUBSDload, ssaop.OpAMD64SUBSSload, ssaop.OpAMD64MULSDload, ssaop.OpAMD64MULSSload,1053		ssaop.OpAMD64DIVSDload, ssaop.OpAMD64DIVSSload:1054		p := s.Prog(v.Op.Asm())1055		p.From.Type = obj.TYPE_MEM1056		p.From.Reg = v.Args[1].Reg()1057		ssagen.AddAux(&p.From, v)1058		p.To.Type = obj.TYPE_REG1059		p.To.Reg = v.Reg()1060	case ssaop.OpAMD64ADDLloadidx1, ssaop.OpAMD64ADDLloadidx4, ssaop.OpAMD64ADDLloadidx8, ssaop.OpAMD64ADDQloadidx1, ssaop.OpAMD64ADDQloadidx8,1061		ssaop.OpAMD64SUBLloadidx1, ssaop.OpAMD64SUBLloadidx4, ssaop.OpAMD64SUBLloadidx8, ssaop.OpAMD64SUBQloadidx1, ssaop.OpAMD64SUBQloadidx8,1062		ssaop.OpAMD64ANDLloadidx1, ssaop.OpAMD64ANDLloadidx4, ssaop.OpAMD64ANDLloadidx8, ssaop.OpAMD64ANDQloadidx1, ssaop.OpAMD64ANDQloadidx8,1063		ssaop.OpAMD64ORLloadidx1, ssaop.OpAMD64ORLloadidx4, ssaop.OpAMD64ORLloadidx8, ssaop.OpAMD64ORQloadidx1, ssaop.OpAMD64ORQloadidx8,1064		ssaop.OpAMD64XORLloadidx1, ssaop.OpAMD64XORLloadidx4, ssaop.OpAMD64XORLloadidx8, ssaop.OpAMD64XORQloadidx1, ssaop.OpAMD64XORQloadidx8,1065		ssaop.OpAMD64ADDSSloadidx1, ssaop.OpAMD64ADDSSloadidx4, ssaop.OpAMD64ADDSDloadidx1, ssaop.OpAMD64ADDSDloadidx8,1066		ssaop.OpAMD64SUBSSloadidx1, ssaop.OpAMD64SUBSSloadidx4, ssaop.OpAMD64SUBSDloadidx1, ssaop.OpAMD64SUBSDloadidx8,1067		ssaop.OpAMD64MULSSloadidx1, ssaop.OpAMD64MULSSloadidx4, ssaop.OpAMD64MULSDloadidx1, ssaop.OpAMD64MULSDloadidx8,1068		ssaop.OpAMD64DIVSSloadidx1, ssaop.OpAMD64DIVSSloadidx4, ssaop.OpAMD64DIVSDloadidx1, ssaop.OpAMD64DIVSDloadidx8:1069		p := s.Prog(v.Op.Asm())10701071		r, i := v.Args[1].Reg(), v.Args[2].Reg()1072		p.From.Type = obj.TYPE_MEM1073		p.From.Scale = v.Op.Scale()1074		if p.From.Scale == 1 && i == x86.REG_SP {1075			r, i = i, r1076		}1077		p.From.Reg = r1078		p.From.Index = i10791080		ssagen.AddAux(&p.From, v)1081		p.To.Type = obj.TYPE_REG1082		p.To.Reg = v.Reg()10831084	case ssaop.OpAMD64LoweredZero:1085		if s.ABI != obj.ABIInternal {1086			// zero X15 manually1087			opregreg(s, x86.AXORPS, x86.REG_X15, x86.REG_X15)1088		}1089		ptrReg := v.Args[0].Reg()1090		n := v.AuxInt1091		if n < 16 {1092			v.Fatalf("Zero too small %d", n)1093		}1094		zero16 := func(off int64) {1095			zero16(s, ptrReg, off)1096		}10971098		// Generate zeroing instructions.1099		var off int641100		for n >= 16 {1101			zero16(off)1102			off += 161103			n -= 161104		}1105		if n != 0 {1106			// use partially overlapped write.1107			// TODO: n <= 8, use smaller write?1108			zero16(off + n - 16)1109		}11101111	case ssaop.OpAMD64LoweredZeroLoop:1112		if s.ABI != obj.ABIInternal {1113			// zero X15 manually1114			opregreg(s, x86.AXORPS, x86.REG_X15, x86.REG_X15)1115		}1116		ptrReg := v.Args[0].Reg()1117		countReg := v.RegTmp()1118		n := v.AuxInt1119		loopSize := int64(64)1120		if n < 3*loopSize {1121			// - a loop count of 0 won't work.1122			// - a loop count of 1 is useless.1123			// - a loop count of 2 is a code size ~tie1124			//     4 instructions to implement the loop1125			//     4 instructions in the loop body1126			//   vs1127			//     8 instructions in the straightline code1128			//   Might as well use straightline code.1129			v.Fatalf("ZeroLoop size too small %d", n)1130		}1131		zero16 := func(off int64) {1132			zero16(s, ptrReg, off)1133		}11341135		// Put iteration count in a register.1136		//   MOVL    $n, countReg1137		p := s.Prog(x86.AMOVL)1138		p.From.Type = obj.TYPE_CONST1139		p.From.Offset = n / loopSize1140		p.To.Type = obj.TYPE_REG1141		p.To.Reg = countReg1142		cntInit := p11431144		// Zero loopSize bytes starting at ptrReg.1145		for i := range loopSize / 16 {1146			zero16(i * 16)1147		}1148		//   ADDQ    $loopSize, ptrReg1149		p = s.Prog(x86.AADDQ)1150		p.From.Type = obj.TYPE_CONST1151		p.From.Offset = loopSize1152		p.To.Type = obj.TYPE_REG1153		p.To.Reg = ptrReg1154		//   DECL    countReg1155		p = s.Prog(x86.ADECL)1156		p.To.Type = obj.TYPE_REG1157		p.To.Reg = countReg1158		// Jump to first instruction in loop if we're not done yet.1159		//   JNE     head1160		p = s.Prog(x86.AJNE)1161		p.To.Type = obj.TYPE_BRANCH1162		p.To.SetTarget(cntInit.Link)11631164		// Multiples of the loop size are now done.1165		n %= loopSize11661167		// Write any fractional portion.1168		var off int641169		for n >= 16 {1170			zero16(off)1171			off += 161172			n -= 161173		}1174		if n != 0 {1175			// Use partially-overlapping write.1176			// TODO: n <= 8, use smaller write?1177			zero16(off + n - 16)1178		}11791180	case ssaop.OpAMD64LoweredMove:1181		dstReg := v.Args[0].Reg()1182		srcReg := v.Args[1].Reg()1183		if dstReg == srcReg {1184			break1185		}1186		tmpReg := int16(x86.REG_X14)1187		n := v.AuxInt1188		if n < 16 {1189			v.Fatalf("Move too small %d", n)1190		}1191		// move 16 bytes from srcReg+off to dstReg+off.1192		move16 := func(off int64) {1193			move16(s, srcReg, dstReg, tmpReg, off)1194		}11951196		// Generate copying instructions.1197		var off int641198		for n >= 16 {1199			move16(off)1200			off += 161201			n -= 161202		}1203		if n != 0 {1204			// use partially overlapped read/write.1205			// TODO: use smaller operations when we can?1206			move16(off + n - 16)1207		}12081209	case ssaop.OpAMD64LoweredMoveLoop:1210		dstReg := v.Args[0].Reg()1211		srcReg := v.Args[1].Reg()1212		if dstReg == srcReg {1213			break1214		}1215		countReg := v.RegTmp()1216		tmpReg := int16(x86.REG_X14)1217		n := v.AuxInt1218		loopSize := int64(64)1219		if n < 3*loopSize {1220			// - a loop count of 0 won't work.1221			// - a loop count of 1 is useless.1222			// - a loop count of 2 is a code size ~tie1223			//     4 instructions to implement the loop1224			//     4 instructions in the loop body1225			//   vs1226			//     8 instructions in the straightline code1227			//   Might as well use straightline code.1228			v.Fatalf("ZeroLoop size too small %d", n)1229		}1230		// move 16 bytes from srcReg+off to dstReg+off.1231		move16 := func(off int64) {1232			move16(s, srcReg, dstReg, tmpReg, off)1233		}12341235		// Put iteration count in a register.1236		//   MOVL    $n, countReg1237		p := s.Prog(x86.AMOVL)1238		p.From.Type = obj.TYPE_CONST1239		p.From.Offset = n / loopSize1240		p.To.Type = obj.TYPE_REG1241		p.To.Reg = countReg1242		cntInit := p12431244		// Copy loopSize bytes starting at srcReg to dstReg.1245		for i := range loopSize / 16 {1246			move16(i * 16)1247		}1248		//   ADDQ    $loopSize, srcReg1249		p = s.Prog(x86.AADDQ)1250		p.From.Type = obj.TYPE_CONST1251		p.From.Offset = loopSize1252		p.To.Type = obj.TYPE_REG1253		p.To.Reg = srcReg1254		//   ADDQ    $loopSize, dstReg1255		p = s.Prog(x86.AADDQ)1256		p.From.Type = obj.TYPE_CONST1257		p.From.Offset = loopSize1258		p.To.Type = obj.TYPE_REG1259		p.To.Reg = dstReg1260		//   DECL    countReg1261		p = s.Prog(x86.ADECL)1262		p.To.Type = obj.TYPE_REG1263		p.To.Reg = countReg1264		// Jump to loop header if we're not done yet.1265		//   JNE     head1266		p = s.Prog(x86.AJNE)1267		p.To.Type = obj.TYPE_BRANCH1268		p.To.SetTarget(cntInit.Link)12691270		// Multiples of the loop size are now done.1271		n %= loopSize12721273		// Copy any fractional portion.1274		var off int641275		for n >= 16 {1276			move16(off)1277			off += 161278			n -= 161279		}1280		if n != 0 {1281			// Use partially-overlapping copy.1282			move16(off + n - 16)1283		}12841285	case ssaop.OpCopy: // TODO: use MOVQreg for reg->reg copies instead of OpCopy?1286		if v.Type.IsMemory() {1287			return1288		}1289		arg := v.Args[0]1290		x := arg.Reg()1291		y := v.Reg()1292		if v.Type.IsSIMD() {1293			x = simdOrMaskReg(arg)1294			y = simdOrMaskReg(v)1295		}1296		if x != y {1297			width := v.Type.Size()1298			if width == 8 && isGPReg(y) && ssa.ZeroUpper32Bits(arg) {1299				// The source was naturally zext-ed from 32 to 64 bits,1300				// but we are asked to do a full 64-bit copy.1301				// Save the REX prefix byte in I-CACHE by using a 32-bit move,1302				// since it zeroes the upper 32 bits anyway.1303				width = 41304			}1305			opregreg(s, moveByRegsWidth(y, x, width), y, x)1306		}1307	case ssaop.OpLoadReg:1308		if v.Type.IsFlags() {1309			v.Fatalf("load flags not implemented: %v", v.LongString())1310			return1311		}1312		r := v.Reg()1313		p := s.Prog(loadByRegWidth(r, v.Type.Size()))1314		ssagen.AddrAuto(&p.From, v.Args[0])1315		p.To.Type = obj.TYPE_REG1316		if v.Type.IsSIMD() {1317			r = simdOrMaskReg(v)1318		}1319		p.To.Reg = r13201321	case ssaop.OpStoreReg:1322		if v.Type.IsFlags() {1323			v.Fatalf("store flags not implemented: %v", v.LongString())1324			return1325		}1326		r := v.Args[0].Reg()1327		if v.Type.IsSIMD() {1328			r = simdOrMaskReg(v.Args[0])1329		}1330		p := s.Prog(storeByRegWidth(r, v.Type.Size()))1331		p.From.Type = obj.TYPE_REG1332		p.From.Reg = r1333		ssagen.AddrAuto(&p.To, v)1334	case ssaop.OpAMD64LoweredHasCPUFeature:1335		// If this load changes width, update zeroUpperBits in AMD64Ops.go.1336		p := s.Prog(x86.AMOVBLZX)1337		p.From.Type = obj.TYPE_MEM1338		ssagen.AddAux(&p.From, v)1339		p.To.Type = obj.TYPE_REG1340		p.To.Reg = v.Reg()1341	case ssaop.OpArgIntReg, ssaop.OpArgFloatReg:1342		// The assembler needs to wrap the entry safepoint/stack growth code with spill/unspill1343		// The loop only runs once.1344		for _, ap := range v.Block.Func.RegArgs {1345			// Pass the spill/unspill information along to the assembler, offset by size of return PC pushed on stack.1346			addr := ssagen.SpillSlotAddr(ap, x86.REG_SP, v.Block.Func.Config.PtrSize)1347			reg := ap.Reg1348			t := ap.Type1349			sz := t.Size()1350			if t.IsSIMD() {1351				reg = simdRegBySize(reg, sz)1352			}1353			s.FuncInfo().AddSpill(1354				obj.RegSpill{Reg: reg, Addr: addr, Unspill: loadByRegWidth(reg, sz), Spill: storeByRegWidth(reg, sz)})1355		}1356		v.Block.Func.RegArgs = nil1357		ssagen.CheckArgReg(v)1358	case ssaop.OpAMD64LoweredGetClosurePtr:1359		// Closure pointer is DX.1360		ssagen.CheckLoweredGetClosurePtr(v)1361	case ssaop.OpAMD64LoweredGetG:1362		if s.ABI == obj.ABIInternal {1363			v.Fatalf("LoweredGetG should not appear in ABIInternal")1364		}1365		r := v.Reg()1366		getgFromTLS(s, r)1367	case ssaop.OpAMD64CALLstatic, ssaop.OpAMD64CALLtail, ssaop.OpAMD64CALLtailinter:1368		if s.ABI == obj.ABI0 && v.Aux.(*ssa.AuxCall).Fn.ABI() == obj.ABIInternal {1369			// zeroing X15 when entering ABIInternal from ABI01370			zeroX15(s)1371			// set G register from TLS1372			getgFromTLS(s, x86.REG_R14)1373		}1374		if v.Op == ssaop.OpAMD64CALLtail || v.Op == ssaop.OpAMD64CALLtailinter {1375			s.TailCall(v)1376			break1377		}1378		s.Call(v)1379		if s.ABI == obj.ABIInternal && v.Aux.(*ssa.AuxCall).Fn.ABI() == obj.ABI0 {1380			// zeroing X15 when entering ABIInternal from ABI01381			zeroX15(s)1382			// set G register from TLS1383			getgFromTLS(s, x86.REG_R14)1384		}1385	case ssaop.OpAMD64CALLclosure, ssaop.OpAMD64CALLinter:1386		s.Call(v)13871388	case ssaop.OpAMD64LoweredGetCallerPC:1389		p := s.Prog(x86.AMOVQ)1390		p.From.Type = obj.TYPE_MEM1391		p.From.Offset = -8 // PC is stored 8 bytes below first parameter.1392		p.From.Name = obj.NAME_PARAM1393		p.To.Type = obj.TYPE_REG1394		p.To.Reg = v.Reg()13951396	case ssaop.OpAMD64LoweredGetCallerSP:1397		// caller's SP is the address of the first arg1398		mov := x86.AMOVQ1399		if types.PtrSize == 4 {1400			mov = x86.AMOVL1401		}1402		p := s.Prog(mov)1403		p.From.Type = obj.TYPE_ADDR1404		p.From.Offset = -base.Ctxt.Arch.FixedFrameSize // 0 on amd64, just to be consistent with other architectures1405		p.From.Name = obj.NAME_PARAM1406		p.To.Type = obj.TYPE_REG1407		p.To.Reg = v.Reg()14081409	case ssaop.OpAMD64LoweredWB:1410		p := s.Prog(obj.ACALL)1411		p.To.Type = obj.TYPE_MEM1412		p.To.Name = obj.NAME_EXTERN1413		// AuxInt encodes how many buffer entries we need.1414		p.To.Sym = ir.Syms.GCWriteBarrier[v.AuxInt-1]14151416	case ssaop.OpAMD64LoweredPanicBoundsRR, ssaop.OpAMD64LoweredPanicBoundsRC, ssaop.OpAMD64LoweredPanicBoundsCR, ssaop.OpAMD64LoweredPanicBoundsCC:1417		// Compute the constant we put in the PCData entry for this call.1418		code, signed := ssa.BoundsKind(v.AuxInt).Code()1419		xIsReg := false1420		yIsReg := false1421		xVal := 01422		yVal := 01423		switch v.Op {1424		case ssaop.OpAMD64LoweredPanicBoundsRR:1425			xIsReg = true1426			xVal = int(v.Args[0].Reg() - x86.REG_AX)1427			yIsReg = true1428			yVal = int(v.Args[1].Reg() - x86.REG_AX)1429		case ssaop.OpAMD64LoweredPanicBoundsRC:1430			xIsReg = true1431			xVal = int(v.Args[0].Reg() - x86.REG_AX)1432			c := v.Aux.(ssa.PanicBoundsC).C1433			if c >= 0 && c <= abi.BoundsMaxConst {1434				yVal = int(c)1435			} else {1436				// Move constant to a register1437				yIsReg = true1438				if yVal == xVal {1439					yVal = 11440				}1441				p := s.Prog(x86.AMOVQ)1442				p.From.Type = obj.TYPE_CONST1443				p.From.Offset = c1444				p.To.Type = obj.TYPE_REG1445				p.To.Reg = x86.REG_AX + int16(yVal)1446			}1447		case ssaop.OpAMD64LoweredPanicBoundsCR:1448			yIsReg = true1449			yVal = int(v.Args[0].Reg() - x86.REG_AX)1450			c := v.Aux.(ssa.PanicBoundsC).C1451			if c >= 0 && c <= abi.BoundsMaxConst {1452				xVal = int(c)1453			} else {1454				// Move constant to a register1455				xIsReg = true1456				if xVal == yVal {1457					xVal = 11458				}1459				p := s.Prog(x86.AMOVQ)1460				p.From.Type = obj.TYPE_CONST1461				p.From.Offset = c1462				p.To.Type = obj.TYPE_REG1463				p.To.Reg = x86.REG_AX + int16(xVal)1464			}1465		case ssaop.OpAMD64LoweredPanicBoundsCC:1466			c := v.Aux.(ssa.PanicBoundsCC).Cx1467			if c >= 0 && c <= abi.BoundsMaxConst {1468				xVal = int(c)1469			} else {1470				// Move constant to a register1471				xIsReg = true1472				p := s.Prog(x86.AMOVQ)1473				p.From.Type = obj.TYPE_CONST1474				p.From.Offset = c1475				p.To.Type = obj.TYPE_REG1476				p.To.Reg = x86.REG_AX + int16(xVal)1477			}1478			c = v.Aux.(ssa.PanicBoundsCC).Cy1479			if c >= 0 && c <= abi.BoundsMaxConst {1480				yVal = int(c)1481			} else {1482				// Move constant to a register1483				yIsReg = true1484				yVal = 11485				p := s.Prog(x86.AMOVQ)1486				p.From.Type = obj.TYPE_CONST1487				p.From.Offset = c1488				p.To.Type = obj.TYPE_REG1489				p.To.Reg = x86.REG_AX + int16(yVal)1490			}1491		}1492		c := abi.BoundsEncode(code, signed, xIsReg, yIsReg, xVal, yVal)14931494		p := s.Prog(obj.APCDATA)1495		p.From.SetConst(abi.PCDATA_PanicBounds)1496		p.To.SetConst(int64(c))1497		p = s.Prog(obj.ACALL)1498		p.To.Type = obj.TYPE_MEM1499		p.To.Name = obj.NAME_EXTERN1500		p.To.Sym = ir.Syms.PanicBounds15011502	case ssaop.OpAMD64NEGQ, ssaop.OpAMD64NEGL,1503		ssaop.OpAMD64BSWAPQ, ssaop.OpAMD64BSWAPL,1504		ssaop.OpAMD64NOTQ, ssaop.OpAMD64NOTL:1505		p := s.Prog(v.Op.Asm())1506		p.To.Type = obj.TYPE_REG1507		p.To.Reg = v.Reg()15081509	case ssaop.OpAMD64NEGLflags:1510		p := s.Prog(v.Op.Asm())1511		p.To.Type = obj.TYPE_REG1512		p.To.Reg = v.Reg0()15131514	case ssaop.OpAMD64ADDQconstflags, ssaop.OpAMD64ADDLconstflags:1515		p := s.Prog(v.Op.Asm())1516		p.From.Type = obj.TYPE_CONST1517		p.From.Offset = v.AuxInt1518		// Note: the inc/dec instructions do not modify1519		// the carry flag like add$1 / sub$1 do.1520		// We currently never use the CF/OF flags from1521		// these instructions, so that is ok.1522		switch {1523		case p.As == x86.AADDQ && p.From.Offset == 1:1524			p.As = x86.AINCQ1525			p.From.Type = obj.TYPE_NONE1526		case p.As == x86.AADDQ && p.From.Offset == -1:1527			p.As = x86.ADECQ1528			p.From.Type = obj.TYPE_NONE1529		case p.As == x86.AADDL && p.From.Offset == 1:1530			p.As = x86.AINCL1531			p.From.Type = obj.TYPE_NONE1532		case p.As == x86.AADDL && p.From.Offset == -1:1533			p.As = x86.ADECL1534			p.From.Type = obj.TYPE_NONE1535		}1536		p.To.Type = obj.TYPE_REG1537		p.To.Reg = v.Reg0()15381539	case ssaop.OpAMD64BSFQ, ssaop.OpAMD64BSRQ, ssaop.OpAMD64BSFL, ssaop.OpAMD64BSRL, ssaop.OpAMD64SQRTSD, ssaop.OpAMD64SQRTSS:1540		p := s.Prog(v.Op.Asm())1541		p.From.Type = obj.TYPE_REG1542		p.From.Reg = v.Args[0].Reg()1543		p.To.Type = obj.TYPE_REG1544		switch v.Op {1545		case ssaop.OpAMD64BSFQ, ssaop.OpAMD64BSRQ:1546			p.To.Reg = v.Reg0()1547		case ssaop.OpAMD64BSFL, ssaop.OpAMD64BSRL, ssaop.OpAMD64SQRTSD, ssaop.OpAMD64SQRTSS:1548			p.To.Reg = v.Reg()1549		}1550	case ssaop.OpAMD64LoweredRound32F, ssaop.OpAMD64LoweredRound64F:1551		// input is already rounded1552	case ssaop.OpAMD64ROUNDSD, ssaop.OpAMD64ROUNDSS:1553		p := s.Prog(v.Op.Asm())1554		val := v.AuxInt1555		// 0 means math.RoundToEven, 1 Floor, 2 Ceil, 3 Trunc1556		if val < 0 || val > 3 {1557			v.Fatalf("Invalid rounding mode")1558		}1559		p.From.Offset = val1560		p.From.Type = obj.TYPE_CONST1561		p.AddRestSourceReg(v.Args[0].Reg())1562		p.To.Type = obj.TYPE_REG1563		p.To.Reg = v.Reg()1564	case ssaop.OpAMD64POPCNTQ, ssaop.OpAMD64POPCNTL,1565		ssaop.OpAMD64TZCNTQ, ssaop.OpAMD64TZCNTL,1566		ssaop.OpAMD64LZCNTQ, ssaop.OpAMD64LZCNTL:1567		if v.Args[0].Reg() != v.Reg() {1568			// POPCNT/TZCNT/LZCNT have a false dependency on the destination register on Intel cpus.1569			// TZCNT/LZCNT problem affects pre-Skylake models. See discussion at https://gcc.gnu.org/bugzilla/show_bug.cgi?id=62011#c7.1570			// Xor register with itself to break the dependency.1571			opregreg(s, x86.AXORL, v.Reg(), v.Reg())1572		}1573		p := s.Prog(v.Op.Asm())1574		p.From.Type = obj.TYPE_REG1575		p.From.Reg = v.Args[0].Reg()1576		p.To.Type = obj.TYPE_REG1577		p.To.Reg = v.Reg()15781579	case ssaop.OpAMD64SETEQ, ssaop.OpAMD64SETNE,1580		ssaop.OpAMD64SETL, ssaop.OpAMD64SETLE,1581		ssaop.OpAMD64SETG, ssaop.OpAMD64SETGE,1582		ssaop.OpAMD64SETGF, ssaop.OpAMD64SETGEF,1583		ssaop.OpAMD64SETB, ssaop.OpAMD64SETBE,1584		ssaop.OpAMD64SETORD, ssaop.OpAMD64SETNAN,1585		ssaop.OpAMD64SETA, ssaop.OpAMD64SETAE,1586		ssaop.OpAMD64SETO:1587		p := s.Prog(v.Op.Asm())1588		p.To.Type = obj.TYPE_REG1589		p.To.Reg = v.Reg()15901591	case ssaop.OpAMD64SETEQstore, ssaop.OpAMD64SETNEstore,1592		ssaop.OpAMD64SETLstore, ssaop.OpAMD64SETLEstore,1593		ssaop.OpAMD64SETGstore, ssaop.OpAMD64SETGEstore,1594		ssaop.OpAMD64SETBstore, ssaop.OpAMD64SETBEstore,1595		ssaop.OpAMD64SETAstore, ssaop.OpAMD64SETAEstore:1596		p := s.Prog(v.Op.Asm())1597		p.To.Type = obj.TYPE_MEM1598		p.To.Reg = v.Args[0].Reg()1599		ssagen.AddAux(&p.To, v)16001601	case ssaop.OpAMD64SETEQstoreidx1, ssaop.OpAMD64SETNEstoreidx1,1602		ssaop.OpAMD64SETLstoreidx1, ssaop.OpAMD64SETLEstoreidx1,1603		ssaop.OpAMD64SETGstoreidx1, ssaop.OpAMD64SETGEstoreidx1,1604		ssaop.OpAMD64SETBstoreidx1, ssaop.OpAMD64SETBEstoreidx1,1605		ssaop.OpAMD64SETAstoreidx1, ssaop.OpAMD64SETAEstoreidx1:1606		p := s.Prog(v.Op.Asm())1607		memIdx(&p.To, v)1608		ssagen.AddAux(&p.To, v)16091610	case ssaop.OpAMD64SETNEF:1611		t := v.RegTmp()1612		p := s.Prog(v.Op.Asm())1613		p.To.Type = obj.TYPE_REG1614		p.To.Reg = v.Reg()1615		q := s.Prog(x86.ASETPS)1616		q.To.Type = obj.TYPE_REG1617		q.To.Reg = t1618		// ORL avoids partial register write and is smaller than ORQ, used by old compiler1619		opregreg(s, x86.AORL, v.Reg(), t)16201621	case ssaop.OpAMD64SETEQF:1622		t := v.RegTmp()1623		p := s.Prog(v.Op.Asm())1624		p.To.Type = obj.TYPE_REG1625		p.To.Reg = v.Reg()1626		q := s.Prog(x86.ASETPC)1627		q.To.Type = obj.TYPE_REG1628		q.To.Reg = t1629		// ANDL avoids partial register write and is smaller than ANDQ, used by old compiler1630		opregreg(s, x86.AANDL, v.Reg(), t)16311632	case ssaop.OpAMD64InvertFlags:1633		v.Fatalf("InvertFlags should never make it to codegen %v", v.LongString())1634	case ssaop.OpAMD64FlagEQ, ssaop.OpAMD64FlagLT_ULT, ssaop.OpAMD64FlagLT_UGT, ssaop.OpAMD64FlagGT_ULT, ssaop.OpAMD64FlagGT_UGT:1635		v.Fatalf("Flag* ops should never make it to codegen %v", v.LongString())1636	case ssaop.OpAMD64AddTupleFirst32, ssaop.OpAMD64AddTupleFirst64:1637		v.Fatalf("AddTupleFirst* should never make it to codegen %v", v.LongString())1638	case ssaop.OpAMD64REPSTOSQ:1639		s.Prog(x86.AREP)1640		s.Prog(x86.ASTOSQ)1641	case ssaop.OpAMD64REPMOVSQ:1642		s.Prog(x86.AREP)1643		s.Prog(x86.AMOVSQ)1644	case ssaop.OpAMD64LoweredNilCheck:1645		// Issue a load which will fault if the input is nil.1646		// TODO: We currently use the 2-byte instruction TESTB AX, (reg).1647		// Should we use the 3-byte TESTB $0, (reg) instead? It is larger1648		// but it doesn't have false dependency on AX.1649		// Or maybe allocate an output register and use MOVL (reg),reg2 ?1650		// That trades clobbering flags for clobbering a register.1651		p := s.Prog(x86.ATESTB)1652		p.From.Type = obj.TYPE_REG1653		p.From.Reg = x86.REG_AX1654		p.To.Type = obj.TYPE_MEM1655		p.To.Reg = v.Args[0].Reg()1656		if logopt.Enabled() {1657			logopt.LogOpt(v.Pos, "nilcheck", "genssa", v.Block.Func.Name)1658		}1659		if base.Debug.Nil != 0 && v.Pos.Line() > 1 { // v.Pos.Line()==1 in generated wrappers1660			base.WarnfAt(v.Pos, "generated nil check")1661		}1662	case ssaop.OpAMD64MOVBatomicload, ssaop.OpAMD64MOVLatomicload, ssaop.OpAMD64MOVQatomicload:1663		p := s.Prog(v.Op.Asm())1664		p.From.Type = obj.TYPE_MEM1665		p.From.Reg = v.Args[0].Reg()1666		ssagen.AddAux(&p.From, v)1667		p.To.Type = obj.TYPE_REG1668		p.To.Reg = v.Reg0()1669	case ssaop.OpAMD64XCHGB, ssaop.OpAMD64XCHGL, ssaop.OpAMD64XCHGQ:1670		p := s.Prog(v.Op.Asm())1671		p.From.Type = obj.TYPE_REG1672		p.From.Reg = v.Reg0()1673		p.To.Type = obj.TYPE_MEM1674		p.To.Reg = v.Args[1].Reg()1675		ssagen.AddAux(&p.To, v)1676	case ssaop.OpAMD64XADDLlock, ssaop.OpAMD64XADDQlock:1677		s.Prog(x86.ALOCK)1678		p := s.Prog(v.Op.Asm())1679		p.From.Type = obj.TYPE_REG1680		p.From.Reg = v.Reg0()1681		p.To.Type = obj.TYPE_MEM1682		p.To.Reg = v.Args[1].Reg()1683		ssagen.AddAux(&p.To, v)1684	case ssaop.OpAMD64CMPXCHGLlock, ssaop.OpAMD64CMPXCHGQlock:1685		if v.Args[1].Reg() != x86.REG_AX {1686			v.Fatalf("input[1] not in AX %s", v.LongString())1687		}1688		s.Prog(x86.ALOCK)1689		p := s.Prog(v.Op.Asm())1690		p.From.Type = obj.TYPE_REG1691		p.From.Reg = v.Args[2].Reg()1692		p.To.Type = obj.TYPE_MEM1693		p.To.Reg = v.Args[0].Reg()1694		ssagen.AddAux(&p.To, v)1695		p = s.Prog(x86.ASETEQ)1696		p.To.Type = obj.TYPE_REG1697		p.To.Reg = v.Reg0()1698	case ssaop.OpAMD64ANDBlock, ssaop.OpAMD64ANDLlock, ssaop.OpAMD64ANDQlock,1699		ssaop.OpAMD64ORBlock, ssaop.OpAMD64ORLlock, ssaop.OpAMD64ORQlock,1700		ssaop.OpAMD64ADDLlock, ssaop.OpAMD64ADDQlock,1701		ssaop.OpAMD64SUBLlock, ssaop.OpAMD64SUBQlock:1702		// Atomic memory operations that don't need to return the old value.1703		s.Prog(x86.ALOCK)1704		p := s.Prog(v.Op.Asm())1705		p.From.Type = obj.TYPE_REG1706		p.From.Reg = v.Args[1].Reg()1707		p.To.Type = obj.TYPE_MEM1708		p.To.Reg = v.Args[0].Reg()1709		ssagen.AddAux(&p.To, v)1710	case ssaop.OpAMD64INCLlock, ssaop.OpAMD64INCQlock,1711		ssaop.OpAMD64DECLlock, ssaop.OpAMD64DECQlock:1712		// Unary atomic memory operations that don't need to return the old value.1713		s.Prog(x86.ALOCK)1714		p := s.Prog(v.Op.Asm())1715		p.To.Type = obj.TYPE_MEM1716		p.To.Reg = v.Args[0].Reg()1717		ssagen.AddAux(&p.To, v)1718	case ssaop.OpAMD64LoweredAtomicAnd64, ssaop.OpAMD64LoweredAtomicOr64, ssaop.OpAMD64LoweredAtomicAnd32, ssaop.OpAMD64LoweredAtomicOr32:1719		// Atomic memory operations that need to return the old value.1720		// We need to do these with compare-and-exchange to get access to the old value.1721		// loop:1722		// MOVQ mask, tmp1723		// MOVQ (addr), AX1724		// ANDQ AX, tmp1725		// LOCK CMPXCHGQ tmp, (addr) : note that AX is implicit old value to compare against1726		// JNE loop1727		// : result in AX1728		//1729		// If the width written to AX changes, update zeroUpperBits in AMD64Ops.go.1730		mov := x86.AMOVQ1731		op := x86.AANDQ1732		cmpxchg := x86.ACMPXCHGQ1733		switch v.Op {1734		case ssaop.OpAMD64LoweredAtomicOr64:1735			op = x86.AORQ1736		case ssaop.OpAMD64LoweredAtomicAnd32:1737			mov = x86.AMOVL1738			op = x86.AANDL1739			cmpxchg = x86.ACMPXCHGL1740		case ssaop.OpAMD64LoweredAtomicOr32:1741			mov = x86.AMOVL1742			op = x86.AORL1743			cmpxchg = x86.ACMPXCHGL1744		}1745		addr := v.Args[0].Reg()1746		mask := v.Args[1].Reg()1747		tmp := v.RegTmp()1748		p1 := s.Prog(mov)1749		p1.From.Type = obj.TYPE_REG1750		p1.From.Reg = mask1751		p1.To.Type = obj.TYPE_REG1752		p1.To.Reg = tmp1753		p2 := s.Prog(mov)1754		p2.From.Type = obj.TYPE_MEM1755		p2.From.Reg = addr1756		ssagen.AddAux(&p2.From, v)1757		p2.To.Type = obj.TYPE_REG1758		p2.To.Reg = x86.REG_AX1759		p3 := s.Prog(op)1760		p3.From.Type = obj.TYPE_REG1761		p3.From.Reg = x86.REG_AX1762		p3.To.Type = obj.TYPE_REG1763		p3.To.Reg = tmp1764		s.Prog(x86.ALOCK)1765		p5 := s.Prog(cmpxchg)1766		p5.From.Type = obj.TYPE_REG1767		p5.From.Reg = tmp1768		p5.To.Type = obj.TYPE_MEM1769		p5.To.Reg = addr1770		ssagen.AddAux(&p5.To, v)1771		p6 := s.Prog(x86.AJNE)1772		p6.To.Type = obj.TYPE_BRANCH1773		p6.To.SetTarget(p1)1774	case ssaop.OpAMD64PrefetchT0, ssaop.OpAMD64PrefetchNTA:1775		p := s.Prog(v.Op.Asm())1776		p.From.Type = obj.TYPE_MEM1777		p.From.Reg = v.Args[0].Reg()1778	case ssaop.OpClobber:1779		p := s.Prog(x86.AMOVL)1780		p.From.Type = obj.TYPE_CONST1781		p.From.Offset = 0xdeaddead1782		p.To.Type = obj.TYPE_MEM1783		p.To.Reg = x86.REG_SP1784		ssagen.AddAux(&p.To, v)1785		p = s.Prog(x86.AMOVL)1786		p.From.Type = obj.TYPE_CONST1787		p.From.Offset = 0xdeaddead1788		p.To.Type = obj.TYPE_MEM1789		p.To.Reg = x86.REG_SP1790		ssagen.AddAux(&p.To, v)1791		p.To.Offset += 41792	case ssaop.OpClobberReg:1793		x := uint64(0xdeaddeaddeaddead)1794		p := s.Prog(x86.AMOVQ)1795		p.From.Type = obj.TYPE_CONST1796		p.From.Offset = int64(x)1797		p.To.Type = obj.TYPE_REG1798		p.To.Reg = v.Reg()17991800	// SIMD ops1801	case ssaop.OpAMD64VZEROUPPER, ssaop.OpAMD64VZEROALL:1802		s.Prog(v.Op.Asm())18031804	case ssaop.OpAMD64Zero128, ssaop.OpAMD64Zero256, ssaop.OpAMD64Zero512: // no code emitted18051806	case ssaop.OpAMD64VMOVSSf2v, ssaop.OpAMD64VMOVSDf2v:1807		// These are for initializing the least 32/64 bits of a SIMD register from a "float".1808		p := s.Prog(v.Op.Asm())1809		p.From.Type = obj.TYPE_REG1810		p.From.Reg = v.Args[0].Reg()1811		p.AddRestSourceReg(x86.REG_X15)1812		p.To.Type = obj.TYPE_REG1813		p.To.Reg = simdReg(v)18141815	case ssaop.OpAMD64VMOVQload, ssaop.OpAMD64VMOVDload,1816		ssaop.OpAMD64VMOVSSload, ssaop.OpAMD64VMOVSDload:1817		p := s.Prog(v.Op.Asm())1818		p.From.Type = obj.TYPE_MEM1819		p.From.Reg = v.Args[0].Reg()1820		ssagen.AddAux(&p.From, v)1821		p.To.Type = obj.TYPE_REG1822		p.To.Reg = simdReg(v)18231824	case ssaop.OpAMD64VMOVSSconst, ssaop.OpAMD64VMOVSDconst:1825		// for loading constants directly into SIMD registers1826		x := simdReg(v)1827		p := s.Prog(v.Op.Asm())1828		p.From.Type = obj.TYPE_FCONST1829		p.From.Val = math.Float64frombits(uint64(v.AuxInt))1830		p.To.Type = obj.TYPE_REG1831		p.To.Reg = x18321833	case ssaop.OpAMD64VMOVD, ssaop.OpAMD64VMOVQ:1834		// These are for initializing the least 32/64 bits of a SIMD register from an "int".1835		p := s.Prog(v.Op.Asm())1836		p.From.Type = obj.TYPE_REG1837		p.From.Reg = v.Args[0].Reg()1838		p.To.Type = obj.TYPE_REG1839		p.To.Reg = simdReg(v)18401841	case ssaop.OpAMD64VMOVDQUload128, ssaop.OpAMD64VMOVDQUload256, ssaop.OpAMD64VMOVDQUload512,1842		ssaop.OpAMD64KMOVBload, ssaop.OpAMD64KMOVWload, ssaop.OpAMD64KMOVDload, ssaop.OpAMD64KMOVQload:1843		p := s.Prog(v.Op.Asm())1844		p.From.Type = obj.TYPE_MEM1845		p.From.Reg = v.Args[0].Reg()1846		ssagen.AddAux(&p.From, v)1847		p.To.Type = obj.TYPE_REG1848		p.To.Reg = simdOrMaskReg(v)1849	case ssaop.OpAMD64VMOVDQUstore128, ssaop.OpAMD64VMOVDQUstore256, ssaop.OpAMD64VMOVDQUstore512,1850		ssaop.OpAMD64KMOVBstore, ssaop.OpAMD64KMOVWstore, ssaop.OpAMD64KMOVDstore, ssaop.OpAMD64KMOVQstore:1851		p := s.Prog(v.Op.Asm())1852		p.From.Type = obj.TYPE_REG1853		p.From.Reg = simdOrMaskReg(v.Args[1])1854		p.To.Type = obj.TYPE_MEM1855		p.To.Reg = v.Args[0].Reg()1856		ssagen.AddAux(&p.To, v)18571858	case ssaop.OpAMD64VPMASK32load128, ssaop.OpAMD64VPMASK64load128, ssaop.OpAMD64VPMASK32load256, ssaop.OpAMD64VPMASK64load256:1859		p := s.Prog(v.Op.Asm())1860		p.From.Type = obj.TYPE_MEM1861		p.From.Reg = v.Args[0].Reg()1862		ssagen.AddAux(&p.From, v)1863		p.To.Type = obj.TYPE_REG1864		p.To.Reg = simdReg(v)1865		p.AddRestSourceReg(simdReg(v.Args[1])) // masking simd reg18661867	case ssaop.OpAMD64VPMASK32store128, ssaop.OpAMD64VPMASK64store128, ssaop.OpAMD64VPMASK32store256, ssaop.OpAMD64VPMASK64store256:1868		p := s.Prog(v.Op.Asm())1869		p.From.Type = obj.TYPE_REG1870		p.From.Reg = simdReg(v.Args[2])1871		p.To.Type = obj.TYPE_MEM1872		p.To.Reg = v.Args[0].Reg()1873		ssagen.AddAux(&p.To, v)1874		p.AddRestSourceReg(simdReg(v.Args[1])) // masking simd reg18751876	case ssaop.OpAMD64VPMASK64load512, ssaop.OpAMD64VPMASK32load512, ssaop.OpAMD64VPMASK16load512, ssaop.OpAMD64VPMASK8load512:1877		p := s.Prog(v.Op.Asm())1878		p.From.Type = obj.TYPE_MEM1879		p.From.Reg = v.Args[0].Reg()1880		ssagen.AddAux(&p.From, v)1881		p.To.Type = obj.TYPE_REG1882		p.To.Reg = simdReg(v)1883		p.AddRestSourceReg(v.Args[1].Reg()) // simd mask reg1884		x86.ParseSuffix(p, "Z")             // must be zero if not in mask18851886	case ssaop.OpAMD64KANDB, ssaop.OpAMD64KANDW, ssaop.OpAMD64KANDD, ssaop.OpAMD64KANDQ,1887		ssaop.OpAMD64KORB, ssaop.OpAMD64KORW, ssaop.OpAMD64KORD, ssaop.OpAMD64KORQ,1888		ssaop.OpAMD64KXORB, ssaop.OpAMD64KXORW, ssaop.OpAMD64KXORD, ssaop.OpAMD64KXORQ,1889		ssaop.OpAMD64KXNORB, ssaop.OpAMD64KXNORW, ssaop.OpAMD64KXNORD, ssaop.OpAMD64KXNORQ: // XNOR == EQ1890		p := s.Prog(v.Op.Asm())1891		p.From.Type = obj.TYPE_REG1892		p.From.Reg = v.Args[0].Reg()1893		p.To.Type = obj.TYPE_REG1894		p.To.Reg = v.Reg()1895		p.AddRestSourceReg(v.Args[1].Reg()) // masking simd reg18961897	case ssaop.OpAMD64VPMASK64store512, ssaop.OpAMD64VPMASK32store512, ssaop.OpAMD64VPMASK16store512, ssaop.OpAMD64VPMASK8store512:1898		p := s.Prog(v.Op.Asm())1899		p.From.Type = obj.TYPE_REG1900		p.From.Reg = simdReg(v.Args[2])1901		p.To.Type = obj.TYPE_MEM1902		p.To.Reg = v.Args[0].Reg()1903		ssagen.AddAux(&p.To, v)1904		p.AddRestSourceReg(v.Args[1].Reg()) // simd mask reg19051906	case ssaop.OpAMD64VPMOVMToVec8x16,1907		ssaop.OpAMD64VPMOVMToVec8x32,1908		ssaop.OpAMD64VPMOVMToVec8x64,1909		ssaop.OpAMD64VPMOVMToVec16x8,1910		ssaop.OpAMD64VPMOVMToVec16x16,1911		ssaop.OpAMD64VPMOVMToVec16x32,1912		ssaop.OpAMD64VPMOVMToVec32x4,1913		ssaop.OpAMD64VPMOVMToVec32x8,1914		ssaop.OpAMD64VPMOVMToVec32x16,1915		ssaop.OpAMD64VPMOVMToVec64x2,1916		ssaop.OpAMD64VPMOVMToVec64x4,1917		ssaop.OpAMD64VPMOVMToVec64x8:1918		p := s.Prog(v.Op.Asm())1919		p.From.Type = obj.TYPE_REG1920		p.From.Reg = v.Args[0].Reg()1921		p.To.Type = obj.TYPE_REG1922		p.To.Reg = simdReg(v)19231924	case ssaop.OpAMD64VPMOVVec8x16ToM,1925		ssaop.OpAMD64VPMOVVec8x32ToM,1926		ssaop.OpAMD64VPMOVVec8x64ToM,1927		ssaop.OpAMD64VPMOVVec16x8ToM,1928		ssaop.OpAMD64VPMOVVec16x16ToM,1929		ssaop.OpAMD64VPMOVVec16x32ToM,1930		ssaop.OpAMD64VPMOVVec32x4ToM,1931		ssaop.OpAMD64VPMOVVec32x8ToM,1932		ssaop.OpAMD64VPMOVVec32x16ToM,1933		ssaop.OpAMD64VPMOVVec64x2ToM,1934		ssaop.OpAMD64VPMOVVec64x4ToM,1935		ssaop.OpAMD64VPMOVVec64x8ToM,1936		ssaop.OpAMD64VPMOVMSKB128,1937		ssaop.OpAMD64VPMOVMSKB256,1938		ssaop.OpAMD64VMOVMSKPS128,1939		ssaop.OpAMD64VMOVMSKPS256,1940		ssaop.OpAMD64VMOVMSKPD128,1941		ssaop.OpAMD64VMOVMSKPD256:1942		p := s.Prog(v.Op.Asm())1943		p.From.Type = obj.TYPE_REG1944		p.From.Reg = simdReg(v.Args[0])1945		p.To.Type = obj.TYPE_REG1946		p.To.Reg = v.Reg()19471948	case ssaop.OpAMD64KMOVQk, ssaop.OpAMD64KMOVDk, ssaop.OpAMD64KMOVWk, ssaop.OpAMD64KMOVBk,1949		ssaop.OpAMD64KMOVQi, ssaop.OpAMD64KMOVDi, ssaop.OpAMD64KMOVWi, ssaop.OpAMD64KMOVBi:1950		// See also ssa.OpAMD64KMOVQload1951		p := s.Prog(v.Op.Asm())1952		p.From.Type = obj.TYPE_REG1953		p.From.Reg = v.Args[0].Reg()1954		p.To.Type = obj.TYPE_REG1955		p.To.Reg = v.Reg()1956	case ssaop.OpAMD64VPTEST:1957		// Some instructions setting flags put their second operand into the destination reg.1958		// See also CMP[BWDQ].1959		p := s.Prog(v.Op.Asm())1960		p.From.Type = obj.TYPE_REG1961		p.From.Reg = simdReg(v.Args[0])1962		p.To.Type = obj.TYPE_REG1963		p.To.Reg = simdReg(v.Args[1])19641965	default:1966		if !ssaGenSIMDValue(s, v) {1967			v.Fatalf("genValue not implemented: %s", v.LongString())1968		}1969	}1970}19711972// zeroX15 zeroes the X15 register.1973func zeroX15(s *ssagen.State) {1974	vxorps := func(s *ssagen.State) {1975		p := s.Prog(x86.AVXORPS)1976		p.From.Type = obj.TYPE_REG1977		p.From.Reg = x86.REG_X151978		p.AddRestSourceReg(x86.REG_X15)1979		p.To.Type = obj.TYPE_REG1980		p.To.Reg = x86.REG_X151981	}1982	if buildcfg.GOAMD64 >= 3 {1983		vxorps(s)1984		return1985	}1986	opregreg(s, x86.AXORPS, x86.REG_X15, x86.REG_X15)1987	// AVX may not be available, check before zeroing the high bits.1988	p := s.Prog(x86.ACMPB)1989	p.From.Type = obj.TYPE_MEM1990	p.From.Name = obj.NAME_EXTERN1991	p.From.Sym = ir.Syms.X86HasAVX1992	p.To.Type = obj.TYPE_CONST1993	p.To.Offset = 11994	jmp := s.Prog(x86.AJNE)1995	jmp.To.Type = obj.TYPE_BRANCH1996	vxorps(s)1997	end := s.Prog(obj.ANOP)1998	jmp.To.SetTarget(end)1999}