1// Copyright 2025 The Go Authors. All rights reserved.2// Use of this source code is governed by a BSD-style3// license that can be found in the LICENSE file.45#include "go_asm.h"6#include "textflag.h"78// Test-only.9TEXT ·ExpandAVX512(SB), NOSPLIT, $0-2410	MOVQ sizeClass+0(FP), CX11	MOVQ packed+8(FP), AX1213	// Call the expander for this size class14	LEAQ ·gcExpandersAVX512(SB), BX15	MOVQ (BX)(CX*8), DX // Move to register first so -spectre works16	CALL DX1718	MOVQ unpacked+16(FP), DI // Expanded output bitmap pointer19	VMOVDQU64 Z1, 0(DI)20	VMOVDQU64 Z2, 64(DI)21	VZEROUPPER22	RET2324TEXT ·scanSpanPackedAVX512(SB), NOSPLIT, $256-4425	// Z1+Z2 = Expand the grey object mask into a grey word mask26	MOVQ objMarks+16(FP), AX27	MOVQ sizeClass+24(FP), CX28	LEAQ ·gcExpandersAVX512(SB), BX29	MOVQ (BX)(CX*8), DX // Move to register first so -spectre works30	CALL DX3132	// Z3+Z4 = Load the pointer mask33	MOVQ ptrMask+32(FP), AX34	VMOVDQU64 0(AX), Z335	VMOVDQU64 64(AX), Z43637	// Z1+Z2 = Combine the grey word mask with the pointer mask to get the scan mask38	VPANDQ Z1, Z3, Z139	VPANDQ Z2, Z4, Z24041	// Now each bit of Z1+Z2 represents one word of the span.42	// Thus, each byte covers 64 bytes of memory, which is also how43	// much we can fix in a Z register.44	//45	// We do a load/compress for each 64 byte frame.46	//47	// Z3+Z4 [128]uint8 = Number of memory words to scan in each 64 byte frame48	VPOPCNTB Z1, Z3 // Requires BITALG49	VPOPCNTB Z2, Z45051	// Store the scan mask and word counts at 0(SP) and 128(SP).52	//53	// TODO: Is it better to read directly from the registers?54	VMOVDQU64 Z1, 0(SP)55	VMOVDQU64 Z2, 64(SP)56	VMOVDQU64 Z3, 128(SP)57	VMOVDQU64 Z4, 192(SP)5859	// SI = Current address in span60	MOVQ mem+0(FP), SI61	// DI = Scan buffer base62	MOVQ bufp+8(FP), DI63	// DX = Index in scan buffer, (DI)(DX*8) = Current position in scan buffer64	MOVQ $0, DX6566	// AX = address in scan mask, 128(AX) = address in popcount67	LEAQ 0(SP), AX6869	// Loop over the 64 byte frames in this span.70	// BX = 1 past the end of the scan mask71	LEAQ 128(SP), BX7273	// Align loop to a cache line so that performance is less sensitive74	// to how this function ends up laid out in memory. This is a hot75	// function in the GC, and this is a tight loop. We don't want76	// performance to waver wildly due to unrelated changes.77	PCALIGN $6478loop:79	// CX = Fetch the mask of words to load from this frame.80	MOVBQZX 0(AX), CX81	// Skip empty frames.82	TESTQ CX, CX83	JZ skip8485	// Load the 64 byte frame.86	KMOVB CX, K187	VMOVDQA64 0(SI), Z18889	// Collect just the pointers from the greyed objects into the scan buffer,90	// i.e., copy the word indices in the mask from Z1 into contiguous memory.91	//92	// N.B. VPCOMPRESSQ supports a memory destination. Unfortunately, on93	// AMD Genoa / Zen 4, using VPCOMPRESSQ with a memory destination94	// imposes a severe performance penalty of around an order of magnitude95	// compared to a register destination.96	//97	// This workaround is unfortunate on other microarchitectures, where a98	// memory destination is slightly faster than adding an additional move99	// instruction, but no where near an order of magnitude. It would be100	// nice to have a Genoa-only variant here.101	//102	// AMD Turin / Zen 5 fixes this issue.103	//104	// See105	// https://lemire.me/blog/2025/02/14/avx-512-gotcha-avoid-compressing-words-to-memory-with-amd-zen-4-processors/.106	VPCOMPRESSQ Z1, K1, Z2107	VMOVDQU64 Z2, (DI)(DX*8)108109	// Advance the scan buffer position by the number of pointers.110	MOVBQZX 128(AX), CX111	ADDQ CX, DX112113skip:114	ADDQ $64, SI115	ADDQ $1, AX116	CMPQ AX, BX117	JB loop118119end:120	MOVL DX, count+40(FP)121	VZEROUPPER122	RET