1// Copyright 2014 The Go Authors. All rights reserved.2// Use of this source code is governed by a BSD-style3// license that can be found in the LICENSE file.45//go:build !plan967#include "go_asm.h"8#include "textflag.h"9#include "asm_amd64.h"1011// See memclrNoHeapPointers Go doc for important implementation constraints.1213// func memclrNoHeapPointers(ptr unsafe.Pointer, n uintptr)14// ABIInternal for performance.15TEXT runtime·memclrNoHeapPointers<ABIInternal>(SB), NOSPLIT, $0-1616	// AX = ptr17	// BX = n18	MOVQ	AX, DI	// DI = ptr19	XORQ	AX, AX2021	// MOVOU seems always faster than REP STOSQ when Enhanced REP STOSQ is not available.22tail:23	// BSR+branch table make almost all memmove/memclr benchmarks worse. Not worth doing.24	TESTQ	BX, BX25	JEQ	_026	CMPQ	BX, $227	JBE	_1or228	CMPQ	BX, $429	JBE	_3or430	CMPQ	BX, $831	JB	_5through732	JE	_833	CMPQ	BX, $1634	JBE	_9through1635	CMPQ	BX, $3236	JBE	_17through3237	CMPQ	BX, $6438	JBE	_33through6439	CMPQ	BX, $12840	JBE	_65through12841	CMPQ	BX, $25642	JBE	_129through2564344	CMPB	internal∕cpu·X86+const_offsetX86HasERMS(SB), $1 // enhanced REP MOVSB/STOSB45	JNE	skip_erms4647	// If the size is less than 2kb, do not use ERMS as it has a big start-up cost.48	// Table 3-4. Relative Performance of Memcpy() Using ERMSB Vs. 128-bit AVX49	// in the Intel Optimization Guide shows better performance for ERMSB starting50	// from 2KB. Benchmarks show the similar threshold for REP STOS vs AVX.51	CMPQ    BX, $204852	JAE	loop_preheader_erms5354skip_erms:55#ifndef hasAVX256	CMPB	internal∕cpu·X86+const_offsetX86HasAVX2(SB), $157	JE	loop_preheader_avx258	// TODO: for really big clears, use MOVNTDQ, even without AVX2.5960loop:61	MOVOU	X15, 0(DI)62	MOVOU	X15, 16(DI)63	MOVOU	X15, 32(DI)64	MOVOU	X15, 48(DI)65	MOVOU	X15, 64(DI)66	MOVOU	X15, 80(DI)67	MOVOU	X15, 96(DI)68	MOVOU	X15, 112(DI)69	MOVOU	X15, 128(DI)70	MOVOU	X15, 144(DI)71	MOVOU	X15, 160(DI)72	MOVOU	X15, 176(DI)73	MOVOU	X15, 192(DI)74	MOVOU	X15, 208(DI)75	MOVOU	X15, 224(DI)76	MOVOU	X15, 240(DI)77	SUBQ	$256, BX78	ADDQ	$256, DI79	CMPQ	BX, $25680	JAE	loop81	JMP	tail82#endif8384loop_preheader_avx2:85	VPXOR X0, X0, X086	// For smaller sizes MOVNTDQ may be faster or slower depending on hardware.87	// For larger sizes it is always faster, even on dual Xeons with 30M cache.88	// TODO take into account actual LLC size. E. g. glibc uses LLC size/2.89	CMPQ    BX, $0x200000090	JAE	loop_preheader_avx2_huge9192loop_avx2:93	VMOVDQU	Y0, 0(DI)94	VMOVDQU	Y0, 32(DI)95	VMOVDQU	Y0, 64(DI)96	VMOVDQU	Y0, 96(DI)97	SUBQ	$128, BX98	ADDQ	$128, DI99	CMPQ	BX, $128100	JAE	loop_avx2101	VMOVDQU  Y0, -32(DI)(BX*1)102	VMOVDQU  Y0, -64(DI)(BX*1)103	VMOVDQU  Y0, -96(DI)(BX*1)104	VMOVDQU  Y0, -128(DI)(BX*1)105	VZEROUPPER106	RET107108loop_preheader_erms:109#ifndef hasAVX2110	CMPB	internal∕cpu·X86+const_offsetX86HasAVX2(SB), $1111	JNE	loop_erms112#endif113114	VPXOR X0, X0, X0115	// At this point both ERMS and AVX2 is supported. While REP STOS can use a no-RFO116	// write protocol, ERMS could show the same or slower performance comparing to117	// Non-Temporal Stores when the size is bigger than LLC depending on hardware.118	CMPQ	BX, $0x2000000119	JAE	loop_preheader_avx2_huge120121loop_erms:122	// STOSQ is used to guarantee that the whole zeroed pointer-sized word is visible123	// for a memory subsystem as the GC requires this.124	MOVQ	BX, CX125	SHRQ	$3, CX126	ANDQ	$7, BX127	REP;	STOSQ128	JMP	tail129130loop_preheader_avx2_huge:131	// Align to 32 byte boundary132	VMOVDQU  Y0, 0(DI)133	MOVQ	DI, SI134	ADDQ	$32, DI135	ANDQ	$~31, DI136	SUBQ	DI, SI137	ADDQ	SI, BX138loop_avx2_huge:139	VMOVNTDQ	Y0, 0(DI)140	VMOVNTDQ	Y0, 32(DI)141	VMOVNTDQ	Y0, 64(DI)142	VMOVNTDQ	Y0, 96(DI)143	SUBQ	$128, BX144	ADDQ	$128, DI145	CMPQ	BX, $128146	JAE	loop_avx2_huge147	// In the description of MOVNTDQ in [1]148	// "... fencing operation implemented with the SFENCE or MFENCE instruction149	// should be used in conjunction with MOVNTDQ instructions..."150	// [1] 64-ia-32-architectures-software-developer-manual-325462.pdf151	SFENCE152	VMOVDQU  Y0, -32(DI)(BX*1)153	VMOVDQU  Y0, -64(DI)(BX*1)154	VMOVDQU  Y0, -96(DI)(BX*1)155	VMOVDQU  Y0, -128(DI)(BX*1)156	VZEROUPPER157	RET158159_1or2:160	MOVB	AX, (DI)161	MOVB	AX, -1(DI)(BX*1)162	RET163_0:164	RET165_3or4:166	MOVW	AX, (DI)167	MOVW	AX, -2(DI)(BX*1)168	RET169_5through7:170	MOVL	AX, (DI)171	MOVL	AX, -4(DI)(BX*1)172	RET173_8:174	// We need a separate case for 8 to make sure we clear pointers atomically.175	MOVQ	AX, (DI)176	RET177_9through16:178	MOVQ	AX, (DI)179	MOVQ	AX, -8(DI)(BX*1)180	RET181_17through32:182	MOVOU	X15, (DI)183	MOVOU	X15, -16(DI)(BX*1)184	RET185_33through64:186	MOVOU	X15, (DI)187	MOVOU	X15, 16(DI)188	MOVOU	X15, -32(DI)(BX*1)189	MOVOU	X15, -16(DI)(BX*1)190	RET191_65through128:192	MOVOU	X15, (DI)193	MOVOU	X15, 16(DI)194	MOVOU	X15, 32(DI)195	MOVOU	X15, 48(DI)196	MOVOU	X15, -64(DI)(BX*1)197	MOVOU	X15, -48(DI)(BX*1)198	MOVOU	X15, -32(DI)(BX*1)199	MOVOU	X15, -16(DI)(BX*1)200	RET201_129through256:202	MOVOU	X15, (DI)203	MOVOU	X15, 16(DI)204	MOVOU	X15, 32(DI)205	MOVOU	X15, 48(DI)206	MOVOU	X15, 64(DI)207	MOVOU	X15, 80(DI)208	MOVOU	X15, 96(DI)209	MOVOU	X15, 112(DI)210	MOVOU	X15, -128(DI)(BX*1)211	MOVOU	X15, -112(DI)(BX*1)212	MOVOU	X15, -96(DI)(BX*1)213	MOVOU	X15, -80(DI)(BX*1)214	MOVOU	X15, -64(DI)(BX*1)215	MOVOU	X15, -48(DI)(BX*1)216	MOVOU	X15, -32(DI)(BX*1)217	MOVOU	X15, -16(DI)(BX*1)218	RET