PageRenderTime 67ms CodeModel.GetById 13ms app.highlight 50ms RepoModel.GetById 1ms app.codeStats 0ms

/drivers/gpu/drm/amd/amdgpu/vega10_ih.c

https://gitlab.com/sunny256/linux
C | 523 lines | 325 code | 79 blank | 119 comment | 26 complexity | 2f222f35c470662f5691269d29ced170 MD5 | raw file
  1/*
  2 * Copyright 2016 Advanced Micro Devices, Inc.
  3 *
  4 * Permission is hereby granted, free of charge, to any person obtaining a
  5 * copy of this software and associated documentation files (the "Software"),
  6 * to deal in the Software without restriction, including without limitation
  7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
  8 * and/or sell copies of the Software, and to permit persons to whom the
  9 * Software is furnished to do so, subject to the following conditions:
 10 *
 11 * The above copyright notice and this permission notice shall be included in
 12 * all copies or substantial portions of the Software.
 13 *
 14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
 17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
 18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
 19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
 20 * OTHER DEALINGS IN THE SOFTWARE.
 21 *
 22 */
 23#include <drm/drmP.h>
 24#include "amdgpu.h"
 25#include "amdgpu_ih.h"
 26#include "soc15.h"
 27
 28
 29#include "vega10/soc15ip.h"
 30#include "vega10/OSSSYS/osssys_4_0_offset.h"
 31#include "vega10/OSSSYS/osssys_4_0_sh_mask.h"
 32
 33#include "soc15_common.h"
 34#include "vega10_ih.h"
 35
 36
 37
 38static void vega10_ih_set_interrupt_funcs(struct amdgpu_device *adev);
 39
 40/**
 41 * vega10_ih_enable_interrupts - Enable the interrupt ring buffer
 42 *
 43 * @adev: amdgpu_device pointer
 44 *
 45 * Enable the interrupt ring buffer (VEGA10).
 46 */
 47static void vega10_ih_enable_interrupts(struct amdgpu_device *adev)
 48{
 49	u32 ih_rb_cntl = RREG32(SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_CNTL));
 50
 51	ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, RB_ENABLE, 1);
 52	ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, ENABLE_INTR, 1);
 53	WREG32(SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_CNTL), ih_rb_cntl);
 54	adev->irq.ih.enabled = true;
 55}
 56
 57/**
 58 * vega10_ih_disable_interrupts - Disable the interrupt ring buffer
 59 *
 60 * @adev: amdgpu_device pointer
 61 *
 62 * Disable the interrupt ring buffer (VEGA10).
 63 */
 64static void vega10_ih_disable_interrupts(struct amdgpu_device *adev)
 65{
 66	u32 ih_rb_cntl = RREG32(SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_CNTL));
 67
 68	ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, RB_ENABLE, 0);
 69	ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, ENABLE_INTR, 0);
 70	WREG32(SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_CNTL), ih_rb_cntl);
 71	/* set rptr, wptr to 0 */
 72	WREG32(SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_RPTR), 0);
 73	WREG32(SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_WPTR), 0);
 74	adev->irq.ih.enabled = false;
 75	adev->irq.ih.rptr = 0;
 76}
 77
 78/**
 79 * vega10_ih_irq_init - init and enable the interrupt ring
 80 *
 81 * @adev: amdgpu_device pointer
 82 *
 83 * Allocate a ring buffer for the interrupt controller,
 84 * enable the RLC, disable interrupts, enable the IH
 85 * ring buffer and enable it (VI).
 86 * Called at device load and reume.
 87 * Returns 0 for success, errors for failure.
 88 */
 89static int vega10_ih_irq_init(struct amdgpu_device *adev)
 90{
 91	int ret = 0;
 92	int rb_bufsz;
 93	u32 ih_rb_cntl, ih_doorbell_rtpr;
 94	u32 tmp;
 95	u64 wptr_off;
 96
 97	/* disable irqs */
 98	vega10_ih_disable_interrupts(adev);
 99
100	if (adev->flags & AMD_IS_APU)
101		nbio_v7_0_ih_control(adev);
102	else
103		nbio_v6_1_ih_control(adev);
104
105	ih_rb_cntl = RREG32(SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_CNTL));
106	/* Ring Buffer base. [39:8] of 40-bit address of the beginning of the ring buffer*/
107	if (adev->irq.ih.use_bus_addr) {
108		WREG32(SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_BASE), adev->irq.ih.rb_dma_addr >> 8);
109		WREG32(SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_BASE_HI), ((u64)adev->irq.ih.rb_dma_addr >> 40) & 0xff);
110		ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, MC_SPACE, 1);
111	} else {
112		WREG32(SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_BASE), adev->irq.ih.gpu_addr >> 8);
113		WREG32(SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_BASE_HI), (adev->irq.ih.gpu_addr >> 40) & 0xff);
114		ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, MC_SPACE, 4);
115	}
116	rb_bufsz = order_base_2(adev->irq.ih.ring_size / 4);
117	ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, WPTR_OVERFLOW_CLEAR, 1);
118	ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, WPTR_OVERFLOW_ENABLE, 1);
119	ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, RB_SIZE, rb_bufsz);
120	/* Ring Buffer write pointer writeback. If enabled, IH_RB_WPTR register value is written to memory */
121	ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, WPTR_WRITEBACK_ENABLE, 1);
122	ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, MC_SNOOP, 1);
123	ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, MC_RO, 0);
124	ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, MC_VMID, 0);
125
126	if (adev->irq.msi_enabled)
127		ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, RPTR_REARM, 1);
128
129	WREG32(SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_CNTL), ih_rb_cntl);
130
131	/* set the writeback address whether it's enabled or not */
132	if (adev->irq.ih.use_bus_addr)
133		wptr_off = adev->irq.ih.rb_dma_addr + (adev->irq.ih.wptr_offs * 4);
134	else
135		wptr_off = adev->wb.gpu_addr + (adev->irq.ih.wptr_offs * 4);
136	WREG32(SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_WPTR_ADDR_LO), lower_32_bits(wptr_off));
137	WREG32(SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_WPTR_ADDR_HI), upper_32_bits(wptr_off) & 0xFF);
138
139	/* set rptr, wptr to 0 */
140	WREG32(SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_RPTR), 0);
141	WREG32(SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_WPTR), 0);
142
143	ih_doorbell_rtpr = RREG32(SOC15_REG_OFFSET(OSSSYS, 0, mmIH_DOORBELL_RPTR));
144	if (adev->irq.ih.use_doorbell) {
145		ih_doorbell_rtpr = REG_SET_FIELD(ih_doorbell_rtpr, IH_DOORBELL_RPTR,
146						 OFFSET, adev->irq.ih.doorbell_index);
147		ih_doorbell_rtpr = REG_SET_FIELD(ih_doorbell_rtpr, IH_DOORBELL_RPTR,
148						 ENABLE, 1);
149	} else {
150		ih_doorbell_rtpr = REG_SET_FIELD(ih_doorbell_rtpr, IH_DOORBELL_RPTR,
151						 ENABLE, 0);
152	}
153	WREG32(SOC15_REG_OFFSET(OSSSYS, 0, mmIH_DOORBELL_RPTR), ih_doorbell_rtpr);
154	if (adev->flags & AMD_IS_APU)
155		nbio_v7_0_ih_doorbell_range(adev, adev->irq.ih.use_doorbell, adev->irq.ih.doorbell_index);
156	else
157		nbio_v6_1_ih_doorbell_range(adev, adev->irq.ih.use_doorbell, adev->irq.ih.doorbell_index);
158
159	tmp = RREG32(SOC15_REG_OFFSET(OSSSYS, 0, mmIH_STORM_CLIENT_LIST_CNTL));
160	tmp = REG_SET_FIELD(tmp, IH_STORM_CLIENT_LIST_CNTL,
161			    CLIENT18_IS_STORM_CLIENT, 1);
162	WREG32(SOC15_REG_OFFSET(OSSSYS, 0, mmIH_STORM_CLIENT_LIST_CNTL), tmp);
163
164	tmp = RREG32(SOC15_REG_OFFSET(OSSSYS, 0, mmIH_INT_FLOOD_CNTL));
165	tmp = REG_SET_FIELD(tmp, IH_INT_FLOOD_CNTL, FLOOD_CNTL_ENABLE, 1);
166	WREG32(SOC15_REG_OFFSET(OSSSYS, 0, mmIH_INT_FLOOD_CNTL), tmp);
167
168	pci_set_master(adev->pdev);
169
170	/* enable interrupts */
171	vega10_ih_enable_interrupts(adev);
172
173	return ret;
174}
175
176/**
177 * vega10_ih_irq_disable - disable interrupts
178 *
179 * @adev: amdgpu_device pointer
180 *
181 * Disable interrupts on the hw (VEGA10).
182 */
183static void vega10_ih_irq_disable(struct amdgpu_device *adev)
184{
185	vega10_ih_disable_interrupts(adev);
186
187	/* Wait and acknowledge irq */
188	mdelay(1);
189}
190
191/**
192 * vega10_ih_get_wptr - get the IH ring buffer wptr
193 *
194 * @adev: amdgpu_device pointer
195 *
196 * Get the IH ring buffer wptr from either the register
197 * or the writeback memory buffer (VEGA10).  Also check for
198 * ring buffer overflow and deal with it.
199 * Returns the value of the wptr.
200 */
201static u32 vega10_ih_get_wptr(struct amdgpu_device *adev)
202{
203	u32 wptr, tmp;
204
205	if (adev->irq.ih.use_bus_addr)
206		wptr = le32_to_cpu(adev->irq.ih.ring[adev->irq.ih.wptr_offs]);
207	else
208		wptr = le32_to_cpu(adev->wb.wb[adev->irq.ih.wptr_offs]);
209
210	if (REG_GET_FIELD(wptr, IH_RB_WPTR, RB_OVERFLOW)) {
211		wptr = REG_SET_FIELD(wptr, IH_RB_WPTR, RB_OVERFLOW, 0);
212
213		/* When a ring buffer overflow happen start parsing interrupt
214		 * from the last not overwritten vector (wptr + 32). Hopefully
215		 * this should allow us to catchup.
216		 */
217		tmp = (wptr + 32) & adev->irq.ih.ptr_mask;
218		dev_warn(adev->dev, "IH ring buffer overflow (0x%08X, 0x%08X, 0x%08X)\n",
219			wptr, adev->irq.ih.rptr, tmp);
220		adev->irq.ih.rptr = tmp;
221
222		tmp = RREG32_NO_KIQ(SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_CNTL));
223		tmp = REG_SET_FIELD(tmp, IH_RB_CNTL, WPTR_OVERFLOW_CLEAR, 1);
224		WREG32_NO_KIQ(SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_CNTL), tmp);
225	}
226	return (wptr & adev->irq.ih.ptr_mask);
227}
228
229/**
230 * vega10_ih_prescreen_iv - prescreen an interrupt vector
231 *
232 * @adev: amdgpu_device pointer
233 *
234 * Returns true if the interrupt vector should be further processed.
235 */
236static bool vega10_ih_prescreen_iv(struct amdgpu_device *adev)
237{
238	u32 ring_index = adev->irq.ih.rptr >> 2;
239	u32 dw0, dw3, dw4, dw5;
240	u16 pasid;
241	u64 addr, key;
242	struct amdgpu_vm *vm;
243	int r;
244
245	dw0 = le32_to_cpu(adev->irq.ih.ring[ring_index + 0]);
246	dw3 = le32_to_cpu(adev->irq.ih.ring[ring_index + 3]);
247	dw4 = le32_to_cpu(adev->irq.ih.ring[ring_index + 4]);
248	dw5 = le32_to_cpu(adev->irq.ih.ring[ring_index + 5]);
249
250	/* Filter retry page faults, let only the first one pass. If
251	 * there are too many outstanding faults, ignore them until
252	 * some faults get cleared.
253	 */
254	switch (dw0 & 0xff) {
255	case AMDGPU_IH_CLIENTID_VMC:
256	case AMDGPU_IH_CLIENTID_UTCL2:
257		break;
258	default:
259		/* Not a VM fault */
260		return true;
261	}
262
263	pasid = dw3 & 0xffff;
264	/* No PASID, can't identify faulting process */
265	if (!pasid)
266		return true;
267
268	/* Not a retry fault, check fault credit */
269	if (!(dw5 & 0x80)) {
270		if (!amdgpu_vm_pasid_fault_credit(adev, pasid))
271			goto ignore_iv;
272		return true;
273	}
274
275	addr = ((u64)(dw5 & 0xf) << 44) | ((u64)dw4 << 12);
276	key = AMDGPU_VM_FAULT(pasid, addr);
277	r = amdgpu_ih_add_fault(adev, key);
278
279	/* Hash table is full or the fault is already being processed,
280	 * ignore further page faults
281	 */
282	if (r != 0)
283		goto ignore_iv;
284
285	/* Track retry faults in per-VM fault FIFO. */
286	spin_lock(&adev->vm_manager.pasid_lock);
287	vm = idr_find(&adev->vm_manager.pasid_idr, pasid);
288	spin_unlock(&adev->vm_manager.pasid_lock);
289	if (WARN_ON_ONCE(!vm)) {
290		/* VM not found, process it normally */
291		amdgpu_ih_clear_fault(adev, key);
292		return true;
293	}
294	/* No locking required with single writer and single reader */
295	r = kfifo_put(&vm->faults, key);
296	if (!r) {
297		/* FIFO is full. Ignore it until there is space */
298		amdgpu_ih_clear_fault(adev, key);
299		goto ignore_iv;
300	}
301
302	/* It's the first fault for this address, process it normally */
303	return true;
304
305ignore_iv:
306	adev->irq.ih.rptr += 32;
307	return false;
308}
309
310/**
311 * vega10_ih_decode_iv - decode an interrupt vector
312 *
313 * @adev: amdgpu_device pointer
314 *
315 * Decodes the interrupt vector at the current rptr
316 * position and also advance the position.
317 */
318static void vega10_ih_decode_iv(struct amdgpu_device *adev,
319				 struct amdgpu_iv_entry *entry)
320{
321	/* wptr/rptr are in bytes! */
322	u32 ring_index = adev->irq.ih.rptr >> 2;
323	uint32_t dw[8];
324
325	dw[0] = le32_to_cpu(adev->irq.ih.ring[ring_index + 0]);
326	dw[1] = le32_to_cpu(adev->irq.ih.ring[ring_index + 1]);
327	dw[2] = le32_to_cpu(adev->irq.ih.ring[ring_index + 2]);
328	dw[3] = le32_to_cpu(adev->irq.ih.ring[ring_index + 3]);
329	dw[4] = le32_to_cpu(adev->irq.ih.ring[ring_index + 4]);
330	dw[5] = le32_to_cpu(adev->irq.ih.ring[ring_index + 5]);
331	dw[6] = le32_to_cpu(adev->irq.ih.ring[ring_index + 6]);
332	dw[7] = le32_to_cpu(adev->irq.ih.ring[ring_index + 7]);
333
334	entry->client_id = dw[0] & 0xff;
335	entry->src_id = (dw[0] >> 8) & 0xff;
336	entry->ring_id = (dw[0] >> 16) & 0xff;
337	entry->vm_id = (dw[0] >> 24) & 0xf;
338	entry->vm_id_src = (dw[0] >> 31);
339	entry->timestamp = dw[1] | ((u64)(dw[2] & 0xffff) << 32);
340	entry->timestamp_src = dw[2] >> 31;
341	entry->pas_id = dw[3] & 0xffff;
342	entry->pasid_src = dw[3] >> 31;
343	entry->src_data[0] = dw[4];
344	entry->src_data[1] = dw[5];
345	entry->src_data[2] = dw[6];
346	entry->src_data[3] = dw[7];
347
348
349	/* wptr/rptr are in bytes! */
350	adev->irq.ih.rptr += 32;
351}
352
353/**
354 * vega10_ih_set_rptr - set the IH ring buffer rptr
355 *
356 * @adev: amdgpu_device pointer
357 *
358 * Set the IH ring buffer rptr.
359 */
360static void vega10_ih_set_rptr(struct amdgpu_device *adev)
361{
362	if (adev->irq.ih.use_doorbell) {
363		/* XXX check if swapping is necessary on BE */
364		if (adev->irq.ih.use_bus_addr)
365			adev->irq.ih.ring[adev->irq.ih.rptr_offs] = adev->irq.ih.rptr;
366		else
367			adev->wb.wb[adev->irq.ih.rptr_offs] = adev->irq.ih.rptr;
368		WDOORBELL32(adev->irq.ih.doorbell_index, adev->irq.ih.rptr);
369	} else {
370		WREG32(SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_RPTR), adev->irq.ih.rptr);
371	}
372}
373
374static int vega10_ih_early_init(void *handle)
375{
376	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
377
378	vega10_ih_set_interrupt_funcs(adev);
379	return 0;
380}
381
382static int vega10_ih_sw_init(void *handle)
383{
384	int r;
385	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
386
387	r = amdgpu_ih_ring_init(adev, 256 * 1024, true);
388	if (r)
389		return r;
390
391	adev->irq.ih.use_doorbell = true;
392	adev->irq.ih.doorbell_index = AMDGPU_DOORBELL64_IH << 1;
393
394	adev->irq.ih.faults = kmalloc(sizeof(*adev->irq.ih.faults), GFP_KERNEL);
395	if (!adev->irq.ih.faults)
396		return -ENOMEM;
397	INIT_CHASH_TABLE(adev->irq.ih.faults->hash,
398			 AMDGPU_PAGEFAULT_HASH_BITS, 8, 0);
399	spin_lock_init(&adev->irq.ih.faults->lock);
400	adev->irq.ih.faults->count = 0;
401
402	r = amdgpu_irq_init(adev);
403
404	return r;
405}
406
407static int vega10_ih_sw_fini(void *handle)
408{
409	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
410
411	amdgpu_irq_fini(adev);
412	amdgpu_ih_ring_fini(adev);
413
414	kfree(adev->irq.ih.faults);
415	adev->irq.ih.faults = NULL;
416
417	return 0;
418}
419
420static int vega10_ih_hw_init(void *handle)
421{
422	int r;
423	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
424
425	r = vega10_ih_irq_init(adev);
426	if (r)
427		return r;
428
429	return 0;
430}
431
432static int vega10_ih_hw_fini(void *handle)
433{
434	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
435
436	vega10_ih_irq_disable(adev);
437
438	return 0;
439}
440
441static int vega10_ih_suspend(void *handle)
442{
443	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
444
445	return vega10_ih_hw_fini(adev);
446}
447
448static int vega10_ih_resume(void *handle)
449{
450	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
451
452	return vega10_ih_hw_init(adev);
453}
454
455static bool vega10_ih_is_idle(void *handle)
456{
457	/* todo */
458	return true;
459}
460
461static int vega10_ih_wait_for_idle(void *handle)
462{
463	/* todo */
464	return -ETIMEDOUT;
465}
466
467static int vega10_ih_soft_reset(void *handle)
468{
469	/* todo */
470
471	return 0;
472}
473
474static int vega10_ih_set_clockgating_state(void *handle,
475					  enum amd_clockgating_state state)
476{
477	return 0;
478}
479
480static int vega10_ih_set_powergating_state(void *handle,
481					  enum amd_powergating_state state)
482{
483	return 0;
484}
485
486const struct amd_ip_funcs vega10_ih_ip_funcs = {
487	.name = "vega10_ih",
488	.early_init = vega10_ih_early_init,
489	.late_init = NULL,
490	.sw_init = vega10_ih_sw_init,
491	.sw_fini = vega10_ih_sw_fini,
492	.hw_init = vega10_ih_hw_init,
493	.hw_fini = vega10_ih_hw_fini,
494	.suspend = vega10_ih_suspend,
495	.resume = vega10_ih_resume,
496	.is_idle = vega10_ih_is_idle,
497	.wait_for_idle = vega10_ih_wait_for_idle,
498	.soft_reset = vega10_ih_soft_reset,
499	.set_clockgating_state = vega10_ih_set_clockgating_state,
500	.set_powergating_state = vega10_ih_set_powergating_state,
501};
502
503static const struct amdgpu_ih_funcs vega10_ih_funcs = {
504	.get_wptr = vega10_ih_get_wptr,
505	.prescreen_iv = vega10_ih_prescreen_iv,
506	.decode_iv = vega10_ih_decode_iv,
507	.set_rptr = vega10_ih_set_rptr
508};
509
510static void vega10_ih_set_interrupt_funcs(struct amdgpu_device *adev)
511{
512	if (adev->irq.ih_funcs == NULL)
513		adev->irq.ih_funcs = &vega10_ih_funcs;
514}
515
516const struct amdgpu_ip_block_version vega10_ih_ip_block =
517{
518	.type = AMD_IP_BLOCK_TYPE_IH,
519	.major = 4,
520	.minor = 0,
521	.rev = 0,
522	.funcs = &vega10_ih_ip_funcs,
523};