PageRenderTime 123ms CodeModel.GetById 26ms app.highlight 82ms RepoModel.GetById 0ms app.codeStats 1ms

/drivers/gpu/drm/i915/i915_irq.c

https://bitbucket.org/digetx/picasso-kernel
C | 2775 lines | 2007 code | 491 blank | 277 comment | 343 complexity | 32a4a86ef09027a5b54f6bdceb024da0 MD5 | raw file
Possible License(s): AGPL-1.0, GPL-2.0, LGPL-2.0
   1/* i915_irq.c -- IRQ support for the I915 -*- linux-c -*-
   2 */
   3/*
   4 * Copyright 2003 Tungsten Graphics, Inc., Cedar Park, Texas.
   5 * All Rights Reserved.
   6 *
   7 * Permission is hereby granted, free of charge, to any person obtaining a
   8 * copy of this software and associated documentation files (the
   9 * "Software"), to deal in the Software without restriction, including
  10 * without limitation the rights to use, copy, modify, merge, publish,
  11 * distribute, sub license, and/or sell copies of the Software, and to
  12 * permit persons to whom the Software is furnished to do so, subject to
  13 * the following conditions:
  14 *
  15 * The above copyright notice and this permission notice (including the
  16 * next paragraph) shall be included in all copies or substantial portions
  17 * of the Software.
  18 *
  19 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
  20 * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  21 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT.
  22 * IN NO EVENT SHALL TUNGSTEN GRAPHICS AND/OR ITS SUPPLIERS BE LIABLE FOR
  23 * ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
  24 * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
  25 * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
  26 *
  27 */
  28
  29#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
  30
  31#include <linux/sysrq.h>
  32#include <linux/slab.h>
  33#include <drm/drmP.h>
  34#include <drm/i915_drm.h>
  35#include "i915_drv.h"
  36#include "i915_trace.h"
  37#include "intel_drv.h"
  38
  39/* For display hotplug interrupt */
  40static void
  41ironlake_enable_display_irq(drm_i915_private_t *dev_priv, u32 mask)
  42{
  43	if ((dev_priv->irq_mask & mask) != 0) {
  44		dev_priv->irq_mask &= ~mask;
  45		I915_WRITE(DEIMR, dev_priv->irq_mask);
  46		POSTING_READ(DEIMR);
  47	}
  48}
  49
  50static inline void
  51ironlake_disable_display_irq(drm_i915_private_t *dev_priv, u32 mask)
  52{
  53	if ((dev_priv->irq_mask & mask) != mask) {
  54		dev_priv->irq_mask |= mask;
  55		I915_WRITE(DEIMR, dev_priv->irq_mask);
  56		POSTING_READ(DEIMR);
  57	}
  58}
  59
  60void
  61i915_enable_pipestat(drm_i915_private_t *dev_priv, int pipe, u32 mask)
  62{
  63	if ((dev_priv->pipestat[pipe] & mask) != mask) {
  64		u32 reg = PIPESTAT(pipe);
  65
  66		dev_priv->pipestat[pipe] |= mask;
  67		/* Enable the interrupt, clear any pending status */
  68		I915_WRITE(reg, dev_priv->pipestat[pipe] | (mask >> 16));
  69		POSTING_READ(reg);
  70	}
  71}
  72
  73void
  74i915_disable_pipestat(drm_i915_private_t *dev_priv, int pipe, u32 mask)
  75{
  76	if ((dev_priv->pipestat[pipe] & mask) != 0) {
  77		u32 reg = PIPESTAT(pipe);
  78
  79		dev_priv->pipestat[pipe] &= ~mask;
  80		I915_WRITE(reg, dev_priv->pipestat[pipe]);
  81		POSTING_READ(reg);
  82	}
  83}
  84
  85/**
  86 * intel_enable_asle - enable ASLE interrupt for OpRegion
  87 */
  88void intel_enable_asle(struct drm_device *dev)
  89{
  90	drm_i915_private_t *dev_priv = dev->dev_private;
  91	unsigned long irqflags;
  92
  93	/* FIXME: opregion/asle for VLV */
  94	if (IS_VALLEYVIEW(dev))
  95		return;
  96
  97	spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
  98
  99	if (HAS_PCH_SPLIT(dev))
 100		ironlake_enable_display_irq(dev_priv, DE_GSE);
 101	else {
 102		i915_enable_pipestat(dev_priv, 1,
 103				     PIPE_LEGACY_BLC_EVENT_ENABLE);
 104		if (INTEL_INFO(dev)->gen >= 4)
 105			i915_enable_pipestat(dev_priv, 0,
 106					     PIPE_LEGACY_BLC_EVENT_ENABLE);
 107	}
 108
 109	spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
 110}
 111
 112/**
 113 * i915_pipe_enabled - check if a pipe is enabled
 114 * @dev: DRM device
 115 * @pipe: pipe to check
 116 *
 117 * Reading certain registers when the pipe is disabled can hang the chip.
 118 * Use this routine to make sure the PLL is running and the pipe is active
 119 * before reading such registers if unsure.
 120 */
 121static int
 122i915_pipe_enabled(struct drm_device *dev, int pipe)
 123{
 124	drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
 125	enum transcoder cpu_transcoder = intel_pipe_to_cpu_transcoder(dev_priv,
 126								      pipe);
 127
 128	return I915_READ(PIPECONF(cpu_transcoder)) & PIPECONF_ENABLE;
 129}
 130
 131/* Called from drm generic code, passed a 'crtc', which
 132 * we use as a pipe index
 133 */
 134static u32 i915_get_vblank_counter(struct drm_device *dev, int pipe)
 135{
 136	drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
 137	unsigned long high_frame;
 138	unsigned long low_frame;
 139	u32 high1, high2, low;
 140
 141	if (!i915_pipe_enabled(dev, pipe)) {
 142		DRM_DEBUG_DRIVER("trying to get vblank count for disabled "
 143				"pipe %c\n", pipe_name(pipe));
 144		return 0;
 145	}
 146
 147	high_frame = PIPEFRAME(pipe);
 148	low_frame = PIPEFRAMEPIXEL(pipe);
 149
 150	/*
 151	 * High & low register fields aren't synchronized, so make sure
 152	 * we get a low value that's stable across two reads of the high
 153	 * register.
 154	 */
 155	do {
 156		high1 = I915_READ(high_frame) & PIPE_FRAME_HIGH_MASK;
 157		low   = I915_READ(low_frame)  & PIPE_FRAME_LOW_MASK;
 158		high2 = I915_READ(high_frame) & PIPE_FRAME_HIGH_MASK;
 159	} while (high1 != high2);
 160
 161	high1 >>= PIPE_FRAME_HIGH_SHIFT;
 162	low >>= PIPE_FRAME_LOW_SHIFT;
 163	return (high1 << 8) | low;
 164}
 165
 166static u32 gm45_get_vblank_counter(struct drm_device *dev, int pipe)
 167{
 168	drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
 169	int reg = PIPE_FRMCOUNT_GM45(pipe);
 170
 171	if (!i915_pipe_enabled(dev, pipe)) {
 172		DRM_DEBUG_DRIVER("trying to get vblank count for disabled "
 173				 "pipe %c\n", pipe_name(pipe));
 174		return 0;
 175	}
 176
 177	return I915_READ(reg);
 178}
 179
 180static int i915_get_crtc_scanoutpos(struct drm_device *dev, int pipe,
 181			     int *vpos, int *hpos)
 182{
 183	drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
 184	u32 vbl = 0, position = 0;
 185	int vbl_start, vbl_end, htotal, vtotal;
 186	bool in_vbl = true;
 187	int ret = 0;
 188	enum transcoder cpu_transcoder = intel_pipe_to_cpu_transcoder(dev_priv,
 189								      pipe);
 190
 191	if (!i915_pipe_enabled(dev, pipe)) {
 192		DRM_DEBUG_DRIVER("trying to get scanoutpos for disabled "
 193				 "pipe %c\n", pipe_name(pipe));
 194		return 0;
 195	}
 196
 197	/* Get vtotal. */
 198	vtotal = 1 + ((I915_READ(VTOTAL(cpu_transcoder)) >> 16) & 0x1fff);
 199
 200	if (INTEL_INFO(dev)->gen >= 4) {
 201		/* No obvious pixelcount register. Only query vertical
 202		 * scanout position from Display scan line register.
 203		 */
 204		position = I915_READ(PIPEDSL(pipe));
 205
 206		/* Decode into vertical scanout position. Don't have
 207		 * horizontal scanout position.
 208		 */
 209		*vpos = position & 0x1fff;
 210		*hpos = 0;
 211	} else {
 212		/* Have access to pixelcount since start of frame.
 213		 * We can split this into vertical and horizontal
 214		 * scanout position.
 215		 */
 216		position = (I915_READ(PIPEFRAMEPIXEL(pipe)) & PIPE_PIXEL_MASK) >> PIPE_PIXEL_SHIFT;
 217
 218		htotal = 1 + ((I915_READ(HTOTAL(cpu_transcoder)) >> 16) & 0x1fff);
 219		*vpos = position / htotal;
 220		*hpos = position - (*vpos * htotal);
 221	}
 222
 223	/* Query vblank area. */
 224	vbl = I915_READ(VBLANK(cpu_transcoder));
 225
 226	/* Test position against vblank region. */
 227	vbl_start = vbl & 0x1fff;
 228	vbl_end = (vbl >> 16) & 0x1fff;
 229
 230	if ((*vpos < vbl_start) || (*vpos > vbl_end))
 231		in_vbl = false;
 232
 233	/* Inside "upper part" of vblank area? Apply corrective offset: */
 234	if (in_vbl && (*vpos >= vbl_start))
 235		*vpos = *vpos - vtotal;
 236
 237	/* Readouts valid? */
 238	if (vbl > 0)
 239		ret |= DRM_SCANOUTPOS_VALID | DRM_SCANOUTPOS_ACCURATE;
 240
 241	/* In vblank? */
 242	if (in_vbl)
 243		ret |= DRM_SCANOUTPOS_INVBL;
 244
 245	return ret;
 246}
 247
 248static int i915_get_vblank_timestamp(struct drm_device *dev, int pipe,
 249			      int *max_error,
 250			      struct timeval *vblank_time,
 251			      unsigned flags)
 252{
 253	struct drm_crtc *crtc;
 254
 255	if (pipe < 0 || pipe >= INTEL_INFO(dev)->num_pipes) {
 256		DRM_ERROR("Invalid crtc %d\n", pipe);
 257		return -EINVAL;
 258	}
 259
 260	/* Get drm_crtc to timestamp: */
 261	crtc = intel_get_crtc_for_pipe(dev, pipe);
 262	if (crtc == NULL) {
 263		DRM_ERROR("Invalid crtc %d\n", pipe);
 264		return -EINVAL;
 265	}
 266
 267	if (!crtc->enabled) {
 268		DRM_DEBUG_KMS("crtc %d is disabled\n", pipe);
 269		return -EBUSY;
 270	}
 271
 272	/* Helper routine in DRM core does all the work: */
 273	return drm_calc_vbltimestamp_from_scanoutpos(dev, pipe, max_error,
 274						     vblank_time, flags,
 275						     crtc);
 276}
 277
 278/*
 279 * Handle hotplug events outside the interrupt handler proper.
 280 */
 281static void i915_hotplug_work_func(struct work_struct *work)
 282{
 283	drm_i915_private_t *dev_priv = container_of(work, drm_i915_private_t,
 284						    hotplug_work);
 285	struct drm_device *dev = dev_priv->dev;
 286	struct drm_mode_config *mode_config = &dev->mode_config;
 287	struct intel_encoder *encoder;
 288
 289	mutex_lock(&mode_config->mutex);
 290	DRM_DEBUG_KMS("running encoder hotplug functions\n");
 291
 292	list_for_each_entry(encoder, &mode_config->encoder_list, base.head)
 293		if (encoder->hot_plug)
 294			encoder->hot_plug(encoder);
 295
 296	mutex_unlock(&mode_config->mutex);
 297
 298	/* Just fire off a uevent and let userspace tell us what to do */
 299	drm_helper_hpd_irq_event(dev);
 300}
 301
 302/* defined intel_pm.c */
 303extern spinlock_t mchdev_lock;
 304
 305static void ironlake_handle_rps_change(struct drm_device *dev)
 306{
 307	drm_i915_private_t *dev_priv = dev->dev_private;
 308	u32 busy_up, busy_down, max_avg, min_avg;
 309	u8 new_delay;
 310	unsigned long flags;
 311
 312	spin_lock_irqsave(&mchdev_lock, flags);
 313
 314	I915_WRITE16(MEMINTRSTS, I915_READ(MEMINTRSTS));
 315
 316	new_delay = dev_priv->ips.cur_delay;
 317
 318	I915_WRITE16(MEMINTRSTS, MEMINT_EVAL_CHG);
 319	busy_up = I915_READ(RCPREVBSYTUPAVG);
 320	busy_down = I915_READ(RCPREVBSYTDNAVG);
 321	max_avg = I915_READ(RCBMAXAVG);
 322	min_avg = I915_READ(RCBMINAVG);
 323
 324	/* Handle RCS change request from hw */
 325	if (busy_up > max_avg) {
 326		if (dev_priv->ips.cur_delay != dev_priv->ips.max_delay)
 327			new_delay = dev_priv->ips.cur_delay - 1;
 328		if (new_delay < dev_priv->ips.max_delay)
 329			new_delay = dev_priv->ips.max_delay;
 330	} else if (busy_down < min_avg) {
 331		if (dev_priv->ips.cur_delay != dev_priv->ips.min_delay)
 332			new_delay = dev_priv->ips.cur_delay + 1;
 333		if (new_delay > dev_priv->ips.min_delay)
 334			new_delay = dev_priv->ips.min_delay;
 335	}
 336
 337	if (ironlake_set_drps(dev, new_delay))
 338		dev_priv->ips.cur_delay = new_delay;
 339
 340	spin_unlock_irqrestore(&mchdev_lock, flags);
 341
 342	return;
 343}
 344
 345static void notify_ring(struct drm_device *dev,
 346			struct intel_ring_buffer *ring)
 347{
 348	struct drm_i915_private *dev_priv = dev->dev_private;
 349
 350	if (ring->obj == NULL)
 351		return;
 352
 353	trace_i915_gem_request_complete(ring, ring->get_seqno(ring, false));
 354
 355	wake_up_all(&ring->irq_queue);
 356	if (i915_enable_hangcheck) {
 357		dev_priv->hangcheck_count = 0;
 358		mod_timer(&dev_priv->hangcheck_timer,
 359			  round_jiffies_up(jiffies + DRM_I915_HANGCHECK_JIFFIES));
 360	}
 361}
 362
 363static void gen6_pm_rps_work(struct work_struct *work)
 364{
 365	drm_i915_private_t *dev_priv = container_of(work, drm_i915_private_t,
 366						    rps.work);
 367	u32 pm_iir, pm_imr;
 368	u8 new_delay;
 369
 370	spin_lock_irq(&dev_priv->rps.lock);
 371	pm_iir = dev_priv->rps.pm_iir;
 372	dev_priv->rps.pm_iir = 0;
 373	pm_imr = I915_READ(GEN6_PMIMR);
 374	I915_WRITE(GEN6_PMIMR, 0);
 375	spin_unlock_irq(&dev_priv->rps.lock);
 376
 377	if ((pm_iir & GEN6_PM_DEFERRED_EVENTS) == 0)
 378		return;
 379
 380	mutex_lock(&dev_priv->rps.hw_lock);
 381
 382	if (pm_iir & GEN6_PM_RP_UP_THRESHOLD)
 383		new_delay = dev_priv->rps.cur_delay + 1;
 384	else
 385		new_delay = dev_priv->rps.cur_delay - 1;
 386
 387	/* sysfs frequency interfaces may have snuck in while servicing the
 388	 * interrupt
 389	 */
 390	if (!(new_delay > dev_priv->rps.max_delay ||
 391	      new_delay < dev_priv->rps.min_delay)) {
 392		gen6_set_rps(dev_priv->dev, new_delay);
 393	}
 394
 395	mutex_unlock(&dev_priv->rps.hw_lock);
 396}
 397
 398
 399/**
 400 * ivybridge_parity_work - Workqueue called when a parity error interrupt
 401 * occurred.
 402 * @work: workqueue struct
 403 *
 404 * Doesn't actually do anything except notify userspace. As a consequence of
 405 * this event, userspace should try to remap the bad rows since statistically
 406 * it is likely the same row is more likely to go bad again.
 407 */
 408static void ivybridge_parity_work(struct work_struct *work)
 409{
 410	drm_i915_private_t *dev_priv = container_of(work, drm_i915_private_t,
 411						    l3_parity.error_work);
 412	u32 error_status, row, bank, subbank;
 413	char *parity_event[5];
 414	uint32_t misccpctl;
 415	unsigned long flags;
 416
 417	/* We must turn off DOP level clock gating to access the L3 registers.
 418	 * In order to prevent a get/put style interface, acquire struct mutex
 419	 * any time we access those registers.
 420	 */
 421	mutex_lock(&dev_priv->dev->struct_mutex);
 422
 423	misccpctl = I915_READ(GEN7_MISCCPCTL);
 424	I915_WRITE(GEN7_MISCCPCTL, misccpctl & ~GEN7_DOP_CLOCK_GATE_ENABLE);
 425	POSTING_READ(GEN7_MISCCPCTL);
 426
 427	error_status = I915_READ(GEN7_L3CDERRST1);
 428	row = GEN7_PARITY_ERROR_ROW(error_status);
 429	bank = GEN7_PARITY_ERROR_BANK(error_status);
 430	subbank = GEN7_PARITY_ERROR_SUBBANK(error_status);
 431
 432	I915_WRITE(GEN7_L3CDERRST1, GEN7_PARITY_ERROR_VALID |
 433				    GEN7_L3CDERRST1_ENABLE);
 434	POSTING_READ(GEN7_L3CDERRST1);
 435
 436	I915_WRITE(GEN7_MISCCPCTL, misccpctl);
 437
 438	spin_lock_irqsave(&dev_priv->irq_lock, flags);
 439	dev_priv->gt_irq_mask &= ~GT_GEN7_L3_PARITY_ERROR_INTERRUPT;
 440	I915_WRITE(GTIMR, dev_priv->gt_irq_mask);
 441	spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
 442
 443	mutex_unlock(&dev_priv->dev->struct_mutex);
 444
 445	parity_event[0] = "L3_PARITY_ERROR=1";
 446	parity_event[1] = kasprintf(GFP_KERNEL, "ROW=%d", row);
 447	parity_event[2] = kasprintf(GFP_KERNEL, "BANK=%d", bank);
 448	parity_event[3] = kasprintf(GFP_KERNEL, "SUBBANK=%d", subbank);
 449	parity_event[4] = NULL;
 450
 451	kobject_uevent_env(&dev_priv->dev->primary->kdev.kobj,
 452			   KOBJ_CHANGE, parity_event);
 453
 454	DRM_DEBUG("Parity error: Row = %d, Bank = %d, Sub bank = %d.\n",
 455		  row, bank, subbank);
 456
 457	kfree(parity_event[3]);
 458	kfree(parity_event[2]);
 459	kfree(parity_event[1]);
 460}
 461
 462static void ivybridge_handle_parity_error(struct drm_device *dev)
 463{
 464	drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
 465	unsigned long flags;
 466
 467	if (!HAS_L3_GPU_CACHE(dev))
 468		return;
 469
 470	spin_lock_irqsave(&dev_priv->irq_lock, flags);
 471	dev_priv->gt_irq_mask |= GT_GEN7_L3_PARITY_ERROR_INTERRUPT;
 472	I915_WRITE(GTIMR, dev_priv->gt_irq_mask);
 473	spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
 474
 475	queue_work(dev_priv->wq, &dev_priv->l3_parity.error_work);
 476}
 477
 478static void snb_gt_irq_handler(struct drm_device *dev,
 479			       struct drm_i915_private *dev_priv,
 480			       u32 gt_iir)
 481{
 482
 483	if (gt_iir & (GEN6_RENDER_USER_INTERRUPT |
 484		      GEN6_RENDER_PIPE_CONTROL_NOTIFY_INTERRUPT))
 485		notify_ring(dev, &dev_priv->ring[RCS]);
 486	if (gt_iir & GEN6_BSD_USER_INTERRUPT)
 487		notify_ring(dev, &dev_priv->ring[VCS]);
 488	if (gt_iir & GEN6_BLITTER_USER_INTERRUPT)
 489		notify_ring(dev, &dev_priv->ring[BCS]);
 490
 491	if (gt_iir & (GT_GEN6_BLT_CS_ERROR_INTERRUPT |
 492		      GT_GEN6_BSD_CS_ERROR_INTERRUPT |
 493		      GT_RENDER_CS_ERROR_INTERRUPT)) {
 494		DRM_ERROR("GT error interrupt 0x%08x\n", gt_iir);
 495		i915_handle_error(dev, false);
 496	}
 497
 498	if (gt_iir & GT_GEN7_L3_PARITY_ERROR_INTERRUPT)
 499		ivybridge_handle_parity_error(dev);
 500}
 501
 502static void gen6_queue_rps_work(struct drm_i915_private *dev_priv,
 503				u32 pm_iir)
 504{
 505	unsigned long flags;
 506
 507	/*
 508	 * IIR bits should never already be set because IMR should
 509	 * prevent an interrupt from being shown in IIR. The warning
 510	 * displays a case where we've unsafely cleared
 511	 * dev_priv->rps.pm_iir. Although missing an interrupt of the same
 512	 * type is not a problem, it displays a problem in the logic.
 513	 *
 514	 * The mask bit in IMR is cleared by dev_priv->rps.work.
 515	 */
 516
 517	spin_lock_irqsave(&dev_priv->rps.lock, flags);
 518	dev_priv->rps.pm_iir |= pm_iir;
 519	I915_WRITE(GEN6_PMIMR, dev_priv->rps.pm_iir);
 520	POSTING_READ(GEN6_PMIMR);
 521	spin_unlock_irqrestore(&dev_priv->rps.lock, flags);
 522
 523	queue_work(dev_priv->wq, &dev_priv->rps.work);
 524}
 525
 526static irqreturn_t valleyview_irq_handler(int irq, void *arg)
 527{
 528	struct drm_device *dev = (struct drm_device *) arg;
 529	drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
 530	u32 iir, gt_iir, pm_iir;
 531	irqreturn_t ret = IRQ_NONE;
 532	unsigned long irqflags;
 533	int pipe;
 534	u32 pipe_stats[I915_MAX_PIPES];
 535	bool blc_event;
 536
 537	atomic_inc(&dev_priv->irq_received);
 538
 539	while (true) {
 540		iir = I915_READ(VLV_IIR);
 541		gt_iir = I915_READ(GTIIR);
 542		pm_iir = I915_READ(GEN6_PMIIR);
 543
 544		if (gt_iir == 0 && pm_iir == 0 && iir == 0)
 545			goto out;
 546
 547		ret = IRQ_HANDLED;
 548
 549		snb_gt_irq_handler(dev, dev_priv, gt_iir);
 550
 551		spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
 552		for_each_pipe(pipe) {
 553			int reg = PIPESTAT(pipe);
 554			pipe_stats[pipe] = I915_READ(reg);
 555
 556			/*
 557			 * Clear the PIPE*STAT regs before the IIR
 558			 */
 559			if (pipe_stats[pipe] & 0x8000ffff) {
 560				if (pipe_stats[pipe] & PIPE_FIFO_UNDERRUN_STATUS)
 561					DRM_DEBUG_DRIVER("pipe %c underrun\n",
 562							 pipe_name(pipe));
 563				I915_WRITE(reg, pipe_stats[pipe]);
 564			}
 565		}
 566		spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
 567
 568		for_each_pipe(pipe) {
 569			if (pipe_stats[pipe] & PIPE_VBLANK_INTERRUPT_STATUS)
 570				drm_handle_vblank(dev, pipe);
 571
 572			if (pipe_stats[pipe] & PLANE_FLIPDONE_INT_STATUS_VLV) {
 573				intel_prepare_page_flip(dev, pipe);
 574				intel_finish_page_flip(dev, pipe);
 575			}
 576		}
 577
 578		/* Consume port.  Then clear IIR or we'll miss events */
 579		if (iir & I915_DISPLAY_PORT_INTERRUPT) {
 580			u32 hotplug_status = I915_READ(PORT_HOTPLUG_STAT);
 581
 582			DRM_DEBUG_DRIVER("hotplug event received, stat 0x%08x\n",
 583					 hotplug_status);
 584			if (hotplug_status & dev_priv->hotplug_supported_mask)
 585				queue_work(dev_priv->wq,
 586					   &dev_priv->hotplug_work);
 587
 588			I915_WRITE(PORT_HOTPLUG_STAT, hotplug_status);
 589			I915_READ(PORT_HOTPLUG_STAT);
 590		}
 591
 592		if (pipe_stats[pipe] & PIPE_LEGACY_BLC_EVENT_STATUS)
 593			blc_event = true;
 594
 595		if (pm_iir & GEN6_PM_DEFERRED_EVENTS)
 596			gen6_queue_rps_work(dev_priv, pm_iir);
 597
 598		I915_WRITE(GTIIR, gt_iir);
 599		I915_WRITE(GEN6_PMIIR, pm_iir);
 600		I915_WRITE(VLV_IIR, iir);
 601	}
 602
 603out:
 604	return ret;
 605}
 606
 607static void ibx_irq_handler(struct drm_device *dev, u32 pch_iir)
 608{
 609	drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
 610	int pipe;
 611
 612	if (pch_iir & SDE_HOTPLUG_MASK)
 613		queue_work(dev_priv->wq, &dev_priv->hotplug_work);
 614
 615	if (pch_iir & SDE_AUDIO_POWER_MASK)
 616		DRM_DEBUG_DRIVER("PCH audio power change on port %d\n",
 617				 (pch_iir & SDE_AUDIO_POWER_MASK) >>
 618				 SDE_AUDIO_POWER_SHIFT);
 619
 620	if (pch_iir & SDE_GMBUS)
 621		DRM_DEBUG_DRIVER("PCH GMBUS interrupt\n");
 622
 623	if (pch_iir & SDE_AUDIO_HDCP_MASK)
 624		DRM_DEBUG_DRIVER("PCH HDCP audio interrupt\n");
 625
 626	if (pch_iir & SDE_AUDIO_TRANS_MASK)
 627		DRM_DEBUG_DRIVER("PCH transcoder audio interrupt\n");
 628
 629	if (pch_iir & SDE_POISON)
 630		DRM_ERROR("PCH poison interrupt\n");
 631
 632	if (pch_iir & SDE_FDI_MASK)
 633		for_each_pipe(pipe)
 634			DRM_DEBUG_DRIVER("  pipe %c FDI IIR: 0x%08x\n",
 635					 pipe_name(pipe),
 636					 I915_READ(FDI_RX_IIR(pipe)));
 637
 638	if (pch_iir & (SDE_TRANSB_CRC_DONE | SDE_TRANSA_CRC_DONE))
 639		DRM_DEBUG_DRIVER("PCH transcoder CRC done interrupt\n");
 640
 641	if (pch_iir & (SDE_TRANSB_CRC_ERR | SDE_TRANSA_CRC_ERR))
 642		DRM_DEBUG_DRIVER("PCH transcoder CRC error interrupt\n");
 643
 644	if (pch_iir & SDE_TRANSB_FIFO_UNDER)
 645		DRM_DEBUG_DRIVER("PCH transcoder B underrun interrupt\n");
 646	if (pch_iir & SDE_TRANSA_FIFO_UNDER)
 647		DRM_DEBUG_DRIVER("PCH transcoder A underrun interrupt\n");
 648}
 649
 650static void cpt_irq_handler(struct drm_device *dev, u32 pch_iir)
 651{
 652	drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
 653	int pipe;
 654
 655	if (pch_iir & SDE_HOTPLUG_MASK_CPT)
 656		queue_work(dev_priv->wq, &dev_priv->hotplug_work);
 657
 658	if (pch_iir & SDE_AUDIO_POWER_MASK_CPT)
 659		DRM_DEBUG_DRIVER("PCH audio power change on port %d\n",
 660				 (pch_iir & SDE_AUDIO_POWER_MASK_CPT) >>
 661				 SDE_AUDIO_POWER_SHIFT_CPT);
 662
 663	if (pch_iir & SDE_AUX_MASK_CPT)
 664		DRM_DEBUG_DRIVER("AUX channel interrupt\n");
 665
 666	if (pch_iir & SDE_GMBUS_CPT)
 667		DRM_DEBUG_DRIVER("PCH GMBUS interrupt\n");
 668
 669	if (pch_iir & SDE_AUDIO_CP_REQ_CPT)
 670		DRM_DEBUG_DRIVER("Audio CP request interrupt\n");
 671
 672	if (pch_iir & SDE_AUDIO_CP_CHG_CPT)
 673		DRM_DEBUG_DRIVER("Audio CP change interrupt\n");
 674
 675	if (pch_iir & SDE_FDI_MASK_CPT)
 676		for_each_pipe(pipe)
 677			DRM_DEBUG_DRIVER("  pipe %c FDI IIR: 0x%08x\n",
 678					 pipe_name(pipe),
 679					 I915_READ(FDI_RX_IIR(pipe)));
 680}
 681
 682static irqreturn_t ivybridge_irq_handler(int irq, void *arg)
 683{
 684	struct drm_device *dev = (struct drm_device *) arg;
 685	drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
 686	u32 de_iir, gt_iir, de_ier, pm_iir;
 687	irqreturn_t ret = IRQ_NONE;
 688	int i;
 689
 690	atomic_inc(&dev_priv->irq_received);
 691
 692	/* disable master interrupt before clearing iir  */
 693	de_ier = I915_READ(DEIER);
 694	I915_WRITE(DEIER, de_ier & ~DE_MASTER_IRQ_CONTROL);
 695
 696	gt_iir = I915_READ(GTIIR);
 697	if (gt_iir) {
 698		snb_gt_irq_handler(dev, dev_priv, gt_iir);
 699		I915_WRITE(GTIIR, gt_iir);
 700		ret = IRQ_HANDLED;
 701	}
 702
 703	de_iir = I915_READ(DEIIR);
 704	if (de_iir) {
 705		if (de_iir & DE_GSE_IVB)
 706			intel_opregion_gse_intr(dev);
 707
 708		for (i = 0; i < 3; i++) {
 709			if (de_iir & (DE_PIPEA_VBLANK_IVB << (5 * i)))
 710				drm_handle_vblank(dev, i);
 711			if (de_iir & (DE_PLANEA_FLIP_DONE_IVB << (5 * i))) {
 712				intel_prepare_page_flip(dev, i);
 713				intel_finish_page_flip_plane(dev, i);
 714			}
 715		}
 716
 717		/* check event from PCH */
 718		if (de_iir & DE_PCH_EVENT_IVB) {
 719			u32 pch_iir = I915_READ(SDEIIR);
 720
 721			cpt_irq_handler(dev, pch_iir);
 722
 723			/* clear PCH hotplug event before clear CPU irq */
 724			I915_WRITE(SDEIIR, pch_iir);
 725		}
 726
 727		I915_WRITE(DEIIR, de_iir);
 728		ret = IRQ_HANDLED;
 729	}
 730
 731	pm_iir = I915_READ(GEN6_PMIIR);
 732	if (pm_iir) {
 733		if (pm_iir & GEN6_PM_DEFERRED_EVENTS)
 734			gen6_queue_rps_work(dev_priv, pm_iir);
 735		I915_WRITE(GEN6_PMIIR, pm_iir);
 736		ret = IRQ_HANDLED;
 737	}
 738
 739	I915_WRITE(DEIER, de_ier);
 740	POSTING_READ(DEIER);
 741
 742	return ret;
 743}
 744
 745static void ilk_gt_irq_handler(struct drm_device *dev,
 746			       struct drm_i915_private *dev_priv,
 747			       u32 gt_iir)
 748{
 749	if (gt_iir & (GT_USER_INTERRUPT | GT_PIPE_NOTIFY))
 750		notify_ring(dev, &dev_priv->ring[RCS]);
 751	if (gt_iir & GT_BSD_USER_INTERRUPT)
 752		notify_ring(dev, &dev_priv->ring[VCS]);
 753}
 754
 755static irqreturn_t ironlake_irq_handler(int irq, void *arg)
 756{
 757	struct drm_device *dev = (struct drm_device *) arg;
 758	drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
 759	int ret = IRQ_NONE;
 760	u32 de_iir, gt_iir, de_ier, pch_iir, pm_iir;
 761
 762	atomic_inc(&dev_priv->irq_received);
 763
 764	/* disable master interrupt before clearing iir  */
 765	de_ier = I915_READ(DEIER);
 766	I915_WRITE(DEIER, de_ier & ~DE_MASTER_IRQ_CONTROL);
 767	POSTING_READ(DEIER);
 768
 769	de_iir = I915_READ(DEIIR);
 770	gt_iir = I915_READ(GTIIR);
 771	pch_iir = I915_READ(SDEIIR);
 772	pm_iir = I915_READ(GEN6_PMIIR);
 773
 774	if (de_iir == 0 && gt_iir == 0 && pch_iir == 0 &&
 775	    (!IS_GEN6(dev) || pm_iir == 0))
 776		goto done;
 777
 778	ret = IRQ_HANDLED;
 779
 780	if (IS_GEN5(dev))
 781		ilk_gt_irq_handler(dev, dev_priv, gt_iir);
 782	else
 783		snb_gt_irq_handler(dev, dev_priv, gt_iir);
 784
 785	if (de_iir & DE_GSE)
 786		intel_opregion_gse_intr(dev);
 787
 788	if (de_iir & DE_PIPEA_VBLANK)
 789		drm_handle_vblank(dev, 0);
 790
 791	if (de_iir & DE_PIPEB_VBLANK)
 792		drm_handle_vblank(dev, 1);
 793
 794	if (de_iir & DE_PLANEA_FLIP_DONE) {
 795		intel_prepare_page_flip(dev, 0);
 796		intel_finish_page_flip_plane(dev, 0);
 797	}
 798
 799	if (de_iir & DE_PLANEB_FLIP_DONE) {
 800		intel_prepare_page_flip(dev, 1);
 801		intel_finish_page_flip_plane(dev, 1);
 802	}
 803
 804	/* check event from PCH */
 805	if (de_iir & DE_PCH_EVENT) {
 806		if (HAS_PCH_CPT(dev))
 807			cpt_irq_handler(dev, pch_iir);
 808		else
 809			ibx_irq_handler(dev, pch_iir);
 810	}
 811
 812	if (IS_GEN5(dev) &&  de_iir & DE_PCU_EVENT)
 813		ironlake_handle_rps_change(dev);
 814
 815	if (IS_GEN6(dev) && pm_iir & GEN6_PM_DEFERRED_EVENTS)
 816		gen6_queue_rps_work(dev_priv, pm_iir);
 817
 818	/* should clear PCH hotplug event before clear CPU irq */
 819	I915_WRITE(SDEIIR, pch_iir);
 820	I915_WRITE(GTIIR, gt_iir);
 821	I915_WRITE(DEIIR, de_iir);
 822	I915_WRITE(GEN6_PMIIR, pm_iir);
 823
 824done:
 825	I915_WRITE(DEIER, de_ier);
 826	POSTING_READ(DEIER);
 827
 828	return ret;
 829}
 830
 831/**
 832 * i915_error_work_func - do process context error handling work
 833 * @work: work struct
 834 *
 835 * Fire an error uevent so userspace can see that a hang or error
 836 * was detected.
 837 */
 838static void i915_error_work_func(struct work_struct *work)
 839{
 840	drm_i915_private_t *dev_priv = container_of(work, drm_i915_private_t,
 841						    error_work);
 842	struct drm_device *dev = dev_priv->dev;
 843	char *error_event[] = { "ERROR=1", NULL };
 844	char *reset_event[] = { "RESET=1", NULL };
 845	char *reset_done_event[] = { "ERROR=0", NULL };
 846
 847	kobject_uevent_env(&dev->primary->kdev.kobj, KOBJ_CHANGE, error_event);
 848
 849	if (atomic_read(&dev_priv->mm.wedged)) {
 850		DRM_DEBUG_DRIVER("resetting chip\n");
 851		kobject_uevent_env(&dev->primary->kdev.kobj, KOBJ_CHANGE, reset_event);
 852		if (!i915_reset(dev)) {
 853			atomic_set(&dev_priv->mm.wedged, 0);
 854			kobject_uevent_env(&dev->primary->kdev.kobj, KOBJ_CHANGE, reset_done_event);
 855		}
 856		complete_all(&dev_priv->error_completion);
 857	}
 858}
 859
 860/* NB: please notice the memset */
 861static void i915_get_extra_instdone(struct drm_device *dev,
 862				    uint32_t *instdone)
 863{
 864	struct drm_i915_private *dev_priv = dev->dev_private;
 865	memset(instdone, 0, sizeof(*instdone) * I915_NUM_INSTDONE_REG);
 866
 867	switch(INTEL_INFO(dev)->gen) {
 868	case 2:
 869	case 3:
 870		instdone[0] = I915_READ(INSTDONE);
 871		break;
 872	case 4:
 873	case 5:
 874	case 6:
 875		instdone[0] = I915_READ(INSTDONE_I965);
 876		instdone[1] = I915_READ(INSTDONE1);
 877		break;
 878	default:
 879		WARN_ONCE(1, "Unsupported platform\n");
 880	case 7:
 881		instdone[0] = I915_READ(GEN7_INSTDONE_1);
 882		instdone[1] = I915_READ(GEN7_SC_INSTDONE);
 883		instdone[2] = I915_READ(GEN7_SAMPLER_INSTDONE);
 884		instdone[3] = I915_READ(GEN7_ROW_INSTDONE);
 885		break;
 886	}
 887}
 888
 889#ifdef CONFIG_DEBUG_FS
 890static struct drm_i915_error_object *
 891i915_error_object_create(struct drm_i915_private *dev_priv,
 892			 struct drm_i915_gem_object *src)
 893{
 894	struct drm_i915_error_object *dst;
 895	int i, count;
 896	u32 reloc_offset;
 897
 898	if (src == NULL || src->pages == NULL)
 899		return NULL;
 900
 901	count = src->base.size / PAGE_SIZE;
 902
 903	dst = kmalloc(sizeof(*dst) + count * sizeof(u32 *), GFP_ATOMIC);
 904	if (dst == NULL)
 905		return NULL;
 906
 907	reloc_offset = src->gtt_offset;
 908	for (i = 0; i < count; i++) {
 909		unsigned long flags;
 910		void *d;
 911
 912		d = kmalloc(PAGE_SIZE, GFP_ATOMIC);
 913		if (d == NULL)
 914			goto unwind;
 915
 916		local_irq_save(flags);
 917		if (reloc_offset < dev_priv->mm.gtt_mappable_end &&
 918		    src->has_global_gtt_mapping) {
 919			void __iomem *s;
 920
 921			/* Simply ignore tiling or any overlapping fence.
 922			 * It's part of the error state, and this hopefully
 923			 * captures what the GPU read.
 924			 */
 925
 926			s = io_mapping_map_atomic_wc(dev_priv->mm.gtt_mapping,
 927						     reloc_offset);
 928			memcpy_fromio(d, s, PAGE_SIZE);
 929			io_mapping_unmap_atomic(s);
 930		} else {
 931			struct page *page;
 932			void *s;
 933
 934			page = i915_gem_object_get_page(src, i);
 935
 936			drm_clflush_pages(&page, 1);
 937
 938			s = kmap_atomic(page);
 939			memcpy(d, s, PAGE_SIZE);
 940			kunmap_atomic(s);
 941
 942			drm_clflush_pages(&page, 1);
 943		}
 944		local_irq_restore(flags);
 945
 946		dst->pages[i] = d;
 947
 948		reloc_offset += PAGE_SIZE;
 949	}
 950	dst->page_count = count;
 951	dst->gtt_offset = src->gtt_offset;
 952
 953	return dst;
 954
 955unwind:
 956	while (i--)
 957		kfree(dst->pages[i]);
 958	kfree(dst);
 959	return NULL;
 960}
 961
 962static void
 963i915_error_object_free(struct drm_i915_error_object *obj)
 964{
 965	int page;
 966
 967	if (obj == NULL)
 968		return;
 969
 970	for (page = 0; page < obj->page_count; page++)
 971		kfree(obj->pages[page]);
 972
 973	kfree(obj);
 974}
 975
 976void
 977i915_error_state_free(struct kref *error_ref)
 978{
 979	struct drm_i915_error_state *error = container_of(error_ref,
 980							  typeof(*error), ref);
 981	int i;
 982
 983	for (i = 0; i < ARRAY_SIZE(error->ring); i++) {
 984		i915_error_object_free(error->ring[i].batchbuffer);
 985		i915_error_object_free(error->ring[i].ringbuffer);
 986		kfree(error->ring[i].requests);
 987	}
 988
 989	kfree(error->active_bo);
 990	kfree(error->overlay);
 991	kfree(error);
 992}
 993static void capture_bo(struct drm_i915_error_buffer *err,
 994		       struct drm_i915_gem_object *obj)
 995{
 996	err->size = obj->base.size;
 997	err->name = obj->base.name;
 998	err->rseqno = obj->last_read_seqno;
 999	err->wseqno = obj->last_write_seqno;
1000	err->gtt_offset = obj->gtt_offset;
1001	err->read_domains = obj->base.read_domains;
1002	err->write_domain = obj->base.write_domain;
1003	err->fence_reg = obj->fence_reg;
1004	err->pinned = 0;
1005	if (obj->pin_count > 0)
1006		err->pinned = 1;
1007	if (obj->user_pin_count > 0)
1008		err->pinned = -1;
1009	err->tiling = obj->tiling_mode;
1010	err->dirty = obj->dirty;
1011	err->purgeable = obj->madv != I915_MADV_WILLNEED;
1012	err->ring = obj->ring ? obj->ring->id : -1;
1013	err->cache_level = obj->cache_level;
1014}
1015
1016static u32 capture_active_bo(struct drm_i915_error_buffer *err,
1017			     int count, struct list_head *head)
1018{
1019	struct drm_i915_gem_object *obj;
1020	int i = 0;
1021
1022	list_for_each_entry(obj, head, mm_list) {
1023		capture_bo(err++, obj);
1024		if (++i == count)
1025			break;
1026	}
1027
1028	return i;
1029}
1030
1031static u32 capture_pinned_bo(struct drm_i915_error_buffer *err,
1032			     int count, struct list_head *head)
1033{
1034	struct drm_i915_gem_object *obj;
1035	int i = 0;
1036
1037	list_for_each_entry(obj, head, gtt_list) {
1038		if (obj->pin_count == 0)
1039			continue;
1040
1041		capture_bo(err++, obj);
1042		if (++i == count)
1043			break;
1044	}
1045
1046	return i;
1047}
1048
1049static void i915_gem_record_fences(struct drm_device *dev,
1050				   struct drm_i915_error_state *error)
1051{
1052	struct drm_i915_private *dev_priv = dev->dev_private;
1053	int i;
1054
1055	/* Fences */
1056	switch (INTEL_INFO(dev)->gen) {
1057	case 7:
1058	case 6:
1059		for (i = 0; i < 16; i++)
1060			error->fence[i] = I915_READ64(FENCE_REG_SANDYBRIDGE_0 + (i * 8));
1061		break;
1062	case 5:
1063	case 4:
1064		for (i = 0; i < 16; i++)
1065			error->fence[i] = I915_READ64(FENCE_REG_965_0 + (i * 8));
1066		break;
1067	case 3:
1068		if (IS_I945G(dev) || IS_I945GM(dev) || IS_G33(dev))
1069			for (i = 0; i < 8; i++)
1070				error->fence[i+8] = I915_READ(FENCE_REG_945_8 + (i * 4));
1071	case 2:
1072		for (i = 0; i < 8; i++)
1073			error->fence[i] = I915_READ(FENCE_REG_830_0 + (i * 4));
1074		break;
1075
1076	}
1077}
1078
1079static struct drm_i915_error_object *
1080i915_error_first_batchbuffer(struct drm_i915_private *dev_priv,
1081			     struct intel_ring_buffer *ring)
1082{
1083	struct drm_i915_gem_object *obj;
1084	u32 seqno;
1085
1086	if (!ring->get_seqno)
1087		return NULL;
1088
1089	if (HAS_BROKEN_CS_TLB(dev_priv->dev)) {
1090		u32 acthd = I915_READ(ACTHD);
1091
1092		if (WARN_ON(ring->id != RCS))
1093			return NULL;
1094
1095		obj = ring->private;
1096		if (acthd >= obj->gtt_offset &&
1097		    acthd < obj->gtt_offset + obj->base.size)
1098			return i915_error_object_create(dev_priv, obj);
1099	}
1100
1101	seqno = ring->get_seqno(ring, false);
1102	list_for_each_entry(obj, &dev_priv->mm.active_list, mm_list) {
1103		if (obj->ring != ring)
1104			continue;
1105
1106		if (i915_seqno_passed(seqno, obj->last_read_seqno))
1107			continue;
1108
1109		if ((obj->base.read_domains & I915_GEM_DOMAIN_COMMAND) == 0)
1110			continue;
1111
1112		/* We need to copy these to an anonymous buffer as the simplest
1113		 * method to avoid being overwritten by userspace.
1114		 */
1115		return i915_error_object_create(dev_priv, obj);
1116	}
1117
1118	return NULL;
1119}
1120
1121static void i915_record_ring_state(struct drm_device *dev,
1122				   struct drm_i915_error_state *error,
1123				   struct intel_ring_buffer *ring)
1124{
1125	struct drm_i915_private *dev_priv = dev->dev_private;
1126
1127	if (INTEL_INFO(dev)->gen >= 6) {
1128		error->rc_psmi[ring->id] = I915_READ(ring->mmio_base + 0x50);
1129		error->fault_reg[ring->id] = I915_READ(RING_FAULT_REG(ring));
1130		error->semaphore_mboxes[ring->id][0]
1131			= I915_READ(RING_SYNC_0(ring->mmio_base));
1132		error->semaphore_mboxes[ring->id][1]
1133			= I915_READ(RING_SYNC_1(ring->mmio_base));
1134		error->semaphore_seqno[ring->id][0] = ring->sync_seqno[0];
1135		error->semaphore_seqno[ring->id][1] = ring->sync_seqno[1];
1136	}
1137
1138	if (INTEL_INFO(dev)->gen >= 4) {
1139		error->faddr[ring->id] = I915_READ(RING_DMA_FADD(ring->mmio_base));
1140		error->ipeir[ring->id] = I915_READ(RING_IPEIR(ring->mmio_base));
1141		error->ipehr[ring->id] = I915_READ(RING_IPEHR(ring->mmio_base));
1142		error->instdone[ring->id] = I915_READ(RING_INSTDONE(ring->mmio_base));
1143		error->instps[ring->id] = I915_READ(RING_INSTPS(ring->mmio_base));
1144		if (ring->id == RCS)
1145			error->bbaddr = I915_READ64(BB_ADDR);
1146	} else {
1147		error->faddr[ring->id] = I915_READ(DMA_FADD_I8XX);
1148		error->ipeir[ring->id] = I915_READ(IPEIR);
1149		error->ipehr[ring->id] = I915_READ(IPEHR);
1150		error->instdone[ring->id] = I915_READ(INSTDONE);
1151	}
1152
1153	error->waiting[ring->id] = waitqueue_active(&ring->irq_queue);
1154	error->instpm[ring->id] = I915_READ(RING_INSTPM(ring->mmio_base));
1155	error->seqno[ring->id] = ring->get_seqno(ring, false);
1156	error->acthd[ring->id] = intel_ring_get_active_head(ring);
1157	error->head[ring->id] = I915_READ_HEAD(ring);
1158	error->tail[ring->id] = I915_READ_TAIL(ring);
1159	error->ctl[ring->id] = I915_READ_CTL(ring);
1160
1161	error->cpu_ring_head[ring->id] = ring->head;
1162	error->cpu_ring_tail[ring->id] = ring->tail;
1163}
1164
1165static void i915_gem_record_rings(struct drm_device *dev,
1166				  struct drm_i915_error_state *error)
1167{
1168	struct drm_i915_private *dev_priv = dev->dev_private;
1169	struct intel_ring_buffer *ring;
1170	struct drm_i915_gem_request *request;
1171	int i, count;
1172
1173	for_each_ring(ring, dev_priv, i) {
1174		i915_record_ring_state(dev, error, ring);
1175
1176		error->ring[i].batchbuffer =
1177			i915_error_first_batchbuffer(dev_priv, ring);
1178
1179		error->ring[i].ringbuffer =
1180			i915_error_object_create(dev_priv, ring->obj);
1181
1182		count = 0;
1183		list_for_each_entry(request, &ring->request_list, list)
1184			count++;
1185
1186		error->ring[i].num_requests = count;
1187		error->ring[i].requests =
1188			kmalloc(count*sizeof(struct drm_i915_error_request),
1189				GFP_ATOMIC);
1190		if (error->ring[i].requests == NULL) {
1191			error->ring[i].num_requests = 0;
1192			continue;
1193		}
1194
1195		count = 0;
1196		list_for_each_entry(request, &ring->request_list, list) {
1197			struct drm_i915_error_request *erq;
1198
1199			erq = &error->ring[i].requests[count++];
1200			erq->seqno = request->seqno;
1201			erq->jiffies = request->emitted_jiffies;
1202			erq->tail = request->tail;
1203		}
1204	}
1205}
1206
1207/**
1208 * i915_capture_error_state - capture an error record for later analysis
1209 * @dev: drm device
1210 *
1211 * Should be called when an error is detected (either a hang or an error
1212 * interrupt) to capture error state from the time of the error.  Fills
1213 * out a structure which becomes available in debugfs for user level tools
1214 * to pick up.
1215 */
1216static void i915_capture_error_state(struct drm_device *dev)
1217{
1218	struct drm_i915_private *dev_priv = dev->dev_private;
1219	struct drm_i915_gem_object *obj;
1220	struct drm_i915_error_state *error;
1221	unsigned long flags;
1222	int i, pipe;
1223
1224	spin_lock_irqsave(&dev_priv->error_lock, flags);
1225	error = dev_priv->first_error;
1226	spin_unlock_irqrestore(&dev_priv->error_lock, flags);
1227	if (error)
1228		return;
1229
1230	/* Account for pipe specific data like PIPE*STAT */
1231	error = kzalloc(sizeof(*error), GFP_ATOMIC);
1232	if (!error) {
1233		DRM_DEBUG_DRIVER("out of memory, not capturing error state\n");
1234		return;
1235	}
1236
1237	DRM_INFO("capturing error event; look for more information in /debug/dri/%d/i915_error_state\n",
1238		 dev->primary->index);
1239
1240	kref_init(&error->ref);
1241	error->eir = I915_READ(EIR);
1242	error->pgtbl_er = I915_READ(PGTBL_ER);
1243	error->ccid = I915_READ(CCID);
1244
1245	if (HAS_PCH_SPLIT(dev))
1246		error->ier = I915_READ(DEIER) | I915_READ(GTIER);
1247	else if (IS_VALLEYVIEW(dev))
1248		error->ier = I915_READ(GTIER) | I915_READ(VLV_IER);
1249	else if (IS_GEN2(dev))
1250		error->ier = I915_READ16(IER);
1251	else
1252		error->ier = I915_READ(IER);
1253
1254	if (INTEL_INFO(dev)->gen >= 6)
1255		error->derrmr = I915_READ(DERRMR);
1256
1257	if (IS_VALLEYVIEW(dev))
1258		error->forcewake = I915_READ(FORCEWAKE_VLV);
1259	else if (INTEL_INFO(dev)->gen >= 7)
1260		error->forcewake = I915_READ(FORCEWAKE_MT);
1261	else if (INTEL_INFO(dev)->gen == 6)
1262		error->forcewake = I915_READ(FORCEWAKE);
1263
1264	for_each_pipe(pipe)
1265		error->pipestat[pipe] = I915_READ(PIPESTAT(pipe));
1266
1267	if (INTEL_INFO(dev)->gen >= 6) {
1268		error->error = I915_READ(ERROR_GEN6);
1269		error->done_reg = I915_READ(DONE_REG);
1270	}
1271
1272	if (INTEL_INFO(dev)->gen == 7)
1273		error->err_int = I915_READ(GEN7_ERR_INT);
1274
1275	i915_get_extra_instdone(dev, error->extra_instdone);
1276
1277	i915_gem_record_fences(dev, error);
1278	i915_gem_record_rings(dev, error);
1279
1280	/* Record buffers on the active and pinned lists. */
1281	error->active_bo = NULL;
1282	error->pinned_bo = NULL;
1283
1284	i = 0;
1285	list_for_each_entry(obj, &dev_priv->mm.active_list, mm_list)
1286		i++;
1287	error->active_bo_count = i;
1288	list_for_each_entry(obj, &dev_priv->mm.bound_list, gtt_list)
1289		if (obj->pin_count)
1290			i++;
1291	error->pinned_bo_count = i - error->active_bo_count;
1292
1293	error->active_bo = NULL;
1294	error->pinned_bo = NULL;
1295	if (i) {
1296		error->active_bo = kmalloc(sizeof(*error->active_bo)*i,
1297					   GFP_ATOMIC);
1298		if (error->active_bo)
1299			error->pinned_bo =
1300				error->active_bo + error->active_bo_count;
1301	}
1302
1303	if (error->active_bo)
1304		error->active_bo_count =
1305			capture_active_bo(error->active_bo,
1306					  error->active_bo_count,
1307					  &dev_priv->mm.active_list);
1308
1309	if (error->pinned_bo)
1310		error->pinned_bo_count =
1311			capture_pinned_bo(error->pinned_bo,
1312					  error->pinned_bo_count,
1313					  &dev_priv->mm.bound_list);
1314
1315	do_gettimeofday(&error->time);
1316
1317	error->overlay = intel_overlay_capture_error_state(dev);
1318	error->display = intel_display_capture_error_state(dev);
1319
1320	spin_lock_irqsave(&dev_priv->error_lock, flags);
1321	if (dev_priv->first_error == NULL) {
1322		dev_priv->first_error = error;
1323		error = NULL;
1324	}
1325	spin_unlock_irqrestore(&dev_priv->error_lock, flags);
1326
1327	if (error)
1328		i915_error_state_free(&error->ref);
1329}
1330
1331void i915_destroy_error_state(struct drm_device *dev)
1332{
1333	struct drm_i915_private *dev_priv = dev->dev_private;
1334	struct drm_i915_error_state *error;
1335	unsigned long flags;
1336
1337	spin_lock_irqsave(&dev_priv->error_lock, flags);
1338	error = dev_priv->first_error;
1339	dev_priv->first_error = NULL;
1340	spin_unlock_irqrestore(&dev_priv->error_lock, flags);
1341
1342	if (error)
1343		kref_put(&error->ref, i915_error_state_free);
1344}
1345#else
1346#define i915_capture_error_state(x)
1347#endif
1348
1349static void i915_report_and_clear_eir(struct drm_device *dev)
1350{
1351	struct drm_i915_private *dev_priv = dev->dev_private;
1352	uint32_t instdone[I915_NUM_INSTDONE_REG];
1353	u32 eir = I915_READ(EIR);
1354	int pipe, i;
1355
1356	if (!eir)
1357		return;
1358
1359	pr_err("render error detected, EIR: 0x%08x\n", eir);
1360
1361	i915_get_extra_instdone(dev, instdone);
1362
1363	if (IS_G4X(dev)) {
1364		if (eir & (GM45_ERROR_MEM_PRIV | GM45_ERROR_CP_PRIV)) {
1365			u32 ipeir = I915_READ(IPEIR_I965);
1366
1367			pr_err("  IPEIR: 0x%08x\n", I915_READ(IPEIR_I965));
1368			pr_err("  IPEHR: 0x%08x\n", I915_READ(IPEHR_I965));
1369			for (i = 0; i < ARRAY_SIZE(instdone); i++)
1370				pr_err("  INSTDONE_%d: 0x%08x\n", i, instdone[i]);
1371			pr_err("  INSTPS: 0x%08x\n", I915_READ(INSTPS));
1372			pr_err("  ACTHD: 0x%08x\n", I915_READ(ACTHD_I965));
1373			I915_WRITE(IPEIR_I965, ipeir);
1374			POSTING_READ(IPEIR_I965);
1375		}
1376		if (eir & GM45_ERROR_PAGE_TABLE) {
1377			u32 pgtbl_err = I915_READ(PGTBL_ER);
1378			pr_err("page table error\n");
1379			pr_err("  PGTBL_ER: 0x%08x\n", pgtbl_err);
1380			I915_WRITE(PGTBL_ER, pgtbl_err);
1381			POSTING_READ(PGTBL_ER);
1382		}
1383	}
1384
1385	if (!IS_GEN2(dev)) {
1386		if (eir & I915_ERROR_PAGE_TABLE) {
1387			u32 pgtbl_err = I915_READ(PGTBL_ER);
1388			pr_err("page table error\n");
1389			pr_err("  PGTBL_ER: 0x%08x\n", pgtbl_err);
1390			I915_WRITE(PGTBL_ER, pgtbl_err);
1391			POSTING_READ(PGTBL_ER);
1392		}
1393	}
1394
1395	if (eir & I915_ERROR_MEMORY_REFRESH) {
1396		pr_err("memory refresh error:\n");
1397		for_each_pipe(pipe)
1398			pr_err("pipe %c stat: 0x%08x\n",
1399			       pipe_name(pipe), I915_READ(PIPESTAT(pipe)));
1400		/* pipestat has already been acked */
1401	}
1402	if (eir & I915_ERROR_INSTRUCTION) {
1403		pr_err("instruction error\n");
1404		pr_err("  INSTPM: 0x%08x\n", I915_READ(INSTPM));
1405		for (i = 0; i < ARRAY_SIZE(instdone); i++)
1406			pr_err("  INSTDONE_%d: 0x%08x\n", i, instdone[i]);
1407		if (INTEL_INFO(dev)->gen < 4) {
1408			u32 ipeir = I915_READ(IPEIR);
1409
1410			pr_err("  IPEIR: 0x%08x\n", I915_READ(IPEIR));
1411			pr_err("  IPEHR: 0x%08x\n", I915_READ(IPEHR));
1412			pr_err("  ACTHD: 0x%08x\n", I915_READ(ACTHD));
1413			I915_WRITE(IPEIR, ipeir);
1414			POSTING_READ(IPEIR);
1415		} else {
1416			u32 ipeir = I915_READ(IPEIR_I965);
1417
1418			pr_err("  IPEIR: 0x%08x\n", I915_READ(IPEIR_I965));
1419			pr_err("  IPEHR: 0x%08x\n", I915_READ(IPEHR_I965));
1420			pr_err("  INSTPS: 0x%08x\n", I915_READ(INSTPS));
1421			pr_err("  ACTHD: 0x%08x\n", I915_READ(ACTHD_I965));
1422			I915_WRITE(IPEIR_I965, ipeir);
1423			POSTING_READ(IPEIR_I965);
1424		}
1425	}
1426
1427	I915_WRITE(EIR, eir);
1428	POSTING_READ(EIR);
1429	eir = I915_READ(EIR);
1430	if (eir) {
1431		/*
1432		 * some errors might have become stuck,
1433		 * mask them.
1434		 */
1435		DRM_ERROR("EIR stuck: 0x%08x, masking\n", eir);
1436		I915_WRITE(EMR, I915_READ(EMR) | eir);
1437		I915_WRITE(IIR, I915_RENDER_COMMAND_PARSER_ERROR_INTERRUPT);
1438	}
1439}
1440
1441/**
1442 * i915_handle_error - handle an error interrupt
1443 * @dev: drm device
1444 *
1445 * Do some basic checking of regsiter state at error interrupt time and
1446 * dump it to the syslog.  Also call i915_capture_error_state() to make
1447 * sure we get a record and make it available in debugfs.  Fire a uevent
1448 * so userspace knows something bad happened (should trigger collection
1449 * of a ring dump etc.).
1450 */
1451void i915_handle_error(struct drm_device *dev, bool wedged)
1452{
1453	struct drm_i915_private *dev_priv = dev->dev_private;
1454	struct intel_ring_buffer *ring;
1455	int i;
1456
1457	i915_capture_error_state(dev);
1458	i915_report_and_clear_eir(dev);
1459
1460	if (wedged) {
1461		INIT_COMPLETION(dev_priv->error_completion);
1462		atomic_set(&dev_priv->mm.wedged, 1);
1463
1464		/*
1465		 * Wakeup waiting processes so they don't hang
1466		 */
1467		for_each_ring(ring, dev_priv, i)
1468			wake_up_all(&ring->irq_queue);
1469	}
1470
1471	queue_work(dev_priv->wq, &dev_priv->error_work);
1472}
1473
1474static void i915_pageflip_stall_check(struct drm_device *dev, int pipe)
1475{
1476	drm_i915_private_t *dev_priv = dev->dev_private;
1477	struct drm_crtc *crtc = dev_priv->pipe_to_crtc_mapping[pipe];
1478	struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
1479	struct drm_i915_gem_object *obj;
1480	struct intel_unpin_work *work;
1481	unsigned long flags;
1482	bool stall_detected;
1483
1484	/* Ignore early vblank irqs */
1485	if (intel_crtc == NULL)
1486		return;
1487
1488	spin_lock_irqsave(&dev->event_lock, flags);
1489	work = intel_crtc->unpin_work;
1490
1491	if (work == NULL ||
1492	    atomic_read(&work->pending) >= INTEL_FLIP_COMPLETE ||
1493	    !work->enable_stall_check) {
1494		/* Either the pending flip IRQ arrived, or we're too early. Don't check */
1495		spin_unlock_irqrestore(&dev->event_lock, flags);
1496		return;
1497	}
1498
1499	/* Potential stall - if we see that the flip has happened, assume a missed interrupt */
1500	obj = work->pending_flip_obj;
1501	if (INTEL_INFO(dev)->gen >= 4) {
1502		int dspsurf = DSPSURF(intel_crtc->plane);
1503		stall_detected = I915_HI_DISPBASE(I915_READ(dspsurf)) ==
1504					obj->gtt_offset;
1505	} else {
1506		int dspaddr = DSPADDR(intel_crtc->plane);
1507		stall_detected = I915_READ(dspaddr) == (obj->gtt_offset +
1508							crtc->y * crtc->fb->pitches[0] +
1509							crtc->x * crtc->fb->bits_per_pixel/8);
1510	}
1511
1512	spin_unlock_irqrestore(&dev->event_lock, flags);
1513
1514	if (stall_detected) {
1515		DRM_DEBUG_DRIVER("Pageflip stall detected\n");
1516		intel_prepare_page_flip(dev, intel_crtc->plane);
1517	}
1518}
1519
1520/* Called from drm generic code, passed 'crtc' which
1521 * we use as a pipe index
1522 */
1523static int i915_enable_vblank(struct drm_device *dev, int pipe)
1524{
1525	drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
1526	unsigned long irqflags;
1527
1528	if (!i915_pipe_enabled(dev, pipe))
1529		return -EINVAL;
1530
1531	spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
1532	if (INTEL_INFO(dev)->gen >= 4)
1533		i915_enable_pipestat(dev_priv, pipe,
1534				     PIPE_START_VBLANK_INTERRUPT_ENABLE);
1535	else
1536		i915_enable_pipestat(dev_priv, pipe,
1537				     PIPE_VBLANK_INTERRUPT_ENABLE);
1538
1539	/* maintain vblank delivery even in deep C-states */
1540	if (dev_priv->info->gen == 3)
1541		I915_WRITE(INSTPM, _MASKED_BIT_DISABLE(INSTPM_AGPBUSY_DIS));
1542	spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
1543
1544	return 0;
1545}
1546
1547static int ironlake_enable_vblank(struct drm_device *dev, int pipe)
1548{
1549	drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
1550	unsigned long irqflags;
1551
1552	if (!i915_pipe_enabled(dev, pipe))
1553		return -EINVAL;
1554
1555	spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
1556	ironlake_enable_display_irq(dev_priv, (pipe == 0) ?
1557				    DE_PIPEA_VBLANK : DE_PIPEB_VBLANK);
1558	spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
1559
1560	return 0;
1561}
1562
1563static int ivybridge_enable_vblank(struct drm_device *dev, int pipe)
1564{
1565	drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
1566	unsigned long irqflags;
1567
1568	if (!i915_pipe_enabled(dev, pipe))
1569		return -EINVAL;
1570
1571	spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
1572	ironlake_enable_display_irq(dev_priv,
1573				    DE_PIPEA_VBLANK_IVB << (5 * pipe));
1574	spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
1575
1576	return 0;
1577}
1578
1579static int valleyview_enable_vblank(struct drm_device *dev, int pipe)
1580{
1581	drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
1582	unsigned long irqflags;
1583	u32 imr;
1584
1585	if (!i915_pipe_enabled(dev, pipe))
1586		return -EINVAL;
1587
1588	spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
1589	imr = I915_READ(VLV_IMR);
1590	if (pipe == 0)
1591		imr &= ~I915_DISPLAY_PIPE_A_VBLANK_INTERRUPT;
1592	else
1593		imr &= ~I915_DISPLAY_PIPE_B_VBLANK_INTERRUPT;
1594	I915_WRITE(VLV_IMR, imr);
1595	i915_enable_pipestat(dev_priv, pipe,
1596			     PIPE_START_VBLANK_INTERRUPT_ENABLE);
1597	spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
1598
1599	return 0;
1600}
1601
1602/* Called from drm generic code, passed 'crtc' which
1603 * we use as a pipe index
1604 */
1605static void i915_disable_vblank(struct drm_device *dev, int pipe)
1606{
1607	drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
1608	unsigned long irqflags;
1609
1610	spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
1611	if (dev_priv->info->gen == 3)
1612		I915_WRITE(INSTPM, _MASKED_BIT_ENABLE(INSTPM_AGPBUSY_DIS));
1613
1614	i915_disable_pipestat(dev_priv, pipe,
1615			      PIPE_VBLANK_INTERRUPT_ENABLE |
1616			      PIPE_START_VBLANK_INTERRUPT_ENABLE);
1617	spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
1618}
1619
1620static void ironlake_disable_vblank(struct drm_device *dev, int pipe)
1621{
1622	drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
1623	unsigned long irqflags;
1624
1625	spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
1626	ironlake_disable_display_irq(dev_priv, (pipe == 0) ?
1627				     DE_PIPEA_VBLANK : DE_PIPEB_VBLANK);
1628	spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
1629}
1630
1631static void ivybridge_disable_vblank(struct drm_device *dev, int pipe)
1632{
1633	drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
1634	unsigned long irqflags;
1635
1636	spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
1637	ironlake_disable_display_irq(dev_priv,
1638				     DE_PIPEA_VBLANK_IVB << (pipe * 5));
1639	spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
1640}
1641
1642static void valleyview_disable_vblank(struct drm_device *dev, int pipe)
1643{
1644	drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
1645	unsigned long irqflags;
1646	u32 imr;
1647
1648	spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
1649	i915_disable_pipestat(dev_priv, pipe,
1650			      PIPE_START_VBLANK_INTERRUPT_ENABLE);
1651	imr = I915_READ(VLV_IMR);
1652	if (pipe == 0)
1653		imr |= I915_DISPLAY_PIPE_A_VBLANK_INTERRUPT;
1654	else
1655		imr |= I915_DISPLAY_PIPE_B_VBLANK_INTERRUPT;
1656	I915_WRITE(VLV_IMR, imr);
1657	spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
1658}
1659
1660static u32
1661ring_last_seqno(struct intel_ring_buffer *ring)
1662{
1663	return list_entry(ring->request_list.prev,
1664			  struct drm_i915_gem_request, list)->seqno;
1665}
1666
1667static bool i915_hangcheck_ring_idle(struct intel_ring_buffer *ring, bool *err)
1668{
1669	if (list_empty(&ring->request_list) ||
1670	    i915_seqno_passed(ring->get_seqno(ring, false),
1671			      ring_last_seqno(ring))) {
1672		/* Issue a wake-up to catch stuck h/w. */
1673		if (waitqueue_active(&ring->irq_queue)) {
1674			DRM_ERROR("Hangcheck timer elapsed... %s idle\n",
1675				  ring->name);
1676			wake_up_all(&ring->irq_queue);
1677			*err = true;
1678		}
1679		return true;
1680	}
1681	return false;
1682}
1683
1684static bool kick_ring(struct intel_ring_buffer *ring)
1685{
1686	struct drm_device *dev = ring->dev;
1687	struct drm_i915_private *dev_priv = dev->dev_private;
1688	u32 tmp = I915_READ_CTL(ring);
1689	if (tmp & RING_WAIT) {
1690		DRM_ERROR("Kicking stuck wait on %s\n",
1691			  ring->name);
1692		I915_WRITE_CTL(ring, tmp);
1693		return true;
1694	}
1695	return false;
1696}
1697
1698static bool i915_hangcheck_hung(struct drm_device *dev)
1699{
1700	drm_i915_private_t *dev_priv = dev->dev_private;
1701
1702	if (dev_priv->hangcheck_count++ > 1) {
1703		bool hung = true;
1704
1705		DRM_ERROR("Hangcheck timer elapsed... GPU hung\n");
1706		i915_handle_error(dev, true);
1707
1708		if (!IS_GEN2(dev)) {
1709			struct intel_ring_buffer *ring;
1710			int i;
1711
1712			/* Is the chip hanging on a WAIT_FOR_EVENT?
1713			 * If so we can simply poke the RB_WAIT bit
1714			 * and break the hang. This should work on
1715			 * all but the second generation chipsets.
1716			 */
1717			for_each_ring(ring, dev_priv, i)
1718				hung &= !kick_ring(ring);
1719		}
1720
1721		return hung;
1722	}
1723
1724	return false;
1725}
1726
1727/**
1728 * This is called when the chip hasn't reported back with completed
1729 * batchbuffers in a long time. The first time this is called we simply record
1730 * ACTHD. If ACTHD hasn't changed by the time the hangcheck timer elapses
1731 * again, we assume the chip is wedged and try to fix it.
1732 */
1733void i915_hangcheck_elapsed(unsigned long data)
1734{
1735	struct drm_device *dev = (struct drm_device *)data;
1736	drm_i915_private_t *dev_priv = dev->dev_private;
1737	uint32_t acthd[I915_NUM_RINGS], instdone[I915_NUM_INSTDONE_REG];
1738	struct intel_ring_buffer *ring;
1739	bool err = false, idle;
1740	int i;
1741
1742	if (!i915_enable_hangcheck)
1743		return;
1744
1745	memset(acthd, 0, sizeof(acthd));
1746	idle = true;
1747	for_each_ring(ring, dev_priv, i) {
1748	    idle &= i915_hangcheck_ring_idle(ring, &err);
1749	    acthd[i] = intel_ring_get_active_head(ring);
1750	}
1751
1752	/* If all work is done then ACTHD clearly hasn't advanced. */
1753	if (idle) {
1754		if (err) {
1755			if (i915_hangcheck_hung(dev))
1756				return;
1757
1758			goto repeat;
1759		}
1760
1761		dev_priv->hangcheck_count = 0;
1762		return;
1763	}
1764
1765	i915_get_extra_instdone(dev, instdone);
1766	if (memcmp(dev_priv->last_acthd, acthd, sizeof(acthd)) == 0 &&
1767	    memcmp(dev_priv->prev_instdone, instdone, sizeof(instdone)) == 0) {
1768		if (i915_hangcheck_hung(dev))
1769			return;
1770	} else {
1771		dev_priv->hangcheck_count = 0;
1772
1773		memcpy(dev_priv->last_acthd, acthd, sizeof(acthd));
1774		memcpy(dev_priv->prev_instdone, instdone, sizeof(instdone));
1775	}
1776
1777repeat:
1778	/* Reset timer case chip hangs without another request being added */
1779	mod_timer(&dev_priv->hangcheck_timer,
1780		  round_jiffies_up(jiffies + DRM_I915_HANGCHECK_JIFFIES));
1781}
1782
1783/* drm_dma.h hooks
1784*/
1785static void ironlake_irq_preinstall(struct drm_device *dev)
1786{
1787	drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
1788
1789	atomic_set(&dev_priv->irq_received, 0);
1790
1791	I915_WRITE(HWSTAM, 0xeffe);
1792
1793	/* XXX hotplug from PCH */
1794
1795	I915_WRITE(DEIMR, 0xffffffff);
1796	I915_WRITE(DEIER, 0x0);
1797	POSTING_READ(DEIER);
1798
1799	/* and GT */
1800	I915_WRITE(GTIMR, 0xffffffff);
1801	I915_WRITE(GTIER, 0x0);
1802	POSTING_READ(GTIER);
1803
1804	/* south display irq */
1805	I915_WRITE(SDEIMR, 0xffffffff);
1806	I915_WRITE(SDEIER, 0x0);
1807	POSTING_READ(SDEIER);
1808}
1809
1810static void valleyview_irq_preinstall(struct drm_device *dev)
1811{
1812	drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
1813	int pipe;
1814
1815	atomic_set(&dev_priv->irq_received, 0);
1816
1817	/* VLV magic */
1818	I915_WRITE(VLV_IMR, 0);
1819	I915_WRITE(RING_IMR(RENDER_RING_BASE), 0);
1820	I915_WRITE(RING_IMR(GEN6_BSD_RING_BASE), 0);
1821	I915_WRITE(RING_IMR(BLT_RING_BASE), 0);
1822
1823	/* and GT */
1824	I915_WRITE(GTIIR, I915_READ(GTIIR));
1825	I915_WRITE(GTIIR, I915_READ(GTIIR));
1826	I915_WRITE(GTIMR, 0xffffffff);
1827	I915_WRITE(GTIER, 0x0);
1828	POSTING_READ(GTIER);
1829
1830	I915_WRITE(DPINVGTT, 0xff);
1831
1832	I915_WRITE(PORT_HOTPLUG_EN, 0);
1833	I915_WRITE(PORT_HOTPLUG_STAT, I915_READ(PORT_HOTPLUG_STAT));
1834	for_each_pipe(pipe)
1835		I915_WRITE(PIPESTAT(pipe), 0xffff);
1836	I915_WRITE(VLV_IIR, 0xffffffff);
1837	I915_WRITE(VLV_IMR, 0xffffffff);
1838	I915_WRITE(VLV_IER, 0x0);
1839	POSTING_READ(VLV_IER);
1840}
1841
1842/*
1843 * Enable digital hotplug on the PCH, and configure the DP short pulse
1844 * duration to 2ms (which is the minimum in the Display Port spec)
1845 *
1846 * This register is the same on all known PCH chips.
1847 */
1848
1849static void ironlake_enable_pch_hotplug(struct drm_device *dev)
1850{
1851	drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
1852	u32	hotplug;
1853
1854	hotplug = I915_READ(PCH_PORT_HOTPLUG);
1855	hotplug &= ~(PORTD_PULSE_DURATION_MASK|PORTC_PULSE_DURATION_MASK|PORTB_PULSE_DURATION_MASK);
1856	hotplug |= PORTD_HOTPLUG_ENABLE | PORTD_PULSE_DURATION_2ms;
1857	hotplug |= PORTC_HOTPLUG_ENABLE | PORTC_PULSE_DURATION_2ms;
1858	hotplug |= PORTB_HOTPLUG_ENABLE | PORTB_PULSE_DURATION_2ms;
1859	I915_WRITE(PCH_PORT_HOTPLUG, hotplug);
1860}
1861
1862static int ironlake_irq_postinstall(struct drm_device *dev)
1863{
1864	drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
1865	/* enable kind of interrupts always enabled */
1866	u32 display_mask = DE_MASTER_IRQ_CONTROL | DE_GSE | DE_PCH_EVENT |
1867			   DE_PLANEA_FLIP_DONE | DE_PLANEB_FLIP_DONE;
1868	u32 render_irqs;
1869	u32 hotplug_mask;
1870
1871	dev_priv->irq_mask = ~display_mask;
1872
1873	/* should always can generate irq */
1874	I915_WRITE(DEIIR, I915_READ(DEIIR));
1875	I915_WRITE(DEIMR, dev_priv->irq_mask);
1876	I915_WRITE(DEIER, display_mask | DE_PIPEA_VBLANK | DE_PIPEB_VBLANK);
1877	POSTING_READ(DEIER);
1878
1879	dev_priv->gt_irq_mask = ~0;
1880
1881	I915_WRITE(GTIIR, I915_READ(GTIIR));
1882	I915_WRITE(GTIMR, dev_priv->gt_irq_mask);
1883
1884	if (IS_GEN6(dev))
1885		render_irqs =
1886			GT_USER_INTERRUPT |
1887			GEN6_BSD_USER_INTERRUPT |
1888			GEN6_BLITTER_USER_INTERRUPT;
1889	else
1890		render_irqs =
1891			GT_USER_INTERRUPT |
1892			GT_PIPE_NOTIFY |
1893			GT_BSD_USER_INTERRUPT;
1894	I915_WRITE(GTIER, render_irqs);
1895	POSTING_READ(GTIER);
1896
1897	if (HAS_PCH_CPT(dev)) {
1898		hotplug_mask = (SDE_CRT_HOTPLUG_CPT |
1899				SDE_PORTB_HOTPLUG_CPT |
1900				SDE_PORTC_HOTPLUG_CPT |
1901				SDE_PORTD_HOTPLUG_CPT);
1902	} else {
1903		hotplug_mask = (SDE_CRT_HOTPLUG |
1904				SDE_PORTB_HOTPLUG |
1905				SDE_PORTC_HOTPLUG |
1906				SDE_PORTD_HOTPLUG |
1907				SDE_AUX_MASK);
1908	}
1909
1910	dev_priv->pch_irq_mask = ~hotplug_mask;
1911
1912	I915_WRITE(SDEIIR, I915_READ(SDEIIR));
1913	I915_WRITE(SDEIMR, dev_priv->pch_irq_mask);
1914	I915_WRITE(SDEIER, hotplug_mask);
1915	POSTING_READ(SDEIER);
1916
1917	ironlake_enable_pch_hotplug(dev);
1918
1919	if (IS_IRONLAKE_M(dev)) {
1920		/* Clear & enable PCU event interrupts */
1921		I915_WRITE(DEIIR, DE_PCU_EVENT);
1922		I915_WRITE(DEIER, I915_READ(DEIER) | DE_PCU_EVENT);
1923		ironlake_enable_display_irq(dev_priv, DE_PCU_EVENT);
1924	}
1925
1926	return 0;
1927}
1928
1929static int ivybridge_irq_postinstall(struct drm_device *dev)
1930{
1931	drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
1932	/* enable kind of interrupts always enabled */
1933	u32 display_mask =
1934		DE_MASTER_IRQ_CONTROL | DE_GSE_IVB | DE_PCH_EVENT_IVB |
1935		DE_PLANEC_FLIP_DONE_IVB |
1936		DE_PLANEB_FLIP_DONE_IVB |
1937		DE_PLANEA_FLIP_DONE_IVB;
1938	u32 render_irqs;
1939	u32 hotplug_mask;
1940
1941	dev_priv->irq_mask = ~display_mask;
1942
1943	/* should always can generate irq */
1944	I915_WRITE(DEIIR, I915_READ(DEIIR));
1945	I915_WRITE(DEIMR, dev_priv->irq_mask);
1946	I915_WRITE(DEIER,
1947		   display_mask |
1948		   DE_PIPEC_VBLANK_IVB |
1949		   DE_PIPEB_VBLANK_IVB |
1950		   DE_PIPEA_VBLANK_IVB);
1951	POSTING_READ(DEIER);
1952
1953	dev_priv->gt_irq_mask = ~GT_GEN7_L3_PARITY_ERROR_INTERRUPT;
1954
1955	I915_WRITE(GTIIR, I915_READ(GTIIR));
1956	I915_WRITE(GTIMR, dev_priv->gt_irq_mask);
1957
1958	render_irqs = GT_USER_INTERRUPT | GEN6_BSD_USER_INTERRUPT |
1959		GEN6_BLITTER_USER_INTERRUPT | GT_GEN7_L3_PARITY_ERROR_INTERRUPT;
1960	I915_WRITE(GTIER, render_irqs);
1961	POSTING_READ(GTIER);
1962
1963	hotplug_mask = (SDE_CRT_HOTPLUG_CPT |
1964			SDE_PORTB_HOTPLUG_CPT |
1965			SDE_PORTC_HOTPLUG_CPT |
1966			SDE_PORTD_HOTPLUG_CPT);
1967	dev_priv->pch_irq_mask = ~hotplug_mask;
1968
1969	I915_WRITE(SDEIIR, I915_READ(SDEIIR));
1970	I915_WRITE(SDEIMR, dev_priv->pch_irq_mask);
1971	I915_WRITE(SDEIER, hotplug_mask);
1972	POSTING_READ(SDEIER);
1973
1974	ironlake_enable_pch_hotplug(dev);
1975
1976	return 0;
1977}
1978
1979static int valleyview_irq_postinstall(struct drm_device *dev)
1980{
1981	drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
1982	u32 enable_mask;
1983	u32 hotplug_en = I915_READ(PORT_HOTPLUG_EN);
1984	u32 pipestat_enable = PLANE_FLIP_DONE_INT_EN_VLV;
1985	u32 render_irqs;
1986	u16 msid;
1987
1988	enable_mask = I915_DISPLAY_PORT_INTERRUPT;
1989	enable_mask |= I915_DISPLAY_PIPE_A_EVENT_INTERRUPT |
1990		I915_DISPLAY_PIPE_A_VBLANK_INTERRUPT |
1991		I915_DISPLAY_PIPE_B_EVENT_INTERRUPT |
1992		I915_DISPLAY_PIPE_B_VBLANK_INTERRUPT;
1993
1994	/*
1995	 *Leave vblank interrupts masked initially.  enable/disable will
1996	 * toggle them based on usage.
1997	 */
1998	dev_priv->irq_mask = (~enable_mask) |
1999		I915_DISPLAY_PIPE_A_VBLANK_INTERRUPT |
2000		I915_DISPLAY_PIPE_B_VBLANK_INTERRUPT;
2001
2002	dev_priv->pipestat[0] = 0;
2003	dev_priv->pipestat[1] = 0;
2004
2005	/* Hack for broken MSIs on VLV */
2006	pci_write_config_dword(dev_priv->dev->pdev, 0x94, 0xfee00000);
2007	pci_read_config_word(dev->pdev, 0x98, &msid);
2008	msid &= 0xff; /* mask out delivery bits */
2009	msid |= (1<<14);
2010	pci_write_config_word(dev_priv->dev->pdev, 0x98, msid);
2011
2012	I915_WRITE(VLV_IMR, dev_priv->irq_mask);
2013	I915_WRITE(VLV_IER, enable_mask);
2014	I915_WRITE(VLV_IIR, 0xffffffff);
2015	I915_WRITE(PIPESTAT(0), 0xffff);
2016	I915_WRITE(PIPESTAT(1), 0xffff);
2017	POSTING_READ(VLV_IER);
2018
2019	i915_enable_pipestat(dev_priv, 0, pipestat_enable);
2020	i915_enable_pipestat(dev_priv, 1, pipestat_enable);
2021
2022	I915_WRITE(VLV_IIR, 0xffffffff);
2023	I915_WRITE(VLV_IIR, 0xffffffff);
2024
2025	I915_WRITE(GTIIR, I915_READ(GTIIR));
2026	I915_WRITE(GTIMR, dev_priv->gt_irq_mask);
2027
2028	render_irqs = GT_USER_INTERRUPT | GEN6_BSD_USER_INTERRUPT |
2029		GEN6_BLITTER_USER_INTERRUPT;
2030	I915_WRITE(GTIER, render_irqs);
2031	POSTING_READ(GTIER);
2032
2033	/* ack & enable invalid PTE error interrupts */
2034#if 0 /* FIXME: add support to irq handler for checking these bits */
2035	I915_WRITE(DPINVGTT, DPINVGTT_STATUS_MASK);
2036	I915_WRITE(DPINVGTT, DPINVGTT_EN_MASK);
2037#endif
2038
2039	I915_WRITE(VLV_MASTER_IER, MASTER_INTERRUPT_ENABLE);
2040	/* Note HDMI and DP share bits */
2041	if (dev_priv->hotplug_supported_mask & HDMIB_HOTPLUG_INT_STATUS)
2042		hotplug_en |= HDMIB_HOTPLUG_INT_EN;
2043	if (dev_priv->hotplug_supported_mask & HDMIC_HOTPLUG_INT_STATUS)
2044		hotplug_en |= HDMIC_HOTPLUG_INT_EN;
2045	if (dev_priv->hotplug_supported_mask & HDMID_HOTPLUG_INT_STATUS)
2046		hotplug_en |= HDMID_HOTPLUG_INT_EN;
2047	if (dev_priv->hotplug_supported_mask & SDVOC_HOTPLUG_INT_STATUS_I915)
2048		hotplug_en |= SDVOC_HOTPLUG_INT_EN;
2049	if (dev_priv->hotplug_supported_mask & SDVOB_HOTPLUG_INT_STATUS_I915)
2050		hotplug_en |= SDVOB_HOTPLUG_INT_EN;
2051	if (dev_priv->hotplug_supported_mask & CRT_HOTPLUG_INT_STATUS) {
2052		hotplug_en |= CRT_HOTPLUG_INT_EN;
2053		hotplug_en |= CRT_HOTPLUG_VOLTAGE_COMPARE_50;
2054	}
2055
2056	I915_WRITE(PORT_HOTPLUG_EN, hotplug_en);
2057
2058	return 0;
2059}
2060
2061static void valleyview_irq_uninstall(struct drm_device *dev)
2062{
2063	drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
2064	int pipe;
2065
2066	if (!dev_priv)
2067		return;
2068
2069	for_each_pipe(pipe)
2070		I915_WRITE(PIPESTAT(pipe), 0xffff);
2071
2072	I915_WRITE(HWSTAM, 0xffffffff);
2073	I915_WRITE(PORT_HOTPLUG_EN, 0);
2074	I915_WRITE(PORT_HOTPLUG_STAT, I915_READ(PORT_HOTPLUG_STAT));
2075	for_each_pipe(pipe)
2076		I915_WRITE(PIPESTAT(pipe), 0xffff);
2077	I915_WRITE(VLV_IIR, 0xffffffff);
2078	I915_WRITE(VLV_IMR, 0xffffffff);
2079	I915_WRITE(VLV_IER, 0x0);
2080	POSTING_READ(VLV_IER);
2081}
2082
2083static void ironlake_irq_uninstall(struct drm_device *dev)
2084{
2085	drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
2086
2087	if (!dev_priv)
2088		return;
2089
2090	I915_WRITE(HWSTAM, 0xffffffff);
2091
2092	I915_WRITE(DEIMR, 0xffffffff);
2093	I915_WRITE(DEIER, 0x0);
2094	I915_WRITE(DEIIR, I915_READ(DEIIR));
2095
2096	I915_WRITE(GTIMR, 0xffffffff);
2097	I915_WRITE(GTIER, 0x0);
2098	I915_WRITE(GTIIR, I915_READ(GTIIR));
2099
2100	I915_WRITE(SDEIMR, 0xffffffff);
2101	I915_WRITE(SDEIER, 0x0);
2102	I915_WRITE(SDEIIR, I915_READ(SDEIIR));
2103}
2104
2105static void i8xx_irq_preinstall(struct drm_device * dev)
2106{
2107	drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
2108	int pipe;
2109
2110	atomic_set(&dev_priv->irq_received, 0);
2111
2112	for_each_pipe(pipe)
2113		I915_WRITE(PIPESTAT(pipe), 0);
2114	I915_WRITE16(IMR, 0xffff);
2115	I915_WRITE16(IER, 0x0);
2116	POSTING_READ16(IER);
2117}
2118
2119static int i8xx_irq_postinstall(struct drm_device *dev)
2120{
2121	drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
2122
2123	dev_priv->pipestat[0] = 0;
2124	dev_priv->pipestat[1] = 0;
2125
2126	I915_WRITE16(EMR,
2127		     ~(I915_ERROR_PAGE_TABLE | I915_ERROR_MEMORY_REFRESH));
2128
2129	/* Unmask the interrupts that we always want on. */
2130	dev_priv->irq_mask =
2131		~(I915_DISPLAY_PIPE_A_EVENT_INTERRUPT |
2132		  I915_DISPLAY_PIPE_B_EVENT_INTERRUPT |
2133		  I915_DISPLAY_PLANE_A_FLIP_PENDING_INTERRUPT |
2134		  I915_DISPLAY_PLANE_B_FLIP_PENDING_INTERRUPT |
2135		  I915_RENDER_COMMAND_PARSER_ERROR_INTERRUPT);
2136	I915_WRITE16(IMR, dev_priv->irq_mask);
2137
2138	I915_WRITE16(IER,
2139		     I915_DISPLAY_PIPE_A_EVENT_INTERRUPT |
2140		     I915_DISPLAY_PIPE_B_EVENT_INTERRUPT |
2141		     I915_RENDER_COMMAND_PARSER_ERROR_INTERRUPT |
2142		     I915_USER_INTERRUPT);
2143	POSTING_READ16(IER);
2144
2145	return 0;
2146}
2147
2148static irqreturn_t i8xx_irq_handler(int irq, void *arg)
2149{
2150	struct drm_device *dev = (struct drm_device *) arg;
2151	drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
2152	u16 iir, new_iir;
2153	u32 pipe_stats[2];
2154	unsigned long irqflags;
2155	int irq_received;
2156	int pipe;
2157	u16 flip_mask =
2158		I915_DISPLAY_PLANE_A_FLIP_PENDING_INTERRUPT |
2159		I915_DISPLAY_PLANE_B_FLIP_PENDING_INTERRUPT;
2160
2161	atomic_inc(&dev_priv->irq_received);
2162
2163	iir = I915_READ16(IIR);
2164	if (iir == 0)
2165		return IRQ_NONE;
2166
2167	while (iir & ~flip_mask) {
2168		/* Can't rely on pipestat interrupt bit in iir as it might
2169		 * have been cleared after the pipestat interrupt was received.
2170		 * It doesn't set the bit in iir again, but it still produces
2171		 * interrupts (for non-MSI).
2172		 */
2173		spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
2174		if (iir & I915_RENDER_COMMAND_PARSER_ERROR_INTERRUPT)
2175			i915_handle_error(dev, false);
2176
2177		for_each_pipe(pipe) {
2178			int reg = PIPESTAT(pipe);
2179			pipe_stats[pipe] = I915_READ(reg);
2180
2181			/*
2182			 * Clear the PIPE*STAT regs before the IIR
2183			 */
2184			if (pipe_stats[pipe] & 0x8000ffff) {
2185				if (pipe_stats[pipe] & PIPE_FIFO_UNDERRUN_STATUS)
2186					DRM_DEBUG_DRIVER("pipe %c underrun\n",
2187							 pipe_name(pipe));
2188				I915_WRITE(reg, pipe_stats[pipe]);
2189				irq_received = 1;
2190			}
2191		}
2192		spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
2193
2194		I915_WRITE16(IIR, iir & ~flip_mask);
2195		new_iir = I915_READ16(IIR); /* Flush posted writes */
2196
2197		i915_update_dri1_breadcrumb(dev);
2198
2199		if (iir & I915_USER_INTERRUPT)
2200			notify_ring(dev, &dev_priv->ring[RCS]);
2201
2202		if (pipe_stats[0] & PIPE_VBLANK_INTERRUPT_STATUS &&
2203		    drm_handle_vblank(dev, 0)) {
2204			if (iir & I915_DISPLAY_PLANE_A_FLIP_PENDING_INTERRUPT) {
2205				intel_prepare_page_flip(dev, 0);
2206				intel_finish_page_flip(dev, 0);
2207				flip_mask &= ~I915_DISPLAY_PLANE_A_FLIP_PENDING_INTERRUPT;
2208			}
2209		}
2210
2211		if (pipe_stats[1] & PIPE_VBLANK_INTERRUPT_STATUS &&
2212		    drm_handle_vblank(dev, 1)) {
2213			if (iir & I915_DISPLAY_PLANE_B_FLIP_PENDING_INTERRUPT) {
2214				intel_prepare_page_flip(dev, 1);
2215				intel_finish_page_flip(dev, 1);
2216				flip_mask &= ~I915_DISPLAY_PLANE_B_FLIP_PENDING_INTERRUPT;
2217			}
2218		}
2219
2220		iir = new_iir;
2221	}
2222
2223	return IRQ_HANDLED;
2224}
2225
2226static void i8xx_irq_uninstall(struct drm_device * dev)
2227{
2228	drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
2229	int pipe;
2230
2231	for_each_pipe(pipe) {
2232		/* Clear enable bits; then clear status bits */
2233		I915_WRITE(PIPESTAT(pipe), 0);
2234		I915_WRITE(PIPESTAT(pipe), I915_READ(PIPESTAT(pipe)));
2235	}
2236	I915_WRITE16(IMR, 0xffff);
2237	I915_WRITE16(IER, 0x0);
2238	I915_WRITE16(IIR, I915_READ16(IIR));
2239}
2240
2241static void i915_irq_preinstall(struct drm_device * dev)
2242{
2243	drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
2244	int pipe;
2245
2246	atomic_set(&dev_priv->irq_received, 0);
2247
2248	if (I915_HAS_HOTPLUG(dev)) {
2249		I915_WRITE(PORT_HOTPLUG_EN, 0);
2250		I915_WRITE(PORT_HOTPLUG_STAT, I915_READ(PORT_HOTPLUG_STAT));
2251	}
2252
2253	I915_WRITE16(HWSTAM, 0xeffe);
2254	for_each_pipe(pipe)
2255		I915_WRITE(PIPESTAT(pipe), 0);
2256	I915_WRITE(IMR, 0xffffffff);
2257	I915_WRITE(IER, 0x0);
2258	POSTING_READ(IER);
2259}
2260
2261static int i915_irq_postinstall(struct drm_device *dev)
2262{
2263	drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
2264	u32 enable_mask;
2265
2266	dev_priv->pipestat[0] = 0;
2267	dev_priv->pipestat[1] = 0;
2268
2269	I915_WRITE(EMR, ~(I915_ERROR_PAGE_TABLE | I915_ERROR_MEMORY_REFRESH));
2270
2271	/* Unmask the interrupts that we always want on. */
2272	dev_priv->irq_mask =
2273		~(I915_ASLE_INTERRUPT |
2274		  I915_DISPLAY_PIPE_A_EVENT_INTERRUPT |
2275		  I915_DISPLAY_PIPE_B_EVENT_INTERRUPT |
2276		  I915_DISPLAY_PLANE_A_FLIP_PENDING_INTERRUPT |
2277		  I915_DISPLAY_PLANE_B_FLIP_PENDING_INTERRUPT |
2278		  I915_RENDER_COMMAND_PARSER_ERROR_INTERRUPT);
2279
2280	enable_mask =
2281		I915_ASLE_INTERRUPT |
2282		I915_DISPLAY_PIPE_A_EVENT_INTERRUPT |
2283		I915_DISPLAY_PIPE_B_EVENT_INTERRUPT |
2284		I915_RENDER_COMMAND_PARSER_ERROR_INTERRUPT |
2285		I915_USER_INTERRUPT;
2286
2287	if (I915_HAS_HOTPLUG(dev)) {
2288		/* Enable in IER... */
2289		enable_mask |= I915_DISPLAY_PORT_INTERRUPT;
2290		/* and unmask in IMR */
2291		dev_priv->irq_mask &= ~I915_DISPLAY_PORT_INTERRUPT;
2292	}
2293
2294	I915_WRITE(IMR, dev_priv->irq_mask);
2295	I915_WRITE(IER, enable_mask);
2296	POSTING_READ(IER);
2297
2298	if (I915_HAS_HOTPLUG(dev)) {
2299		u32 hotplug_en = I915_READ(PORT_HOTPLUG_EN);
2300
2301		if (dev_priv->hotplug_supported_mask & HDMIB_HOTPLUG_INT_STATUS)
2302			hotplug_en |= HDMIB_HOTPLUG_INT_EN;
2303		if (dev_priv->hotplug_supported_mask & HDMIC_HOTPLUG_INT_STATUS)
2304			hotplug_en |= HDMIC_HOTPLUG_INT_EN;
2305		if (dev_priv->hotplug_supported_mask & HDMID_HOTPLUG_INT_STATUS)
2306			hotplug_en |= HDMID_HOTPLUG_INT_EN;
2307		if (dev_priv->hotplug_supported_mask & SDVOC_HOTPLUG_INT_STATUS_I915)
2308			hotplug_en |= SDVOC_HOTPLUG_INT_EN;
2309		if (dev_priv->hotplug_supported_mask & SDVOB_HOTPLUG_INT_STATUS_I915)
2310			hotplug_en |= SDVOB_HOTPLUG_INT_EN;
2311		if (dev_priv->hotplug_supported_mask & CRT_HOTPLUG_INT_STATUS) {
2312			hotplug_en |= CRT_HOTPLUG_INT_EN;
2313			hotplug_en |= CRT_HOTPLUG_VOLTAGE_COMPARE_50;
2314		}
2315
2316		/* Ignore TV since it's buggy */
2317
2318		I915_WRITE(PORT_HOTPLUG_EN, hotplug_en);
2319	}
2320
2321	intel_opregion_enable_asle(dev);
2322
2323	return 0;
2324}
2325
2326static irqreturn_t i915_irq_handler(int irq, void *arg)
2327{
2328	struct drm_device *dev = (struct drm_device *) arg;
2329	drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
2330	u32 iir, new_iir, pipe_stats[I915_MAX_PIPES];
2331	unsigned long irqflags;
2332	u32 flip_mask =
2333		I915_DISPLAY_PLANE_A_FLIP_PENDING_INTERRUPT |
2334		I915_DISPLAY_PLANE_B_FLIP_PENDING_INTERRUPT;
2335	u32 flip[2] = {
2336		I915_DISPLAY_PLANE_A_FLIP_PENDING_INTERRUPT,
2337		I915_DISPLAY_PLANE_B_FLIP_PENDING_INTERRUPT
2338	};
2339	int pipe, ret = IRQ_NONE;
2340
2341	atomic_inc(&dev_priv->irq_received);
2342
2343	iir = I915_READ(IIR);
2344	do {
2345		bool irq_received = (iir & ~flip_mask) != 0;
2346		bool blc_event = false;
2347
2348		/* Can't rely on pipestat interrupt bit in iir as it might
2349		 * have been cleared after the pipestat interrupt was received.
2350		 * It doesn't set the bit in iir again, but it still produces
2351		 * interrupts (for non-MSI).
2352		 */
2353		spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
2354		if (iir & I915_RENDER_COMMAND_PARSER_ERROR_INTERRUPT)
2355			i915_handle_error(dev, false);
2356
2357		for_each_pipe(pipe) {
2358			int reg = PIPESTAT(pipe);
2359			pipe_stats[pipe] = I915_READ(reg);
2360
2361			/* Clear the PIPE*STAT regs before the IIR */
2362			if (pipe_stats[pipe] & 0x8000ffff) {
2363				if (pipe_stats[pipe] & PIPE_FIFO_UNDERRUN_STATUS)
2364					DRM_DEBUG_DRIVER("pipe %c underrun\n",
2365							 pipe_name(pipe));
2366				I915_WRITE(reg, pipe_stats[pipe]);
2367				irq_received = true;
2368			}
2369		}
2370		spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
2371
2372		if (!irq_received)
2373			break;
2374
2375		/* Consume port.  Then clear IIR or we'll miss events */
2376		if ((I915_HAS_HOTPLUG(dev)) &&
2377		    (iir & I915_DISPLAY_PORT_INTERRUPT)) {
2378			u32 hotplug_status = I915_READ(PORT_HOTPLUG_STAT);
2379
2380			DRM_DEBUG_DRIVER("hotplug event received, stat 0x%08x\n",
2381				  hotplug_status);
2382			if (hotplug_status & dev_priv->hotplug_supported_mask)
2383				queue_work(dev_priv->wq,
2384					   &dev_priv->hotplug_work);
2385
2386			I915_WRITE(PORT_HOTPLUG_STAT, hotplug_status);
2387			POSTING_READ(PORT_HOTPLUG_STAT);
2388		}
2389
2390		I915_WRITE(IIR, iir & ~flip_mask);
2391		new_iir = I915_READ(IIR); /* Flush posted writes */
2392
2393		if (iir & I915_USER_INTERRUPT)
2394			notify_ring(dev, &dev_priv->ring[RCS]);
2395
2396		for_each_pipe(pipe) {
2397			int plane = pipe;
2398			if (IS_MOBILE(dev))
2399				plane = !plane;
2400			if (pipe_stats[pipe] & PIPE_VBLANK_INTERRUPT_STATUS &&
2401			    drm_handle_vblank(dev, pipe)) {
2402				if (iir & flip[plane]) {
2403					intel_prepare_page_flip(dev, plane);
2404					intel_finish_page_flip(dev, pipe);
2405					flip_mask &= ~flip[plane];
2406				}
2407			}
2408
2409			if (pipe_stats[pipe] & PIPE_LEGACY_BLC_EVENT_STATUS)
2410				blc_event = true;
2411		}
2412
2413		if (blc_event || (iir & I915_ASLE_INTERRUPT))
2414			intel_opregion_asle_intr(dev);
2415
2416		/* With MSI, interrupts are only generated when iir
2417		 * transitions from zero to nonzero.  If another bit got
2418		 * set while we were handling the existing iir bits, then
2419		 * we would never get another interrupt.
2420		 *
2421		 * This is fine on non-MSI as well, as if we hit this path
2422		 * we avoid exiting the interrupt handler only to generate
2423		 * another one.
2424		 *
2425		 * Note that for MSI this could cause a stray interrupt report
2426		 * if an interrupt landed in the time between writing IIR and
2427		 * the posting read.  This should be rare enough to never
2428		 * trigger the 99% of 100,000 interrupts test for disabling
2429		 * stray interrupts.
2430		 */
2431		ret = IRQ_HANDLED;
2432		iir = new_iir;
2433	} while (iir & ~flip_mask);
2434
2435	i915_update_dri1_breadcrumb(dev);
2436
2437	return ret;
2438}
2439
2440static void i915_irq_uninstall(struct drm_device * dev)
2441{
2442	drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
2443	int pipe;
2444
2445	if (I915_HAS_HOTPLUG(dev)) {
2446		I915_WRITE(PORT_HOTPLUG_EN, 0);
2447		I915_WRITE(PORT_HOTPLUG_STAT, I915_READ(PORT_HOTPLUG_STAT));
2448	}
2449
2450	I915_WRITE16(HWSTAM, 0xffff);
2451	for_each_pipe(pipe) {
2452		/* Clear enable bits; then clear status bits */
2453		I915_WRITE(PIPESTAT(pipe), 0);
2454		I915_WRITE(PIPESTAT(pipe), I915_READ(PIPESTAT(pipe)));
2455	}
2456	I915_WRITE(IMR, 0xffffffff);
2457	I915_WRITE(IER, 0x0);
2458
2459	I915_WRITE(IIR, I915_READ(IIR));
2460}
2461
2462static void i965_irq_preinstall(struct drm_device * dev)
2463{
2464	drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
2465	int pipe;
2466
2467	atomic_set(&dev_priv->irq_received, 0);
2468
2469	I915_WRITE(PORT_HOTPLUG_EN, 0);
2470	I915_WRITE(PORT_HOTPLUG_STAT, I915_READ(PORT_HOTPLUG_STAT));
2471
2472	I915_WRITE(HWSTAM, 0xeffe);
2473	for_each_pipe(pipe)
2474		I915_WRITE(PIPESTAT(pipe), 0);
2475	I915_WRITE(IMR, 0xffffffff);
2476	I915_WRITE(IER, 0x0);
2477	POSTING_READ(IER);
2478}
2479
2480static int i965_irq_postinstall(struct drm_device *dev)
2481{
2482	drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
2483	u32 hotplug_en;
2484	u32 enable_mask;
2485	u32 error_mask;
2486
2487	/* Unmask the interrupts that we always want on. */
2488	dev_priv->irq_mask = ~(I915_ASLE_INTERRUPT |
2489			       I915_DISPLAY_PORT_INTERRUPT |
2490			       I915_DISPLAY_PIPE_A_EVENT_INTERRUPT |
2491			       I915_DISPLAY_PIPE_B_EVENT_INTERRUPT |
2492			       I915_DISPLAY_PLANE_A_FLIP_PENDING_INTERRUPT |
2493			       I915_DISPLAY_PLANE_B_FLIP_PENDING_INTERRUPT |
2494			       I915_RENDER_COMMAND_PARSER_ERROR_INTERRUPT);
2495
2496	enable_mask = ~dev_priv->irq_mask;
2497	enable_mask |= I915_USER_INTERRUPT;
2498
2499	if (IS_G4X(dev))
2500		enable_mask |= I915_BSD_USER_INTERRUPT;
2501
2502	dev_priv->pipestat[0] = 0;
2503	dev_priv->pipestat[1] = 0;
2504
2505	/*
2506	 * Enable some error detection, note the instruction error mask
2507	 * bit is reserved, so we leave it masked.
2508	 */
2509	if (IS_G4X(dev)) {
2510		error_mask = ~(GM45_ERROR_PAGE_TABLE |
2511			       GM45_ERROR_MEM_PRIV |
2512			       GM45_ERROR_CP_PRIV |
2513			       I915_ERROR_MEMORY_REFRESH);
2514	} else {
2515		error_mask = ~(I915_ERROR_PAGE_TABLE |
2516			       I915_ERROR_MEMORY_REFRESH);
2517	}
2518	I915_WRITE(EMR, error_mask);
2519
2520	I915_WRITE(IMR, dev_priv->irq_mask);
2521	I915_WRITE(IER, enable_mask);
2522	POSTING_READ(IER);
2523
2524	/* Note HDMI and DP share hotplug bits */
2525	hotplug_en = 0;
2526	if (dev_priv->hotplug_supported_mask & HDMIB_HOTPLUG_INT_STATUS)
2527		hotplug_en |= HDMIB_HOTPLUG_INT_EN;
2528	if (dev_priv->hotplug_supported_mask & HDMIC_HOTPLUG_INT_STATUS)
2529		hotplug_en |= HDMIC_HOTPLUG_INT_EN;
2530	if (dev_priv->hotplug_supported_mask & HDMID_HOTPLUG_INT_STATUS)
2531		hotplug_en |= HDMID_HOTPLUG_INT_EN;
2532	if (IS_G4X(dev)) {
2533		if (dev_priv->hotplug_supported_mask & SDVOC_HOTPLUG_INT_STATUS_G4X)
2534			hotplug_en |= SDVOC_HOTPLUG_INT_EN;
2535		if (dev_priv->hotplug_supported_mask & SDVOB_HOTPLUG_INT_STATUS_G4X)
2536			hotplug_en |= SDVOB_HOTPLUG_INT_EN;
2537	} else {
2538		if (dev_priv->hotplug_supported_mask & SDVOC_HOTPLUG_INT_STATUS_I965)
2539			hotplug_en |= SDVOC_HOTPLUG_INT_EN;
2540		if (dev_priv->hotplug_supported_mask & SDVOB_HOTPLUG_INT_STATUS_I965)
2541			hotplug_en |= SDVOB_HOTPLUG_INT_EN;
2542	}
2543	if (dev_priv->hotplug_supported_mask & CRT_HOTPLUG_INT_STATUS) {
2544		hotplug_en |= CRT_HOTPLUG_INT_EN;
2545
2546		/* Programming the CRT detection parameters tends
2547		   to generate a spurious hotplug event about three
2548		   seconds later.  So just do it once.
2549		   */
2550		if (IS_G4X(dev))
2551			hotplug_en |= CRT_HOTPLUG_ACTIVATION_PERIOD_64;
2552		hotplug_en |= CRT_HOTPLUG_VOLTAGE_COMPARE_50;
2553	}
2554
2555	/* Ignore TV since it's buggy */
2556
2557	I915_WRITE(PORT_HOTPLUG_EN, hotplug_en);
2558
2559	intel_opregion_enable_asle(dev);
2560
2561	return 0;
2562}
2563
2564static irqreturn_t i965_irq_handler(int irq, void *arg)
2565{
2566	struct drm_device *dev = (struct drm_device *) arg;
2567	drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
2568	u32 iir, new_iir;
2569	u32 pipe_stats[I915_MAX_PIPES];
2570	unsigned long irqflags;
2571	int irq_received;
2572	int ret = IRQ_NONE, pipe;
2573
2574	atomic_inc(&dev_priv->irq_received);
2575
2576	iir = I915_READ(IIR);
2577
2578	for (;;) {
2579		bool blc_event = false;
2580
2581		irq_received = iir != 0;
2582
2583		/* Can't rely on pipestat interrupt bit in iir as it might
2584		 * have been cleared after the pipestat interrupt was received.
2585		 * It doesn't set the bit in iir again, but it still produces
2586		 * interrupts (for non-MSI).
2587		 */
2588		spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
2589		if (iir & I915_RENDER_COMMAND_PARSER_ERROR_INTERRUPT)
2590			i915_handle_error(dev, false);
2591
2592		for_each_pipe(pipe) {
2593			int reg = PIPESTAT(pipe);
2594			pipe_stats[pipe] = I915_READ(reg);
2595
2596			/*
2597			 * Clear the PIPE*STAT regs before the IIR
2598			 */
2599			if (pipe_stats[pipe] & 0x8000ffff) {
2600				if (pipe_stats[pipe] & PIPE_FIFO_UNDERRUN_STATUS)
2601					DRM_DEBUG_DRIVER("pipe %c underrun\n",
2602							 pipe_name(pipe));
2603				I915_WRITE(reg, pipe_stats[pipe]);
2604				irq_received = 1;
2605			}
2606		}
2607		spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
2608
2609		if (!irq_received)
2610			break;
2611
2612		ret = IRQ_HANDLED;
2613
2614		/* Consume port.  Then clear IIR or we'll miss events */
2615		if (iir & I915_DISPLAY_PORT_INTERRUPT) {
2616			u32 hotplug_status = I915_READ(PORT_HOTPLUG_STAT);
2617
2618			DRM_DEBUG_DRIVER("hotplug event received, stat 0x%08x\n",
2619				  hotplug_status);
2620			if (hotplug_status & dev_priv->hotplug_supported_mask)
2621				queue_work(dev_priv->wq,
2622					   &dev_priv->hotplug_work);
2623
2624			I915_WRITE(PORT_HOTPLUG_STAT, hotplug_status);
2625			I915_READ(PORT_HOTPLUG_STAT);
2626		}
2627
2628		I915_WRITE(IIR, iir);
2629		new_iir = I915_READ(IIR); /* Flush posted writes */
2630
2631		if (iir & I915_USER_INTERRUPT)
2632			notify_ring(dev, &dev_priv->ring[RCS]);
2633		if (iir & I915_BSD_USER_INTERRUPT)
2634			notify_ring(dev, &dev_priv->ring[VCS]);
2635
2636		if (iir & I915_DISPLAY_PLANE_A_FLIP_PENDING_INTERRUPT)
2637			intel_prepare_page_flip(dev, 0);
2638
2639		if (iir & I915_DISPLAY_PLANE_B_FLIP_PENDING_INTERRUPT)
2640			intel_prepare_page_flip(dev, 1);
2641
2642		for_each_pipe(pipe) {
2643			if (pipe_stats[pipe] & PIPE_START_VBLANK_INTERRUPT_STATUS &&
2644			    drm_handle_vblank(dev, pipe)) {
2645				i915_pageflip_stall_check(dev, pipe);
2646				intel_finish_page_flip(dev, pipe);
2647			}
2648
2649			if (pipe_stats[pipe] & PIPE_LEGACY_BLC_EVENT_STATUS)
2650				blc_event = true;
2651		}
2652
2653
2654		if (blc_event || (iir & I915_ASLE_INTERRUPT))
2655			intel_opregion_asle_intr(dev);
2656
2657		/* With MSI, interrupts are only generated when iir
2658		 * transitions from zero to nonzero.  If another bit got
2659		 * set while we were handling the existing iir bits, then
2660		 * we would never get another interrupt.
2661		 *
2662		 * This is fine on non-MSI as well, as if we hit this path
2663		 * we avoid exiting the interrupt handler only to generate
2664		 * another one.
2665		 *
2666		 * Note that for MSI this could cause a stray interrupt report
2667		 * if an interrupt landed in the time between writing IIR and
2668		 * the posting read.  This should be rare enough to never
2669		 * trigger the 99% of 100,000 interrupts test for disabling
2670		 * stray interrupts.
2671		 */
2672		iir = new_iir;
2673	}
2674
2675	i915_update_dri1_breadcrumb(dev);
2676
2677	return ret;
2678}
2679
2680static void i965_irq_uninstall(struct drm_device * dev)
2681{
2682	drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
2683	int pipe;
2684
2685	if (!dev_priv)
2686		return;
2687
2688	I915_WRITE(PORT_HOTPLUG_EN, 0);
2689	I915_WRITE(PORT_HOTPLUG_STAT, I915_READ(PORT_HOTPLUG_STAT));
2690
2691	I915_WRITE(HWSTAM, 0xffffffff);
2692	for_each_pipe(pipe)
2693		I915_WRITE(PIPESTAT(pipe), 0);
2694	I915_WRITE(IMR, 0xffffffff);
2695	I915_WRITE(IER, 0x0);
2696
2697	for_each_pipe(pipe)
2698		I915_WRITE(PIPESTAT(pipe),
2699			   I915_READ(PIPESTAT(pipe)) & 0x8000ffff);
2700	I915_WRITE(IIR, I915_READ(IIR));
2701}
2702
2703void intel_irq_init(struct drm_device *dev)
2704{
2705	struct drm_i915_private *dev_priv = dev->dev_private;
2706
2707	INIT_WORK(&dev_priv->hotplug_work, i915_hotplug_work_func);
2708	INIT_WORK(&dev_priv->error_work, i915_error_work_func);
2709	INIT_WORK(&dev_priv->rps.work, gen6_pm_rps_work);
2710	INIT_WORK(&dev_priv->l3_parity.error_work, ivybridge_parity_work);
2711
2712	dev->driver->get_vblank_counter = i915_get_vblank_counter;
2713	dev->max_vblank_count = 0xffffff; /* only 24 bits of frame count */
2714	if (IS_G4X(dev) || INTEL_INFO(dev)->gen >= 5) {
2715		dev->max_vblank_count = 0xffffffff; /* full 32 bit counter */
2716		dev->driver->get_vblank_counter = gm45_get_vblank_counter;
2717	}
2718
2719	if (drm_core_check_feature(dev, DRIVER_MODESET))
2720		dev->driver->get_vblank_timestamp = i915_get_vblank_timestamp;
2721	else
2722		dev->driver->get_vblank_timestamp = NULL;
2723	dev->driver->get_scanout_position = i915_get_crtc_scanoutpos;
2724
2725	if (IS_VALLEYVIEW(dev)) {
2726		dev->driver->irq_handler = valleyview_irq_handler;
2727		dev->driver->irq_preinstall = valleyview_irq_preinstall;
2728		dev->driver->irq_postinstall = valleyview_irq_postinstall;
2729		dev->driver->irq_uninstall = valleyview_irq_uninstall;
2730		dev->driver->enable_vblank = valleyview_enable_vblank;
2731		dev->driver->disable_vblank = valleyview_disable_vblank;
2732	} else if (IS_IVYBRIDGE(dev)) {
2733		/* Share pre & uninstall handlers with ILK/SNB */
2734		dev->driver->irq_handler = ivybridge_irq_handler;
2735		dev->driver->irq_preinstall = ironlake_irq_preinstall;
2736		dev->driver->irq_postinstall = ivybridge_irq_postinstall;
2737		dev->driver->irq_uninstall = ironlake_irq_uninstall;
2738		dev->driver->enable_vblank = ivybridge_enable_vblank;
2739		dev->driver->disable_vblank = ivybridge_disable_vblank;
2740	} else if (IS_HASWELL(dev)) {
2741		/* Share interrupts handling with IVB */
2742		dev->driver->irq_handler = ivybridge_irq_handler;
2743		dev->driver->irq_preinstall = ironlake_irq_preinstall;
2744		dev->driver->irq_postinstall = ivybridge_irq_postinstall;
2745		dev->driver->irq_uninstall = ironlake_irq_uninstall;
2746		dev->driver->enable_vblank = ivybridge_enable_vblank;
2747		dev->driver->disable_vblank = ivybridge_disable_vblank;
2748	} else if (HAS_PCH_SPLIT(dev)) {
2749		dev->driver->irq_handler = ironlake_irq_handler;
2750		dev->driver->irq_preinstall = ironlake_irq_preinstall;
2751		dev->driver->irq_postinstall = ironlake_irq_postinstall;
2752		dev->driver->irq_uninstall = ironlake_irq_uninstall;
2753		dev->driver->enable_vblank = ironlake_enable_vblank;
2754		dev->driver->disable_vblank = ironlake_disable_vblank;
2755	} else {
2756		if (INTEL_INFO(dev)->gen == 2) {
2757			dev->driver->irq_preinstall = i8xx_irq_preinstall;
2758			dev->driver->irq_postinstall = i8xx_irq_postinstall;
2759			dev->driver->irq_handler = i8xx_irq_handler;
2760			dev->driver->irq_uninstall = i8xx_irq_uninstall;
2761		} else if (INTEL_INFO(dev)->gen == 3) {
2762			dev->driver->irq_preinstall = i915_irq_preinstall;
2763			dev->driver->irq_postinstall = i915_irq_postinstall;
2764			dev->driver->irq_uninstall = i915_irq_uninstall;
2765			dev->driver->irq_handler = i915_irq_handler;
2766		} else {
2767			dev->driver->irq_preinstall = i965_irq_preinstall;
2768			dev->driver->irq_postinstall = i965_irq_postinstall;
2769			dev->driver->irq_uninstall = i965_irq_uninstall;
2770			dev->driver->irq_handler = i965_irq_handler;
2771		}
2772		dev->driver->enable_vblank = i915_enable_vblank;
2773		dev->driver->disable_vblank = i915_disable_vblank;
2774	}
2775}