PageRenderTime 131ms CodeModel.GetById 34ms app.highlight 76ms RepoModel.GetById 2ms app.codeStats 0ms

/drivers/gpu/drm/nouveau/nouveau_bo.c

https://bitbucket.org/ndreys/linux-sunxi
C | 1115 lines | 907 code | 157 blank | 51 comment | 146 complexity | fb5ab0f88cd116019e6e31e747886b5d MD5 | raw file
Possible License(s): GPL-2.0, LGPL-2.0, AGPL-1.0
   1/*
   2 * Copyright 2007 Dave Airlied
   3 * All Rights Reserved.
   4 *
   5 * Permission is hereby granted, free of charge, to any person obtaining a
   6 * copy of this software and associated documentation files (the "Software"),
   7 * to deal in the Software without restriction, including without limitation
   8 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   9 * and/or sell copies of the Software, and to permit persons to whom the
  10 * Software is furnished to do so, subject to the following conditions:
  11 *
  12 * The above copyright notice and this permission notice (including the next
  13 * paragraph) shall be included in all copies or substantial portions of the
  14 * Software.
  15 *
  16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  19 * VA LINUX SYSTEMS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
  20 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  21 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  22 * OTHER DEALINGS IN THE SOFTWARE.
  23 */
  24/*
  25 * Authors: Dave Airlied <airlied@linux.ie>
  26 *	    Ben Skeggs   <darktama@iinet.net.au>
  27 *	    Jeremy Kolb  <jkolb@brandeis.edu>
  28 */
  29
  30#include "drmP.h"
  31
  32#include "nouveau_drm.h"
  33#include "nouveau_drv.h"
  34#include "nouveau_dma.h"
  35#include "nouveau_mm.h"
  36#include "nouveau_vm.h"
  37
  38#include <linux/log2.h>
  39#include <linux/slab.h>
  40
  41static void
  42nouveau_bo_del_ttm(struct ttm_buffer_object *bo)
  43{
  44	struct drm_nouveau_private *dev_priv = nouveau_bdev(bo->bdev);
  45	struct drm_device *dev = dev_priv->dev;
  46	struct nouveau_bo *nvbo = nouveau_bo(bo);
  47
  48	if (unlikely(nvbo->gem))
  49		DRM_ERROR("bo %p still attached to GEM object\n", bo);
  50
  51	nv10_mem_put_tile_region(dev, nvbo->tile, NULL);
  52	if (nvbo->vma.node) {
  53		nouveau_vm_unmap(&nvbo->vma);
  54		nouveau_vm_put(&nvbo->vma);
  55	}
  56	kfree(nvbo);
  57}
  58
  59static void
  60nouveau_bo_fixup_align(struct nouveau_bo *nvbo, u32 flags,
  61		       int *align, int *size, int *page_shift)
  62{
  63	struct drm_nouveau_private *dev_priv = nouveau_bdev(nvbo->bo.bdev);
  64
  65	if (dev_priv->card_type < NV_50) {
  66		if (nvbo->tile_mode) {
  67			if (dev_priv->chipset >= 0x40) {
  68				*align = 65536;
  69				*size = roundup(*size, 64 * nvbo->tile_mode);
  70
  71			} else if (dev_priv->chipset >= 0x30) {
  72				*align = 32768;
  73				*size = roundup(*size, 64 * nvbo->tile_mode);
  74
  75			} else if (dev_priv->chipset >= 0x20) {
  76				*align = 16384;
  77				*size = roundup(*size, 64 * nvbo->tile_mode);
  78
  79			} else if (dev_priv->chipset >= 0x10) {
  80				*align = 16384;
  81				*size = roundup(*size, 32 * nvbo->tile_mode);
  82			}
  83		}
  84	} else {
  85		if (likely(dev_priv->chan_vm)) {
  86			if (!(flags & TTM_PL_FLAG_TT) &&  *size > 256 * 1024)
  87				*page_shift = dev_priv->chan_vm->lpg_shift;
  88			else
  89				*page_shift = dev_priv->chan_vm->spg_shift;
  90		} else {
  91			*page_shift = 12;
  92		}
  93
  94		*size = roundup(*size, (1 << *page_shift));
  95		*align = max((1 << *page_shift), *align);
  96	}
  97
  98	*size = roundup(*size, PAGE_SIZE);
  99}
 100
 101int
 102nouveau_bo_new(struct drm_device *dev, struct nouveau_channel *chan,
 103	       int size, int align, uint32_t flags, uint32_t tile_mode,
 104	       uint32_t tile_flags, struct nouveau_bo **pnvbo)
 105{
 106	struct drm_nouveau_private *dev_priv = dev->dev_private;
 107	struct nouveau_bo *nvbo;
 108	int ret = 0, page_shift = 0;
 109
 110	nvbo = kzalloc(sizeof(struct nouveau_bo), GFP_KERNEL);
 111	if (!nvbo)
 112		return -ENOMEM;
 113	INIT_LIST_HEAD(&nvbo->head);
 114	INIT_LIST_HEAD(&nvbo->entry);
 115	nvbo->tile_mode = tile_mode;
 116	nvbo->tile_flags = tile_flags;
 117	nvbo->bo.bdev = &dev_priv->ttm.bdev;
 118
 119	nouveau_bo_fixup_align(nvbo, flags, &align, &size, &page_shift);
 120	align >>= PAGE_SHIFT;
 121
 122	if (dev_priv->chan_vm) {
 123		ret = nouveau_vm_get(dev_priv->chan_vm, size, page_shift,
 124				     NV_MEM_ACCESS_RW, &nvbo->vma);
 125		if (ret) {
 126			kfree(nvbo);
 127			return ret;
 128		}
 129	}
 130
 131	nvbo->bo.mem.num_pages = size >> PAGE_SHIFT;
 132	nouveau_bo_placement_set(nvbo, flags, 0);
 133
 134	nvbo->channel = chan;
 135	ret = ttm_bo_init(&dev_priv->ttm.bdev, &nvbo->bo, size,
 136			  ttm_bo_type_device, &nvbo->placement, align, 0,
 137			  false, NULL, size, nouveau_bo_del_ttm);
 138	if (ret) {
 139		/* ttm will call nouveau_bo_del_ttm if it fails.. */
 140		return ret;
 141	}
 142	nvbo->channel = NULL;
 143
 144	if (nvbo->vma.node)
 145		nvbo->bo.offset = nvbo->vma.offset;
 146	*pnvbo = nvbo;
 147	return 0;
 148}
 149
 150static void
 151set_placement_list(uint32_t *pl, unsigned *n, uint32_t type, uint32_t flags)
 152{
 153	*n = 0;
 154
 155	if (type & TTM_PL_FLAG_VRAM)
 156		pl[(*n)++] = TTM_PL_FLAG_VRAM | flags;
 157	if (type & TTM_PL_FLAG_TT)
 158		pl[(*n)++] = TTM_PL_FLAG_TT | flags;
 159	if (type & TTM_PL_FLAG_SYSTEM)
 160		pl[(*n)++] = TTM_PL_FLAG_SYSTEM | flags;
 161}
 162
 163static void
 164set_placement_range(struct nouveau_bo *nvbo, uint32_t type)
 165{
 166	struct drm_nouveau_private *dev_priv = nouveau_bdev(nvbo->bo.bdev);
 167	int vram_pages = dev_priv->vram_size >> PAGE_SHIFT;
 168
 169	if (dev_priv->card_type == NV_10 &&
 170	    nvbo->tile_mode && (type & TTM_PL_FLAG_VRAM) &&
 171	    nvbo->bo.mem.num_pages < vram_pages / 2) {
 172		/*
 173		 * Make sure that the color and depth buffers are handled
 174		 * by independent memory controller units. Up to a 9x
 175		 * speed up when alpha-blending and depth-test are enabled
 176		 * at the same time.
 177		 */
 178		if (nvbo->tile_flags & NOUVEAU_GEM_TILE_ZETA) {
 179			nvbo->placement.fpfn = vram_pages / 2;
 180			nvbo->placement.lpfn = ~0;
 181		} else {
 182			nvbo->placement.fpfn = 0;
 183			nvbo->placement.lpfn = vram_pages / 2;
 184		}
 185	}
 186}
 187
 188void
 189nouveau_bo_placement_set(struct nouveau_bo *nvbo, uint32_t type, uint32_t busy)
 190{
 191	struct ttm_placement *pl = &nvbo->placement;
 192	uint32_t flags = TTM_PL_MASK_CACHING |
 193		(nvbo->pin_refcnt ? TTM_PL_FLAG_NO_EVICT : 0);
 194
 195	pl->placement = nvbo->placements;
 196	set_placement_list(nvbo->placements, &pl->num_placement,
 197			   type, flags);
 198
 199	pl->busy_placement = nvbo->busy_placements;
 200	set_placement_list(nvbo->busy_placements, &pl->num_busy_placement,
 201			   type | busy, flags);
 202
 203	set_placement_range(nvbo, type);
 204}
 205
 206int
 207nouveau_bo_pin(struct nouveau_bo *nvbo, uint32_t memtype)
 208{
 209	struct drm_nouveau_private *dev_priv = nouveau_bdev(nvbo->bo.bdev);
 210	struct ttm_buffer_object *bo = &nvbo->bo;
 211	int ret;
 212
 213	if (nvbo->pin_refcnt && !(memtype & (1 << bo->mem.mem_type))) {
 214		NV_ERROR(nouveau_bdev(bo->bdev)->dev,
 215			 "bo %p pinned elsewhere: 0x%08x vs 0x%08x\n", bo,
 216			 1 << bo->mem.mem_type, memtype);
 217		return -EINVAL;
 218	}
 219
 220	if (nvbo->pin_refcnt++)
 221		return 0;
 222
 223	ret = ttm_bo_reserve(bo, false, false, false, 0);
 224	if (ret)
 225		goto out;
 226
 227	nouveau_bo_placement_set(nvbo, memtype, 0);
 228
 229	ret = nouveau_bo_validate(nvbo, false, false, false);
 230	if (ret == 0) {
 231		switch (bo->mem.mem_type) {
 232		case TTM_PL_VRAM:
 233			dev_priv->fb_aper_free -= bo->mem.size;
 234			break;
 235		case TTM_PL_TT:
 236			dev_priv->gart_info.aper_free -= bo->mem.size;
 237			break;
 238		default:
 239			break;
 240		}
 241	}
 242	ttm_bo_unreserve(bo);
 243out:
 244	if (unlikely(ret))
 245		nvbo->pin_refcnt--;
 246	return ret;
 247}
 248
 249int
 250nouveau_bo_unpin(struct nouveau_bo *nvbo)
 251{
 252	struct drm_nouveau_private *dev_priv = nouveau_bdev(nvbo->bo.bdev);
 253	struct ttm_buffer_object *bo = &nvbo->bo;
 254	int ret;
 255
 256	if (--nvbo->pin_refcnt)
 257		return 0;
 258
 259	ret = ttm_bo_reserve(bo, false, false, false, 0);
 260	if (ret)
 261		return ret;
 262
 263	nouveau_bo_placement_set(nvbo, bo->mem.placement, 0);
 264
 265	ret = nouveau_bo_validate(nvbo, false, false, false);
 266	if (ret == 0) {
 267		switch (bo->mem.mem_type) {
 268		case TTM_PL_VRAM:
 269			dev_priv->fb_aper_free += bo->mem.size;
 270			break;
 271		case TTM_PL_TT:
 272			dev_priv->gart_info.aper_free += bo->mem.size;
 273			break;
 274		default:
 275			break;
 276		}
 277	}
 278
 279	ttm_bo_unreserve(bo);
 280	return ret;
 281}
 282
 283int
 284nouveau_bo_map(struct nouveau_bo *nvbo)
 285{
 286	int ret;
 287
 288	ret = ttm_bo_reserve(&nvbo->bo, false, false, false, 0);
 289	if (ret)
 290		return ret;
 291
 292	ret = ttm_bo_kmap(&nvbo->bo, 0, nvbo->bo.mem.num_pages, &nvbo->kmap);
 293	ttm_bo_unreserve(&nvbo->bo);
 294	return ret;
 295}
 296
 297void
 298nouveau_bo_unmap(struct nouveau_bo *nvbo)
 299{
 300	if (nvbo)
 301		ttm_bo_kunmap(&nvbo->kmap);
 302}
 303
 304int
 305nouveau_bo_validate(struct nouveau_bo *nvbo, bool interruptible,
 306		    bool no_wait_reserve, bool no_wait_gpu)
 307{
 308	int ret;
 309
 310	ret = ttm_bo_validate(&nvbo->bo, &nvbo->placement, interruptible,
 311			      no_wait_reserve, no_wait_gpu);
 312	if (ret)
 313		return ret;
 314
 315	if (nvbo->vma.node)
 316		nvbo->bo.offset = nvbo->vma.offset;
 317	return 0;
 318}
 319
 320u16
 321nouveau_bo_rd16(struct nouveau_bo *nvbo, unsigned index)
 322{
 323	bool is_iomem;
 324	u16 *mem = ttm_kmap_obj_virtual(&nvbo->kmap, &is_iomem);
 325	mem = &mem[index];
 326	if (is_iomem)
 327		return ioread16_native((void __force __iomem *)mem);
 328	else
 329		return *mem;
 330}
 331
 332void
 333nouveau_bo_wr16(struct nouveau_bo *nvbo, unsigned index, u16 val)
 334{
 335	bool is_iomem;
 336	u16 *mem = ttm_kmap_obj_virtual(&nvbo->kmap, &is_iomem);
 337	mem = &mem[index];
 338	if (is_iomem)
 339		iowrite16_native(val, (void __force __iomem *)mem);
 340	else
 341		*mem = val;
 342}
 343
 344u32
 345nouveau_bo_rd32(struct nouveau_bo *nvbo, unsigned index)
 346{
 347	bool is_iomem;
 348	u32 *mem = ttm_kmap_obj_virtual(&nvbo->kmap, &is_iomem);
 349	mem = &mem[index];
 350	if (is_iomem)
 351		return ioread32_native((void __force __iomem *)mem);
 352	else
 353		return *mem;
 354}
 355
 356void
 357nouveau_bo_wr32(struct nouveau_bo *nvbo, unsigned index, u32 val)
 358{
 359	bool is_iomem;
 360	u32 *mem = ttm_kmap_obj_virtual(&nvbo->kmap, &is_iomem);
 361	mem = &mem[index];
 362	if (is_iomem)
 363		iowrite32_native(val, (void __force __iomem *)mem);
 364	else
 365		*mem = val;
 366}
 367
 368static struct ttm_backend *
 369nouveau_bo_create_ttm_backend_entry(struct ttm_bo_device *bdev)
 370{
 371	struct drm_nouveau_private *dev_priv = nouveau_bdev(bdev);
 372	struct drm_device *dev = dev_priv->dev;
 373
 374	switch (dev_priv->gart_info.type) {
 375#if __OS_HAS_AGP
 376	case NOUVEAU_GART_AGP:
 377		return ttm_agp_backend_init(bdev, dev->agp->bridge);
 378#endif
 379	case NOUVEAU_GART_PDMA:
 380	case NOUVEAU_GART_HW:
 381		return nouveau_sgdma_init_ttm(dev);
 382	default:
 383		NV_ERROR(dev, "Unknown GART type %d\n",
 384			 dev_priv->gart_info.type);
 385		break;
 386	}
 387
 388	return NULL;
 389}
 390
 391static int
 392nouveau_bo_invalidate_caches(struct ttm_bo_device *bdev, uint32_t flags)
 393{
 394	/* We'll do this from user space. */
 395	return 0;
 396}
 397
 398static int
 399nouveau_bo_init_mem_type(struct ttm_bo_device *bdev, uint32_t type,
 400			 struct ttm_mem_type_manager *man)
 401{
 402	struct drm_nouveau_private *dev_priv = nouveau_bdev(bdev);
 403	struct drm_device *dev = dev_priv->dev;
 404
 405	switch (type) {
 406	case TTM_PL_SYSTEM:
 407		man->flags = TTM_MEMTYPE_FLAG_MAPPABLE;
 408		man->available_caching = TTM_PL_MASK_CACHING;
 409		man->default_caching = TTM_PL_FLAG_CACHED;
 410		break;
 411	case TTM_PL_VRAM:
 412		if (dev_priv->card_type >= NV_50) {
 413			man->func = &nouveau_vram_manager;
 414			man->io_reserve_fastpath = false;
 415			man->use_io_reserve_lru = true;
 416		} else {
 417			man->func = &ttm_bo_manager_func;
 418		}
 419		man->flags = TTM_MEMTYPE_FLAG_FIXED |
 420			     TTM_MEMTYPE_FLAG_MAPPABLE;
 421		man->available_caching = TTM_PL_FLAG_UNCACHED |
 422					 TTM_PL_FLAG_WC;
 423		man->default_caching = TTM_PL_FLAG_WC;
 424		break;
 425	case TTM_PL_TT:
 426		if (dev_priv->card_type >= NV_50)
 427			man->func = &nouveau_gart_manager;
 428		else
 429			man->func = &ttm_bo_manager_func;
 430		switch (dev_priv->gart_info.type) {
 431		case NOUVEAU_GART_AGP:
 432			man->flags = TTM_MEMTYPE_FLAG_MAPPABLE;
 433			man->available_caching = TTM_PL_FLAG_UNCACHED |
 434				TTM_PL_FLAG_WC;
 435			man->default_caching = TTM_PL_FLAG_WC;
 436			break;
 437		case NOUVEAU_GART_PDMA:
 438		case NOUVEAU_GART_HW:
 439			man->flags = TTM_MEMTYPE_FLAG_MAPPABLE |
 440				     TTM_MEMTYPE_FLAG_CMA;
 441			man->available_caching = TTM_PL_MASK_CACHING;
 442			man->default_caching = TTM_PL_FLAG_CACHED;
 443			man->gpu_offset = dev_priv->gart_info.aper_base;
 444			break;
 445		default:
 446			NV_ERROR(dev, "Unknown GART type: %d\n",
 447				 dev_priv->gart_info.type);
 448			return -EINVAL;
 449		}
 450		break;
 451	default:
 452		NV_ERROR(dev, "Unsupported memory type %u\n", (unsigned)type);
 453		return -EINVAL;
 454	}
 455	return 0;
 456}
 457
 458static void
 459nouveau_bo_evict_flags(struct ttm_buffer_object *bo, struct ttm_placement *pl)
 460{
 461	struct nouveau_bo *nvbo = nouveau_bo(bo);
 462
 463	switch (bo->mem.mem_type) {
 464	case TTM_PL_VRAM:
 465		nouveau_bo_placement_set(nvbo, TTM_PL_FLAG_TT,
 466					 TTM_PL_FLAG_SYSTEM);
 467		break;
 468	default:
 469		nouveau_bo_placement_set(nvbo, TTM_PL_FLAG_SYSTEM, 0);
 470		break;
 471	}
 472
 473	*pl = nvbo->placement;
 474}
 475
 476
 477/* GPU-assisted copy using NV_MEMORY_TO_MEMORY_FORMAT, can access
 478 * TTM_PL_{VRAM,TT} directly.
 479 */
 480
 481static int
 482nouveau_bo_move_accel_cleanup(struct nouveau_channel *chan,
 483			      struct nouveau_bo *nvbo, bool evict,
 484			      bool no_wait_reserve, bool no_wait_gpu,
 485			      struct ttm_mem_reg *new_mem)
 486{
 487	struct nouveau_fence *fence = NULL;
 488	int ret;
 489
 490	ret = nouveau_fence_new(chan, &fence, true);
 491	if (ret)
 492		return ret;
 493
 494	ret = ttm_bo_move_accel_cleanup(&nvbo->bo, fence, NULL, evict,
 495					no_wait_reserve, no_wait_gpu, new_mem);
 496	nouveau_fence_unref(&fence);
 497	return ret;
 498}
 499
 500static int
 501nvc0_bo_move_m2mf(struct nouveau_channel *chan, struct ttm_buffer_object *bo,
 502		  struct ttm_mem_reg *old_mem, struct ttm_mem_reg *new_mem)
 503{
 504	struct nouveau_mem *old_node = old_mem->mm_node;
 505	struct nouveau_mem *new_node = new_mem->mm_node;
 506	struct nouveau_bo *nvbo = nouveau_bo(bo);
 507	u32 page_count = new_mem->num_pages;
 508	u64 src_offset, dst_offset;
 509	int ret;
 510
 511	src_offset = old_node->tmp_vma.offset;
 512	if (new_node->tmp_vma.node)
 513		dst_offset = new_node->tmp_vma.offset;
 514	else
 515		dst_offset = nvbo->vma.offset;
 516
 517	page_count = new_mem->num_pages;
 518	while (page_count) {
 519		int line_count = (page_count > 2047) ? 2047 : page_count;
 520
 521		ret = RING_SPACE(chan, 12);
 522		if (ret)
 523			return ret;
 524
 525		BEGIN_NVC0(chan, 2, NvSubM2MF, 0x0238, 2);
 526		OUT_RING  (chan, upper_32_bits(dst_offset));
 527		OUT_RING  (chan, lower_32_bits(dst_offset));
 528		BEGIN_NVC0(chan, 2, NvSubM2MF, 0x030c, 6);
 529		OUT_RING  (chan, upper_32_bits(src_offset));
 530		OUT_RING  (chan, lower_32_bits(src_offset));
 531		OUT_RING  (chan, PAGE_SIZE); /* src_pitch */
 532		OUT_RING  (chan, PAGE_SIZE); /* dst_pitch */
 533		OUT_RING  (chan, PAGE_SIZE); /* line_length */
 534		OUT_RING  (chan, line_count);
 535		BEGIN_NVC0(chan, 2, NvSubM2MF, 0x0300, 1);
 536		OUT_RING  (chan, 0x00100110);
 537
 538		page_count -= line_count;
 539		src_offset += (PAGE_SIZE * line_count);
 540		dst_offset += (PAGE_SIZE * line_count);
 541	}
 542
 543	return 0;
 544}
 545
 546static int
 547nv50_bo_move_m2mf(struct nouveau_channel *chan, struct ttm_buffer_object *bo,
 548		  struct ttm_mem_reg *old_mem, struct ttm_mem_reg *new_mem)
 549{
 550	struct nouveau_mem *old_node = old_mem->mm_node;
 551	struct nouveau_mem *new_node = new_mem->mm_node;
 552	struct nouveau_bo *nvbo = nouveau_bo(bo);
 553	u64 length = (new_mem->num_pages << PAGE_SHIFT);
 554	u64 src_offset, dst_offset;
 555	int ret;
 556
 557	src_offset = old_node->tmp_vma.offset;
 558	if (new_node->tmp_vma.node)
 559		dst_offset = new_node->tmp_vma.offset;
 560	else
 561		dst_offset = nvbo->vma.offset;
 562
 563	while (length) {
 564		u32 amount, stride, height;
 565
 566		amount  = min(length, (u64)(4 * 1024 * 1024));
 567		stride  = 16 * 4;
 568		height  = amount / stride;
 569
 570		if (new_mem->mem_type == TTM_PL_VRAM &&
 571		    nouveau_bo_tile_layout(nvbo)) {
 572			ret = RING_SPACE(chan, 8);
 573			if (ret)
 574				return ret;
 575
 576			BEGIN_RING(chan, NvSubM2MF, 0x0200, 7);
 577			OUT_RING  (chan, 0);
 578			OUT_RING  (chan, 0);
 579			OUT_RING  (chan, stride);
 580			OUT_RING  (chan, height);
 581			OUT_RING  (chan, 1);
 582			OUT_RING  (chan, 0);
 583			OUT_RING  (chan, 0);
 584		} else {
 585			ret = RING_SPACE(chan, 2);
 586			if (ret)
 587				return ret;
 588
 589			BEGIN_RING(chan, NvSubM2MF, 0x0200, 1);
 590			OUT_RING  (chan, 1);
 591		}
 592		if (old_mem->mem_type == TTM_PL_VRAM &&
 593		    nouveau_bo_tile_layout(nvbo)) {
 594			ret = RING_SPACE(chan, 8);
 595			if (ret)
 596				return ret;
 597
 598			BEGIN_RING(chan, NvSubM2MF, 0x021c, 7);
 599			OUT_RING  (chan, 0);
 600			OUT_RING  (chan, 0);
 601			OUT_RING  (chan, stride);
 602			OUT_RING  (chan, height);
 603			OUT_RING  (chan, 1);
 604			OUT_RING  (chan, 0);
 605			OUT_RING  (chan, 0);
 606		} else {
 607			ret = RING_SPACE(chan, 2);
 608			if (ret)
 609				return ret;
 610
 611			BEGIN_RING(chan, NvSubM2MF, 0x021c, 1);
 612			OUT_RING  (chan, 1);
 613		}
 614
 615		ret = RING_SPACE(chan, 14);
 616		if (ret)
 617			return ret;
 618
 619		BEGIN_RING(chan, NvSubM2MF, 0x0238, 2);
 620		OUT_RING  (chan, upper_32_bits(src_offset));
 621		OUT_RING  (chan, upper_32_bits(dst_offset));
 622		BEGIN_RING(chan, NvSubM2MF, 0x030c, 8);
 623		OUT_RING  (chan, lower_32_bits(src_offset));
 624		OUT_RING  (chan, lower_32_bits(dst_offset));
 625		OUT_RING  (chan, stride);
 626		OUT_RING  (chan, stride);
 627		OUT_RING  (chan, stride);
 628		OUT_RING  (chan, height);
 629		OUT_RING  (chan, 0x00000101);
 630		OUT_RING  (chan, 0x00000000);
 631		BEGIN_RING(chan, NvSubM2MF, NV_MEMORY_TO_MEMORY_FORMAT_NOP, 1);
 632		OUT_RING  (chan, 0);
 633
 634		length -= amount;
 635		src_offset += amount;
 636		dst_offset += amount;
 637	}
 638
 639	return 0;
 640}
 641
 642static inline uint32_t
 643nouveau_bo_mem_ctxdma(struct ttm_buffer_object *bo,
 644		      struct nouveau_channel *chan, struct ttm_mem_reg *mem)
 645{
 646	if (mem->mem_type == TTM_PL_TT)
 647		return chan->gart_handle;
 648	return chan->vram_handle;
 649}
 650
 651static int
 652nv04_bo_move_m2mf(struct nouveau_channel *chan, struct ttm_buffer_object *bo,
 653		  struct ttm_mem_reg *old_mem, struct ttm_mem_reg *new_mem)
 654{
 655	u32 src_offset = old_mem->start << PAGE_SHIFT;
 656	u32 dst_offset = new_mem->start << PAGE_SHIFT;
 657	u32 page_count = new_mem->num_pages;
 658	int ret;
 659
 660	ret = RING_SPACE(chan, 3);
 661	if (ret)
 662		return ret;
 663
 664	BEGIN_RING(chan, NvSubM2MF, NV_MEMORY_TO_MEMORY_FORMAT_DMA_SOURCE, 2);
 665	OUT_RING  (chan, nouveau_bo_mem_ctxdma(bo, chan, old_mem));
 666	OUT_RING  (chan, nouveau_bo_mem_ctxdma(bo, chan, new_mem));
 667
 668	page_count = new_mem->num_pages;
 669	while (page_count) {
 670		int line_count = (page_count > 2047) ? 2047 : page_count;
 671
 672		ret = RING_SPACE(chan, 11);
 673		if (ret)
 674			return ret;
 675
 676		BEGIN_RING(chan, NvSubM2MF,
 677				 NV_MEMORY_TO_MEMORY_FORMAT_OFFSET_IN, 8);
 678		OUT_RING  (chan, src_offset);
 679		OUT_RING  (chan, dst_offset);
 680		OUT_RING  (chan, PAGE_SIZE); /* src_pitch */
 681		OUT_RING  (chan, PAGE_SIZE); /* dst_pitch */
 682		OUT_RING  (chan, PAGE_SIZE); /* line_length */
 683		OUT_RING  (chan, line_count);
 684		OUT_RING  (chan, 0x00000101);
 685		OUT_RING  (chan, 0x00000000);
 686		BEGIN_RING(chan, NvSubM2MF, NV_MEMORY_TO_MEMORY_FORMAT_NOP, 1);
 687		OUT_RING  (chan, 0);
 688
 689		page_count -= line_count;
 690		src_offset += (PAGE_SIZE * line_count);
 691		dst_offset += (PAGE_SIZE * line_count);
 692	}
 693
 694	return 0;
 695}
 696
 697static int
 698nouveau_bo_move_m2mf(struct ttm_buffer_object *bo, int evict, bool intr,
 699		     bool no_wait_reserve, bool no_wait_gpu,
 700		     struct ttm_mem_reg *new_mem)
 701{
 702	struct drm_nouveau_private *dev_priv = nouveau_bdev(bo->bdev);
 703	struct nouveau_bo *nvbo = nouveau_bo(bo);
 704	struct ttm_mem_reg *old_mem = &bo->mem;
 705	struct nouveau_channel *chan;
 706	int ret;
 707
 708	chan = nvbo->channel;
 709	if (!chan) {
 710		chan = dev_priv->channel;
 711		mutex_lock_nested(&chan->mutex, NOUVEAU_KCHANNEL_MUTEX);
 712	}
 713
 714	/* create temporary vma for old memory, this will get cleaned
 715	 * up after ttm destroys the ttm_mem_reg
 716	 */
 717	if (dev_priv->card_type >= NV_50) {
 718		struct nouveau_mem *node = old_mem->mm_node;
 719		if (!node->tmp_vma.node) {
 720			u32 page_shift = nvbo->vma.node->type;
 721			if (old_mem->mem_type == TTM_PL_TT)
 722				page_shift = nvbo->vma.vm->spg_shift;
 723
 724			ret = nouveau_vm_get(chan->vm,
 725					     old_mem->num_pages << PAGE_SHIFT,
 726					     page_shift, NV_MEM_ACCESS_RO,
 727					     &node->tmp_vma);
 728			if (ret)
 729				goto out;
 730		}
 731
 732		if (old_mem->mem_type == TTM_PL_VRAM)
 733			nouveau_vm_map(&node->tmp_vma, node);
 734		else {
 735			nouveau_vm_map_sg(&node->tmp_vma, 0,
 736					  old_mem->num_pages << PAGE_SHIFT,
 737					  node, node->pages);
 738		}
 739	}
 740
 741	if (dev_priv->card_type < NV_50)
 742		ret = nv04_bo_move_m2mf(chan, bo, &bo->mem, new_mem);
 743	else
 744	if (dev_priv->card_type < NV_C0)
 745		ret = nv50_bo_move_m2mf(chan, bo, &bo->mem, new_mem);
 746	else
 747		ret = nvc0_bo_move_m2mf(chan, bo, &bo->mem, new_mem);
 748	if (ret == 0) {
 749		ret = nouveau_bo_move_accel_cleanup(chan, nvbo, evict,
 750						    no_wait_reserve,
 751						    no_wait_gpu, new_mem);
 752	}
 753
 754out:
 755	if (chan == dev_priv->channel)
 756		mutex_unlock(&chan->mutex);
 757	return ret;
 758}
 759
 760static int
 761nouveau_bo_move_flipd(struct ttm_buffer_object *bo, bool evict, bool intr,
 762		      bool no_wait_reserve, bool no_wait_gpu,
 763		      struct ttm_mem_reg *new_mem)
 764{
 765	struct drm_nouveau_private *dev_priv = nouveau_bdev(bo->bdev);
 766	u32 placement_memtype = TTM_PL_FLAG_TT | TTM_PL_MASK_CACHING;
 767	struct ttm_placement placement;
 768	struct ttm_mem_reg tmp_mem;
 769	int ret;
 770
 771	placement.fpfn = placement.lpfn = 0;
 772	placement.num_placement = placement.num_busy_placement = 1;
 773	placement.placement = placement.busy_placement = &placement_memtype;
 774
 775	tmp_mem = *new_mem;
 776	tmp_mem.mm_node = NULL;
 777	ret = ttm_bo_mem_space(bo, &placement, &tmp_mem, intr, no_wait_reserve, no_wait_gpu);
 778	if (ret)
 779		return ret;
 780
 781	ret = ttm_tt_bind(bo->ttm, &tmp_mem);
 782	if (ret)
 783		goto out;
 784
 785	if (dev_priv->card_type >= NV_50) {
 786		struct nouveau_bo *nvbo = nouveau_bo(bo);
 787		struct nouveau_mem *node = tmp_mem.mm_node;
 788		struct nouveau_vma *vma = &nvbo->vma;
 789		if (vma->node->type != vma->vm->spg_shift)
 790			vma = &node->tmp_vma;
 791		nouveau_vm_map_sg(vma, 0, tmp_mem.num_pages << PAGE_SHIFT,
 792				  node, node->pages);
 793	}
 794
 795	ret = nouveau_bo_move_m2mf(bo, true, intr, no_wait_reserve, no_wait_gpu, &tmp_mem);
 796
 797	if (dev_priv->card_type >= NV_50) {
 798		struct nouveau_bo *nvbo = nouveau_bo(bo);
 799		nouveau_vm_unmap(&nvbo->vma);
 800	}
 801
 802	if (ret)
 803		goto out;
 804
 805	ret = ttm_bo_move_ttm(bo, true, no_wait_reserve, no_wait_gpu, new_mem);
 806out:
 807	ttm_bo_mem_put(bo, &tmp_mem);
 808	return ret;
 809}
 810
 811static int
 812nouveau_bo_move_flips(struct ttm_buffer_object *bo, bool evict, bool intr,
 813		      bool no_wait_reserve, bool no_wait_gpu,
 814		      struct ttm_mem_reg *new_mem)
 815{
 816	u32 placement_memtype = TTM_PL_FLAG_TT | TTM_PL_MASK_CACHING;
 817	struct ttm_placement placement;
 818	struct ttm_mem_reg tmp_mem;
 819	int ret;
 820
 821	placement.fpfn = placement.lpfn = 0;
 822	placement.num_placement = placement.num_busy_placement = 1;
 823	placement.placement = placement.busy_placement = &placement_memtype;
 824
 825	tmp_mem = *new_mem;
 826	tmp_mem.mm_node = NULL;
 827	ret = ttm_bo_mem_space(bo, &placement, &tmp_mem, intr, no_wait_reserve, no_wait_gpu);
 828	if (ret)
 829		return ret;
 830
 831	ret = ttm_bo_move_ttm(bo, true, no_wait_reserve, no_wait_gpu, &tmp_mem);
 832	if (ret)
 833		goto out;
 834
 835	ret = nouveau_bo_move_m2mf(bo, true, intr, no_wait_reserve, no_wait_gpu, new_mem);
 836	if (ret)
 837		goto out;
 838
 839out:
 840	ttm_bo_mem_put(bo, &tmp_mem);
 841	return ret;
 842}
 843
 844static void
 845nouveau_bo_move_ntfy(struct ttm_buffer_object *bo, struct ttm_mem_reg *new_mem)
 846{
 847	struct drm_nouveau_private *dev_priv = nouveau_bdev(bo->bdev);
 848	struct nouveau_mem *node = new_mem->mm_node;
 849	struct nouveau_bo *nvbo = nouveau_bo(bo);
 850	struct nouveau_vma *vma = &nvbo->vma;
 851	struct nouveau_vm *vm = vma->vm;
 852
 853	if (dev_priv->card_type < NV_50)
 854		return;
 855
 856	switch (new_mem->mem_type) {
 857	case TTM_PL_VRAM:
 858		nouveau_vm_map(vma, node);
 859		break;
 860	case TTM_PL_TT:
 861		if (vma->node->type != vm->spg_shift) {
 862			nouveau_vm_unmap(vma);
 863			vma = &node->tmp_vma;
 864		}
 865		nouveau_vm_map_sg(vma, 0, new_mem->num_pages << PAGE_SHIFT,
 866				  node, node->pages);
 867		break;
 868	default:
 869		nouveau_vm_unmap(&nvbo->vma);
 870		break;
 871	}
 872}
 873
 874static int
 875nouveau_bo_vm_bind(struct ttm_buffer_object *bo, struct ttm_mem_reg *new_mem,
 876		   struct nouveau_tile_reg **new_tile)
 877{
 878	struct drm_nouveau_private *dev_priv = nouveau_bdev(bo->bdev);
 879	struct drm_device *dev = dev_priv->dev;
 880	struct nouveau_bo *nvbo = nouveau_bo(bo);
 881	u64 offset = new_mem->start << PAGE_SHIFT;
 882
 883	*new_tile = NULL;
 884	if (new_mem->mem_type != TTM_PL_VRAM)
 885		return 0;
 886
 887	if (dev_priv->card_type >= NV_10) {
 888		*new_tile = nv10_mem_set_tiling(dev, offset, new_mem->size,
 889						nvbo->tile_mode,
 890						nvbo->tile_flags);
 891	}
 892
 893	return 0;
 894}
 895
 896static void
 897nouveau_bo_vm_cleanup(struct ttm_buffer_object *bo,
 898		      struct nouveau_tile_reg *new_tile,
 899		      struct nouveau_tile_reg **old_tile)
 900{
 901	struct drm_nouveau_private *dev_priv = nouveau_bdev(bo->bdev);
 902	struct drm_device *dev = dev_priv->dev;
 903
 904	nv10_mem_put_tile_region(dev, *old_tile, bo->sync_obj);
 905	*old_tile = new_tile;
 906}
 907
 908static int
 909nouveau_bo_move(struct ttm_buffer_object *bo, bool evict, bool intr,
 910		bool no_wait_reserve, bool no_wait_gpu,
 911		struct ttm_mem_reg *new_mem)
 912{
 913	struct drm_nouveau_private *dev_priv = nouveau_bdev(bo->bdev);
 914	struct nouveau_bo *nvbo = nouveau_bo(bo);
 915	struct ttm_mem_reg *old_mem = &bo->mem;
 916	struct nouveau_tile_reg *new_tile = NULL;
 917	int ret = 0;
 918
 919	if (dev_priv->card_type < NV_50) {
 920		ret = nouveau_bo_vm_bind(bo, new_mem, &new_tile);
 921		if (ret)
 922			return ret;
 923	}
 924
 925	/* Fake bo copy. */
 926	if (old_mem->mem_type == TTM_PL_SYSTEM && !bo->ttm) {
 927		BUG_ON(bo->mem.mm_node != NULL);
 928		bo->mem = *new_mem;
 929		new_mem->mm_node = NULL;
 930		goto out;
 931	}
 932
 933	/* Software copy if the card isn't up and running yet. */
 934	if (!dev_priv->channel) {
 935		ret = ttm_bo_move_memcpy(bo, evict, no_wait_reserve, no_wait_gpu, new_mem);
 936		goto out;
 937	}
 938
 939	/* Hardware assisted copy. */
 940	if (new_mem->mem_type == TTM_PL_SYSTEM)
 941		ret = nouveau_bo_move_flipd(bo, evict, intr, no_wait_reserve, no_wait_gpu, new_mem);
 942	else if (old_mem->mem_type == TTM_PL_SYSTEM)
 943		ret = nouveau_bo_move_flips(bo, evict, intr, no_wait_reserve, no_wait_gpu, new_mem);
 944	else
 945		ret = nouveau_bo_move_m2mf(bo, evict, intr, no_wait_reserve, no_wait_gpu, new_mem);
 946
 947	if (!ret)
 948		goto out;
 949
 950	/* Fallback to software copy. */
 951	ret = ttm_bo_move_memcpy(bo, evict, no_wait_reserve, no_wait_gpu, new_mem);
 952
 953out:
 954	if (dev_priv->card_type < NV_50) {
 955		if (ret)
 956			nouveau_bo_vm_cleanup(bo, NULL, &new_tile);
 957		else
 958			nouveau_bo_vm_cleanup(bo, new_tile, &nvbo->tile);
 959	}
 960
 961	return ret;
 962}
 963
 964static int
 965nouveau_bo_verify_access(struct ttm_buffer_object *bo, struct file *filp)
 966{
 967	return 0;
 968}
 969
 970static int
 971nouveau_ttm_io_mem_reserve(struct ttm_bo_device *bdev, struct ttm_mem_reg *mem)
 972{
 973	struct ttm_mem_type_manager *man = &bdev->man[mem->mem_type];
 974	struct drm_nouveau_private *dev_priv = nouveau_bdev(bdev);
 975	struct drm_device *dev = dev_priv->dev;
 976	int ret;
 977
 978	mem->bus.addr = NULL;
 979	mem->bus.offset = 0;
 980	mem->bus.size = mem->num_pages << PAGE_SHIFT;
 981	mem->bus.base = 0;
 982	mem->bus.is_iomem = false;
 983	if (!(man->flags & TTM_MEMTYPE_FLAG_MAPPABLE))
 984		return -EINVAL;
 985	switch (mem->mem_type) {
 986	case TTM_PL_SYSTEM:
 987		/* System memory */
 988		return 0;
 989	case TTM_PL_TT:
 990#if __OS_HAS_AGP
 991		if (dev_priv->gart_info.type == NOUVEAU_GART_AGP) {
 992			mem->bus.offset = mem->start << PAGE_SHIFT;
 993			mem->bus.base = dev_priv->gart_info.aper_base;
 994			mem->bus.is_iomem = true;
 995		}
 996#endif
 997		break;
 998	case TTM_PL_VRAM:
 999	{
1000		struct nouveau_mem *node = mem->mm_node;
1001		u8 page_shift;
1002
1003		if (!dev_priv->bar1_vm) {
1004			mem->bus.offset = mem->start << PAGE_SHIFT;
1005			mem->bus.base = pci_resource_start(dev->pdev, 1);
1006			mem->bus.is_iomem = true;
1007			break;
1008		}
1009
1010		if (dev_priv->card_type == NV_C0)
1011			page_shift = node->page_shift;
1012		else
1013			page_shift = 12;
1014
1015		ret = nouveau_vm_get(dev_priv->bar1_vm, mem->bus.size,
1016				     page_shift, NV_MEM_ACCESS_RW,
1017				     &node->bar_vma);
1018		if (ret)
1019			return ret;
1020
1021		nouveau_vm_map(&node->bar_vma, node);
1022		if (ret) {
1023			nouveau_vm_put(&node->bar_vma);
1024			return ret;
1025		}
1026
1027		mem->bus.offset = node->bar_vma.offset;
1028		if (dev_priv->card_type == NV_50) /*XXX*/
1029			mem->bus.offset -= 0x0020000000ULL;
1030		mem->bus.base = pci_resource_start(dev->pdev, 1);
1031		mem->bus.is_iomem = true;
1032	}
1033		break;
1034	default:
1035		return -EINVAL;
1036	}
1037	return 0;
1038}
1039
1040static void
1041nouveau_ttm_io_mem_free(struct ttm_bo_device *bdev, struct ttm_mem_reg *mem)
1042{
1043	struct drm_nouveau_private *dev_priv = nouveau_bdev(bdev);
1044	struct nouveau_mem *node = mem->mm_node;
1045
1046	if (!dev_priv->bar1_vm || mem->mem_type != TTM_PL_VRAM)
1047		return;
1048
1049	if (!node->bar_vma.node)
1050		return;
1051
1052	nouveau_vm_unmap(&node->bar_vma);
1053	nouveau_vm_put(&node->bar_vma);
1054}
1055
1056static int
1057nouveau_ttm_fault_reserve_notify(struct ttm_buffer_object *bo)
1058{
1059	struct drm_nouveau_private *dev_priv = nouveau_bdev(bo->bdev);
1060	struct nouveau_bo *nvbo = nouveau_bo(bo);
1061
1062	/* as long as the bo isn't in vram, and isn't tiled, we've got
1063	 * nothing to do here.
1064	 */
1065	if (bo->mem.mem_type != TTM_PL_VRAM) {
1066		if (dev_priv->card_type < NV_50 ||
1067		    !nouveau_bo_tile_layout(nvbo))
1068			return 0;
1069	}
1070
1071	/* make sure bo is in mappable vram */
1072	if (bo->mem.start + bo->mem.num_pages < dev_priv->fb_mappable_pages)
1073		return 0;
1074
1075
1076	nvbo->placement.fpfn = 0;
1077	nvbo->placement.lpfn = dev_priv->fb_mappable_pages;
1078	nouveau_bo_placement_set(nvbo, TTM_PL_FLAG_VRAM, 0);
1079	return nouveau_bo_validate(nvbo, false, true, false);
1080}
1081
1082void
1083nouveau_bo_fence(struct nouveau_bo *nvbo, struct nouveau_fence *fence)
1084{
1085	struct nouveau_fence *old_fence;
1086
1087	if (likely(fence))
1088		nouveau_fence_ref(fence);
1089
1090	spin_lock(&nvbo->bo.bdev->fence_lock);
1091	old_fence = nvbo->bo.sync_obj;
1092	nvbo->bo.sync_obj = fence;
1093	spin_unlock(&nvbo->bo.bdev->fence_lock);
1094
1095	nouveau_fence_unref(&old_fence);
1096}
1097
1098struct ttm_bo_driver nouveau_bo_driver = {
1099	.create_ttm_backend_entry = nouveau_bo_create_ttm_backend_entry,
1100	.invalidate_caches = nouveau_bo_invalidate_caches,
1101	.init_mem_type = nouveau_bo_init_mem_type,
1102	.evict_flags = nouveau_bo_evict_flags,
1103	.move_notify = nouveau_bo_move_ntfy,
1104	.move = nouveau_bo_move,
1105	.verify_access = nouveau_bo_verify_access,
1106	.sync_obj_signaled = __nouveau_fence_signalled,
1107	.sync_obj_wait = __nouveau_fence_wait,
1108	.sync_obj_flush = __nouveau_fence_flush,
1109	.sync_obj_unref = __nouveau_fence_unref,
1110	.sync_obj_ref = __nouveau_fence_ref,
1111	.fault_reserve_notify = &nouveau_ttm_fault_reserve_notify,
1112	.io_mem_reserve = &nouveau_ttm_io_mem_reserve,
1113	.io_mem_free = &nouveau_ttm_io_mem_free,
1114};
1115