nouveau_gem.c revision 44d847b7439bdea0b6c5640446427daa3ebcc7fa
1/* 2 * Copyright (C) 2008 Ben Skeggs. 3 * All Rights Reserved. 4 * 5 * Permission is hereby granted, free of charge, to any person obtaining 6 * a copy of this software and associated documentation files (the 7 * "Software"), to deal in the Software without restriction, including 8 * without limitation the rights to use, copy, modify, merge, publish, 9 * distribute, sublicense, and/or sell copies of the Software, and to 10 * permit persons to whom the Software is furnished to do so, subject to 11 * the following conditions: 12 * 13 * The above copyright notice and this permission notice (including the 14 * next paragraph) shall be included in all copies or substantial 15 * portions of the Software. 16 * 17 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 18 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 19 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. 20 * IN NO EVENT SHALL THE COPYRIGHT OWNER(S) AND/OR ITS SUPPLIERS BE 21 * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION 22 * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION 23 * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 24 * 25 */ 26 27#include <subdev/fb.h> 28 29#include "nouveau_drm.h" 30#include "nouveau_dma.h" 31#include "nouveau_fence.h" 32#include "nouveau_abi16.h" 33 34#include "nouveau_ttm.h" 35#include "nouveau_gem.h" 36 37void 38nouveau_gem_object_del(struct drm_gem_object *gem) 39{ 40 struct nouveau_bo *nvbo = nouveau_gem_object(gem); 41 struct ttm_buffer_object *bo = &nvbo->bo; 42 43 if (gem->import_attach) 44 drm_prime_gem_destroy(gem, nvbo->bo.sg); 45 46 drm_gem_object_release(gem); 47 48 /* reset filp so nouveau_bo_del_ttm() can test for it */ 49 gem->filp = NULL; 50 ttm_bo_unref(&bo); 51} 52 53int 54nouveau_gem_object_open(struct drm_gem_object *gem, struct drm_file *file_priv) 55{ 56 struct nouveau_cli *cli = nouveau_cli(file_priv); 57 struct nouveau_bo *nvbo = nouveau_gem_object(gem); 58 struct nouveau_vma *vma; 59 int ret; 60 61 if (!cli->base.vm) 62 return 0; 63 64 ret = ttm_bo_reserve(&nvbo->bo, false, false, false, 0); 65 if (ret) 66 return ret; 67 68 vma = nouveau_bo_vma_find(nvbo, cli->base.vm); 69 if (!vma) { 70 vma = kzalloc(sizeof(*vma), GFP_KERNEL); 71 if (!vma) { 72 ret = -ENOMEM; 73 goto out; 74 } 75 76 ret = nouveau_bo_vma_add(nvbo, cli->base.vm, vma); 77 if (ret) { 78 kfree(vma); 79 goto out; 80 } 81 } else { 82 vma->refcount++; 83 } 84 85out: 86 ttm_bo_unreserve(&nvbo->bo); 87 return ret; 88} 89 90static void 91nouveau_gem_object_delete(void *data) 92{ 93 struct nouveau_vma *vma = data; 94 nouveau_vm_unmap(vma); 95 nouveau_vm_put(vma); 96 kfree(vma); 97} 98 99static void 100nouveau_gem_object_unmap(struct nouveau_bo *nvbo, struct nouveau_vma *vma) 101{ 102 const bool mapped = nvbo->bo.mem.mem_type != TTM_PL_SYSTEM; 103 struct nouveau_fence *fence = NULL; 104 105 list_del(&vma->head); 106 107 if (mapped) { 108 spin_lock(&nvbo->bo.bdev->fence_lock); 109 fence = nouveau_fence_ref(nvbo->bo.sync_obj); 110 spin_unlock(&nvbo->bo.bdev->fence_lock); 111 } 112 113 if (fence) { 114 nouveau_fence_work(fence, nouveau_gem_object_delete, vma); 115 } else { 116 if (mapped) 117 nouveau_vm_unmap(vma); 118 nouveau_vm_put(vma); 119 kfree(vma); 120 } 121 nouveau_fence_unref(&fence); 122} 123 124void 125nouveau_gem_object_close(struct drm_gem_object *gem, struct drm_file *file_priv) 126{ 127 struct nouveau_cli *cli = nouveau_cli(file_priv); 128 struct nouveau_bo *nvbo = nouveau_gem_object(gem); 129 struct nouveau_vma *vma; 130 int ret; 131 132 if (!cli->base.vm) 133 return; 134 135 ret = ttm_bo_reserve(&nvbo->bo, false, false, false, 0); 136 if (ret) 137 return; 138 139 vma = nouveau_bo_vma_find(nvbo, cli->base.vm); 140 if (vma) { 141 if (--vma->refcount == 0) 142 nouveau_gem_object_unmap(nvbo, vma); 143 } 144 ttm_bo_unreserve(&nvbo->bo); 145} 146 147int 148nouveau_gem_new(struct drm_device *dev, int size, int align, uint32_t domain, 149 uint32_t tile_mode, uint32_t tile_flags, 150 struct nouveau_bo **pnvbo) 151{ 152 struct nouveau_drm *drm = nouveau_drm(dev); 153 struct nouveau_bo *nvbo; 154 u32 flags = 0; 155 int ret; 156 157 if (domain & NOUVEAU_GEM_DOMAIN_VRAM) 158 flags |= TTM_PL_FLAG_VRAM; 159 if (domain & NOUVEAU_GEM_DOMAIN_GART) 160 flags |= TTM_PL_FLAG_TT; 161 if (!flags || domain & NOUVEAU_GEM_DOMAIN_CPU) 162 flags |= TTM_PL_FLAG_SYSTEM; 163 164 ret = nouveau_bo_new(dev, size, align, flags, tile_mode, 165 tile_flags, NULL, pnvbo); 166 if (ret) 167 return ret; 168 nvbo = *pnvbo; 169 170 /* we restrict allowed domains on nv50+ to only the types 171 * that were requested at creation time. not possibly on 172 * earlier chips without busting the ABI. 173 */ 174 nvbo->valid_domains = NOUVEAU_GEM_DOMAIN_VRAM | 175 NOUVEAU_GEM_DOMAIN_GART; 176 if (nv_device(drm->device)->card_type >= NV_50) 177 nvbo->valid_domains &= domain; 178 179 /* Initialize the embedded gem-object. We return a single gem-reference 180 * to the caller, instead of a normal nouveau_bo ttm reference. */ 181 ret = drm_gem_object_init(dev, &nvbo->gem, nvbo->bo.mem.size); 182 if (ret) { 183 nouveau_bo_ref(NULL, pnvbo); 184 return -ENOMEM; 185 } 186 187 nvbo->bo.persistent_swap_storage = nvbo->gem.filp; 188 return 0; 189} 190 191static int 192nouveau_gem_info(struct drm_file *file_priv, struct drm_gem_object *gem, 193 struct drm_nouveau_gem_info *rep) 194{ 195 struct nouveau_cli *cli = nouveau_cli(file_priv); 196 struct nouveau_bo *nvbo = nouveau_gem_object(gem); 197 struct nouveau_vma *vma; 198 199 if (nvbo->bo.mem.mem_type == TTM_PL_TT) 200 rep->domain = NOUVEAU_GEM_DOMAIN_GART; 201 else 202 rep->domain = NOUVEAU_GEM_DOMAIN_VRAM; 203 204 rep->offset = nvbo->bo.offset; 205 if (cli->base.vm) { 206 vma = nouveau_bo_vma_find(nvbo, cli->base.vm); 207 if (!vma) 208 return -EINVAL; 209 210 rep->offset = vma->offset; 211 } 212 213 rep->size = nvbo->bo.mem.num_pages << PAGE_SHIFT; 214 rep->map_handle = drm_vma_node_offset_addr(&nvbo->bo.vma_node); 215 rep->tile_mode = nvbo->tile_mode; 216 rep->tile_flags = nvbo->tile_flags; 217 return 0; 218} 219 220int 221nouveau_gem_ioctl_new(struct drm_device *dev, void *data, 222 struct drm_file *file_priv) 223{ 224 struct nouveau_drm *drm = nouveau_drm(dev); 225 struct nouveau_cli *cli = nouveau_cli(file_priv); 226 struct nouveau_fb *pfb = nouveau_fb(drm->device); 227 struct drm_nouveau_gem_new *req = data; 228 struct nouveau_bo *nvbo = NULL; 229 int ret = 0; 230 231 if (!pfb->memtype_valid(pfb, req->info.tile_flags)) { 232 NV_ERROR(cli, "bad page flags: 0x%08x\n", req->info.tile_flags); 233 return -EINVAL; 234 } 235 236 ret = nouveau_gem_new(dev, req->info.size, req->align, 237 req->info.domain, req->info.tile_mode, 238 req->info.tile_flags, &nvbo); 239 if (ret) 240 return ret; 241 242 ret = drm_gem_handle_create(file_priv, &nvbo->gem, &req->info.handle); 243 if (ret == 0) { 244 ret = nouveau_gem_info(file_priv, &nvbo->gem, &req->info); 245 if (ret) 246 drm_gem_handle_delete(file_priv, req->info.handle); 247 } 248 249 /* drop reference from allocate - handle holds it now */ 250 drm_gem_object_unreference_unlocked(&nvbo->gem); 251 return ret; 252} 253 254static int 255nouveau_gem_set_domain(struct drm_gem_object *gem, uint32_t read_domains, 256 uint32_t write_domains, uint32_t valid_domains) 257{ 258 struct nouveau_bo *nvbo = nouveau_gem_object(gem); 259 struct ttm_buffer_object *bo = &nvbo->bo; 260 uint32_t domains = valid_domains & nvbo->valid_domains & 261 (write_domains ? write_domains : read_domains); 262 uint32_t pref_flags = 0, valid_flags = 0; 263 264 if (!domains) 265 return -EINVAL; 266 267 if (valid_domains & NOUVEAU_GEM_DOMAIN_VRAM) 268 valid_flags |= TTM_PL_FLAG_VRAM; 269 270 if (valid_domains & NOUVEAU_GEM_DOMAIN_GART) 271 valid_flags |= TTM_PL_FLAG_TT; 272 273 if ((domains & NOUVEAU_GEM_DOMAIN_VRAM) && 274 bo->mem.mem_type == TTM_PL_VRAM) 275 pref_flags |= TTM_PL_FLAG_VRAM; 276 277 else if ((domains & NOUVEAU_GEM_DOMAIN_GART) && 278 bo->mem.mem_type == TTM_PL_TT) 279 pref_flags |= TTM_PL_FLAG_TT; 280 281 else if (domains & NOUVEAU_GEM_DOMAIN_VRAM) 282 pref_flags |= TTM_PL_FLAG_VRAM; 283 284 else 285 pref_flags |= TTM_PL_FLAG_TT; 286 287 nouveau_bo_placement_set(nvbo, pref_flags, valid_flags); 288 289 return 0; 290} 291 292struct validate_op { 293 struct list_head vram_list; 294 struct list_head gart_list; 295 struct list_head both_list; 296 struct ww_acquire_ctx ticket; 297}; 298 299static void 300validate_fini_list(struct list_head *list, struct nouveau_fence *fence, 301 struct ww_acquire_ctx *ticket) 302{ 303 struct list_head *entry, *tmp; 304 struct nouveau_bo *nvbo; 305 306 list_for_each_safe(entry, tmp, list) { 307 nvbo = list_entry(entry, struct nouveau_bo, entry); 308 309 if (likely(fence)) 310 nouveau_bo_fence(nvbo, fence); 311 312 if (unlikely(nvbo->validate_mapped)) { 313 ttm_bo_kunmap(&nvbo->kmap); 314 nvbo->validate_mapped = false; 315 } 316 317 list_del(&nvbo->entry); 318 nvbo->reserved_by = NULL; 319 ttm_bo_unreserve_ticket(&nvbo->bo, ticket); 320 drm_gem_object_unreference_unlocked(&nvbo->gem); 321 } 322} 323 324static void 325validate_fini_no_ticket(struct validate_op *op, struct nouveau_fence *fence) 326{ 327 validate_fini_list(&op->vram_list, fence, &op->ticket); 328 validate_fini_list(&op->gart_list, fence, &op->ticket); 329 validate_fini_list(&op->both_list, fence, &op->ticket); 330} 331 332static void 333validate_fini(struct validate_op *op, struct nouveau_fence *fence) 334{ 335 validate_fini_no_ticket(op, fence); 336 ww_acquire_fini(&op->ticket); 337} 338 339static int 340validate_init(struct nouveau_channel *chan, struct drm_file *file_priv, 341 struct drm_nouveau_gem_pushbuf_bo *pbbo, 342 int nr_buffers, struct validate_op *op) 343{ 344 struct nouveau_cli *cli = nouveau_cli(file_priv); 345 struct drm_device *dev = chan->drm->dev; 346 int trycnt = 0; 347 int ret, i; 348 struct nouveau_bo *res_bo = NULL; 349 350 ww_acquire_init(&op->ticket, &reservation_ww_class); 351retry: 352 if (++trycnt > 100000) { 353 NV_ERROR(cli, "%s failed and gave up.\n", __func__); 354 return -EINVAL; 355 } 356 357 for (i = 0; i < nr_buffers; i++) { 358 struct drm_nouveau_gem_pushbuf_bo *b = &pbbo[i]; 359 struct drm_gem_object *gem; 360 struct nouveau_bo *nvbo; 361 362 gem = drm_gem_object_lookup(dev, file_priv, b->handle); 363 if (!gem) { 364 NV_ERROR(cli, "Unknown handle 0x%08x\n", b->handle); 365 ww_acquire_done(&op->ticket); 366 validate_fini(op, NULL); 367 return -ENOENT; 368 } 369 nvbo = nouveau_gem_object(gem); 370 if (nvbo == res_bo) { 371 res_bo = NULL; 372 drm_gem_object_unreference_unlocked(gem); 373 continue; 374 } 375 376 if (nvbo->reserved_by && nvbo->reserved_by == file_priv) { 377 NV_ERROR(cli, "multiple instances of buffer %d on " 378 "validation list\n", b->handle); 379 drm_gem_object_unreference_unlocked(gem); 380 ww_acquire_done(&op->ticket); 381 validate_fini(op, NULL); 382 return -EINVAL; 383 } 384 385 ret = ttm_bo_reserve(&nvbo->bo, true, false, true, &op->ticket); 386 if (ret) { 387 validate_fini_no_ticket(op, NULL); 388 if (unlikely(ret == -EDEADLK)) { 389 ret = ttm_bo_reserve_slowpath(&nvbo->bo, true, 390 &op->ticket); 391 if (!ret) 392 res_bo = nvbo; 393 } 394 if (unlikely(ret)) { 395 ww_acquire_done(&op->ticket); 396 ww_acquire_fini(&op->ticket); 397 drm_gem_object_unreference_unlocked(gem); 398 if (ret != -ERESTARTSYS) 399 NV_ERROR(cli, "fail reserve\n"); 400 return ret; 401 } 402 } 403 404 b->user_priv = (uint64_t)(unsigned long)nvbo; 405 nvbo->reserved_by = file_priv; 406 nvbo->pbbo_index = i; 407 if ((b->valid_domains & NOUVEAU_GEM_DOMAIN_VRAM) && 408 (b->valid_domains & NOUVEAU_GEM_DOMAIN_GART)) 409 list_add_tail(&nvbo->entry, &op->both_list); 410 else 411 if (b->valid_domains & NOUVEAU_GEM_DOMAIN_VRAM) 412 list_add_tail(&nvbo->entry, &op->vram_list); 413 else 414 if (b->valid_domains & NOUVEAU_GEM_DOMAIN_GART) 415 list_add_tail(&nvbo->entry, &op->gart_list); 416 else { 417 NV_ERROR(cli, "invalid valid domains: 0x%08x\n", 418 b->valid_domains); 419 list_add_tail(&nvbo->entry, &op->both_list); 420 ww_acquire_done(&op->ticket); 421 validate_fini(op, NULL); 422 return -EINVAL; 423 } 424 if (nvbo == res_bo) 425 goto retry; 426 } 427 428 ww_acquire_done(&op->ticket); 429 return 0; 430} 431 432static int 433validate_sync(struct nouveau_channel *chan, struct nouveau_bo *nvbo) 434{ 435 struct nouveau_fence *fence = NULL; 436 int ret = 0; 437 438 spin_lock(&nvbo->bo.bdev->fence_lock); 439 fence = nouveau_fence_ref(nvbo->bo.sync_obj); 440 spin_unlock(&nvbo->bo.bdev->fence_lock); 441 442 if (fence) { 443 ret = nouveau_fence_sync(fence, chan); 444 nouveau_fence_unref(&fence); 445 } 446 447 return ret; 448} 449 450static int 451validate_list(struct nouveau_channel *chan, struct nouveau_cli *cli, 452 struct list_head *list, struct drm_nouveau_gem_pushbuf_bo *pbbo, 453 uint64_t user_pbbo_ptr) 454{ 455 struct nouveau_drm *drm = chan->drm; 456 struct drm_nouveau_gem_pushbuf_bo __user *upbbo = 457 (void __force __user *)(uintptr_t)user_pbbo_ptr; 458 struct nouveau_bo *nvbo; 459 int ret, relocs = 0; 460 461 list_for_each_entry(nvbo, list, entry) { 462 struct drm_nouveau_gem_pushbuf_bo *b = &pbbo[nvbo->pbbo_index]; 463 464 ret = nouveau_gem_set_domain(&nvbo->gem, b->read_domains, 465 b->write_domains, 466 b->valid_domains); 467 if (unlikely(ret)) { 468 NV_ERROR(cli, "fail set_domain\n"); 469 return ret; 470 } 471 472 ret = nouveau_bo_validate(nvbo, true, false); 473 if (unlikely(ret)) { 474 if (ret != -ERESTARTSYS) 475 NV_ERROR(cli, "fail ttm_validate\n"); 476 return ret; 477 } 478 479 ret = validate_sync(chan, nvbo); 480 if (unlikely(ret)) { 481 NV_ERROR(cli, "fail post-validate sync\n"); 482 return ret; 483 } 484 485 if (nv_device(drm->device)->card_type < NV_50) { 486 if (nvbo->bo.offset == b->presumed.offset && 487 ((nvbo->bo.mem.mem_type == TTM_PL_VRAM && 488 b->presumed.domain & NOUVEAU_GEM_DOMAIN_VRAM) || 489 (nvbo->bo.mem.mem_type == TTM_PL_TT && 490 b->presumed.domain & NOUVEAU_GEM_DOMAIN_GART))) 491 continue; 492 493 if (nvbo->bo.mem.mem_type == TTM_PL_TT) 494 b->presumed.domain = NOUVEAU_GEM_DOMAIN_GART; 495 else 496 b->presumed.domain = NOUVEAU_GEM_DOMAIN_VRAM; 497 b->presumed.offset = nvbo->bo.offset; 498 b->presumed.valid = 0; 499 relocs++; 500 501 if (copy_to_user(&upbbo[nvbo->pbbo_index].presumed, 502 &b->presumed, sizeof(b->presumed))) 503 return -EFAULT; 504 } 505 } 506 507 return relocs; 508} 509 510static int 511nouveau_gem_pushbuf_validate(struct nouveau_channel *chan, 512 struct drm_file *file_priv, 513 struct drm_nouveau_gem_pushbuf_bo *pbbo, 514 uint64_t user_buffers, int nr_buffers, 515 struct validate_op *op, int *apply_relocs) 516{ 517 struct nouveau_cli *cli = nouveau_cli(file_priv); 518 int ret, relocs = 0; 519 520 INIT_LIST_HEAD(&op->vram_list); 521 INIT_LIST_HEAD(&op->gart_list); 522 INIT_LIST_HEAD(&op->both_list); 523 524 if (nr_buffers == 0) 525 return 0; 526 527 ret = validate_init(chan, file_priv, pbbo, nr_buffers, op); 528 if (unlikely(ret)) { 529 if (ret != -ERESTARTSYS) 530 NV_ERROR(cli, "validate_init\n"); 531 return ret; 532 } 533 534 ret = validate_list(chan, cli, &op->vram_list, pbbo, user_buffers); 535 if (unlikely(ret < 0)) { 536 if (ret != -ERESTARTSYS) 537 NV_ERROR(cli, "validate vram_list\n"); 538 validate_fini(op, NULL); 539 return ret; 540 } 541 relocs += ret; 542 543 ret = validate_list(chan, cli, &op->gart_list, pbbo, user_buffers); 544 if (unlikely(ret < 0)) { 545 if (ret != -ERESTARTSYS) 546 NV_ERROR(cli, "validate gart_list\n"); 547 validate_fini(op, NULL); 548 return ret; 549 } 550 relocs += ret; 551 552 ret = validate_list(chan, cli, &op->both_list, pbbo, user_buffers); 553 if (unlikely(ret < 0)) { 554 if (ret != -ERESTARTSYS) 555 NV_ERROR(cli, "validate both_list\n"); 556 validate_fini(op, NULL); 557 return ret; 558 } 559 relocs += ret; 560 561 *apply_relocs = relocs; 562 return 0; 563} 564 565static inline void 566u_free(void *addr) 567{ 568 if (!is_vmalloc_addr(addr)) 569 kfree(addr); 570 else 571 vfree(addr); 572} 573 574static inline void * 575u_memcpya(uint64_t user, unsigned nmemb, unsigned size) 576{ 577 void *mem; 578 void __user *userptr = (void __force __user *)(uintptr_t)user; 579 580 size *= nmemb; 581 582 mem = kmalloc(size, GFP_KERNEL | __GFP_NOWARN); 583 if (!mem) 584 mem = vmalloc(size); 585 if (!mem) 586 return ERR_PTR(-ENOMEM); 587 588 if (copy_from_user(mem, userptr, size)) { 589 u_free(mem); 590 return ERR_PTR(-EFAULT); 591 } 592 593 return mem; 594} 595 596static int 597nouveau_gem_pushbuf_reloc_apply(struct nouveau_cli *cli, 598 struct drm_nouveau_gem_pushbuf *req, 599 struct drm_nouveau_gem_pushbuf_bo *bo) 600{ 601 struct drm_nouveau_gem_pushbuf_reloc *reloc = NULL; 602 int ret = 0; 603 unsigned i; 604 605 reloc = u_memcpya(req->relocs, req->nr_relocs, sizeof(*reloc)); 606 if (IS_ERR(reloc)) 607 return PTR_ERR(reloc); 608 609 for (i = 0; i < req->nr_relocs; i++) { 610 struct drm_nouveau_gem_pushbuf_reloc *r = &reloc[i]; 611 struct drm_nouveau_gem_pushbuf_bo *b; 612 struct nouveau_bo *nvbo; 613 uint32_t data; 614 615 if (unlikely(r->bo_index > req->nr_buffers)) { 616 NV_ERROR(cli, "reloc bo index invalid\n"); 617 ret = -EINVAL; 618 break; 619 } 620 621 b = &bo[r->bo_index]; 622 if (b->presumed.valid) 623 continue; 624 625 if (unlikely(r->reloc_bo_index > req->nr_buffers)) { 626 NV_ERROR(cli, "reloc container bo index invalid\n"); 627 ret = -EINVAL; 628 break; 629 } 630 nvbo = (void *)(unsigned long)bo[r->reloc_bo_index].user_priv; 631 632 if (unlikely(r->reloc_bo_offset + 4 > 633 nvbo->bo.mem.num_pages << PAGE_SHIFT)) { 634 NV_ERROR(cli, "reloc outside of bo\n"); 635 ret = -EINVAL; 636 break; 637 } 638 639 if (!nvbo->kmap.virtual) { 640 ret = ttm_bo_kmap(&nvbo->bo, 0, nvbo->bo.mem.num_pages, 641 &nvbo->kmap); 642 if (ret) { 643 NV_ERROR(cli, "failed kmap for reloc\n"); 644 break; 645 } 646 nvbo->validate_mapped = true; 647 } 648 649 if (r->flags & NOUVEAU_GEM_RELOC_LOW) 650 data = b->presumed.offset + r->data; 651 else 652 if (r->flags & NOUVEAU_GEM_RELOC_HIGH) 653 data = (b->presumed.offset + r->data) >> 32; 654 else 655 data = r->data; 656 657 if (r->flags & NOUVEAU_GEM_RELOC_OR) { 658 if (b->presumed.domain == NOUVEAU_GEM_DOMAIN_GART) 659 data |= r->tor; 660 else 661 data |= r->vor; 662 } 663 664 spin_lock(&nvbo->bo.bdev->fence_lock); 665 ret = ttm_bo_wait(&nvbo->bo, false, false, false); 666 spin_unlock(&nvbo->bo.bdev->fence_lock); 667 if (ret) { 668 NV_ERROR(cli, "reloc wait_idle failed: %d\n", ret); 669 break; 670 } 671 672 nouveau_bo_wr32(nvbo, r->reloc_bo_offset >> 2, data); 673 } 674 675 u_free(reloc); 676 return ret; 677} 678 679int 680nouveau_gem_ioctl_pushbuf(struct drm_device *dev, void *data, 681 struct drm_file *file_priv) 682{ 683 struct nouveau_abi16 *abi16 = nouveau_abi16_get(file_priv, dev); 684 struct nouveau_cli *cli = nouveau_cli(file_priv); 685 struct nouveau_abi16_chan *temp; 686 struct nouveau_drm *drm = nouveau_drm(dev); 687 struct drm_nouveau_gem_pushbuf *req = data; 688 struct drm_nouveau_gem_pushbuf_push *push; 689 struct drm_nouveau_gem_pushbuf_bo *bo; 690 struct nouveau_channel *chan = NULL; 691 struct validate_op op; 692 struct nouveau_fence *fence = NULL; 693 int i, j, ret = 0, do_reloc = 0; 694 695 if (unlikely(!abi16)) 696 return -ENOMEM; 697 698 list_for_each_entry(temp, &abi16->channels, head) { 699 if (temp->chan->handle == (NVDRM_CHAN | req->channel)) { 700 chan = temp->chan; 701 break; 702 } 703 } 704 705 if (!chan) 706 return nouveau_abi16_put(abi16, -ENOENT); 707 708 req->vram_available = drm->gem.vram_available; 709 req->gart_available = drm->gem.gart_available; 710 if (unlikely(req->nr_push == 0)) 711 goto out_next; 712 713 if (unlikely(req->nr_push > NOUVEAU_GEM_MAX_PUSH)) { 714 NV_ERROR(cli, "pushbuf push count exceeds limit: %d max %d\n", 715 req->nr_push, NOUVEAU_GEM_MAX_PUSH); 716 return nouveau_abi16_put(abi16, -EINVAL); 717 } 718 719 if (unlikely(req->nr_buffers > NOUVEAU_GEM_MAX_BUFFERS)) { 720 NV_ERROR(cli, "pushbuf bo count exceeds limit: %d max %d\n", 721 req->nr_buffers, NOUVEAU_GEM_MAX_BUFFERS); 722 return nouveau_abi16_put(abi16, -EINVAL); 723 } 724 725 if (unlikely(req->nr_relocs > NOUVEAU_GEM_MAX_RELOCS)) { 726 NV_ERROR(cli, "pushbuf reloc count exceeds limit: %d max %d\n", 727 req->nr_relocs, NOUVEAU_GEM_MAX_RELOCS); 728 return nouveau_abi16_put(abi16, -EINVAL); 729 } 730 731 push = u_memcpya(req->push, req->nr_push, sizeof(*push)); 732 if (IS_ERR(push)) 733 return nouveau_abi16_put(abi16, PTR_ERR(push)); 734 735 bo = u_memcpya(req->buffers, req->nr_buffers, sizeof(*bo)); 736 if (IS_ERR(bo)) { 737 u_free(push); 738 return nouveau_abi16_put(abi16, PTR_ERR(bo)); 739 } 740 741 /* Ensure all push buffers are on validate list */ 742 for (i = 0; i < req->nr_push; i++) { 743 if (push[i].bo_index >= req->nr_buffers) { 744 NV_ERROR(cli, "push %d buffer not in list\n", i); 745 ret = -EINVAL; 746 goto out_prevalid; 747 } 748 } 749 750 /* Validate buffer list */ 751 ret = nouveau_gem_pushbuf_validate(chan, file_priv, bo, req->buffers, 752 req->nr_buffers, &op, &do_reloc); 753 if (ret) { 754 if (ret != -ERESTARTSYS) 755 NV_ERROR(cli, "validate: %d\n", ret); 756 goto out_prevalid; 757 } 758 759 /* Apply any relocations that are required */ 760 if (do_reloc) { 761 ret = nouveau_gem_pushbuf_reloc_apply(cli, req, bo); 762 if (ret) { 763 NV_ERROR(cli, "reloc apply: %d\n", ret); 764 goto out; 765 } 766 } 767 768 if (chan->dma.ib_max) { 769 ret = nouveau_dma_wait(chan, req->nr_push + 1, 16); 770 if (ret) { 771 NV_ERROR(cli, "nv50cal_space: %d\n", ret); 772 goto out; 773 } 774 775 for (i = 0; i < req->nr_push; i++) { 776 struct nouveau_bo *nvbo = (void *)(unsigned long) 777 bo[push[i].bo_index].user_priv; 778 779 nv50_dma_push(chan, nvbo, push[i].offset, 780 push[i].length); 781 } 782 } else 783 if (nv_device(drm->device)->chipset >= 0x25) { 784 ret = RING_SPACE(chan, req->nr_push * 2); 785 if (ret) { 786 NV_ERROR(cli, "cal_space: %d\n", ret); 787 goto out; 788 } 789 790 for (i = 0; i < req->nr_push; i++) { 791 struct nouveau_bo *nvbo = (void *)(unsigned long) 792 bo[push[i].bo_index].user_priv; 793 794 OUT_RING(chan, (nvbo->bo.offset + push[i].offset) | 2); 795 OUT_RING(chan, 0); 796 } 797 } else { 798 ret = RING_SPACE(chan, req->nr_push * (2 + NOUVEAU_DMA_SKIPS)); 799 if (ret) { 800 NV_ERROR(cli, "jmp_space: %d\n", ret); 801 goto out; 802 } 803 804 for (i = 0; i < req->nr_push; i++) { 805 struct nouveau_bo *nvbo = (void *)(unsigned long) 806 bo[push[i].bo_index].user_priv; 807 uint32_t cmd; 808 809 cmd = chan->push.vma.offset + ((chan->dma.cur + 2) << 2); 810 cmd |= 0x20000000; 811 if (unlikely(cmd != req->suffix0)) { 812 if (!nvbo->kmap.virtual) { 813 ret = ttm_bo_kmap(&nvbo->bo, 0, 814 nvbo->bo.mem. 815 num_pages, 816 &nvbo->kmap); 817 if (ret) { 818 WIND_RING(chan); 819 goto out; 820 } 821 nvbo->validate_mapped = true; 822 } 823 824 nouveau_bo_wr32(nvbo, (push[i].offset + 825 push[i].length - 8) / 4, cmd); 826 } 827 828 OUT_RING(chan, 0x20000000 | 829 (nvbo->bo.offset + push[i].offset)); 830 OUT_RING(chan, 0); 831 for (j = 0; j < NOUVEAU_DMA_SKIPS; j++) 832 OUT_RING(chan, 0); 833 } 834 } 835 836 ret = nouveau_fence_new(chan, false, &fence); 837 if (ret) { 838 NV_ERROR(cli, "error fencing pushbuf: %d\n", ret); 839 WIND_RING(chan); 840 goto out; 841 } 842 843out: 844 validate_fini(&op, fence); 845 nouveau_fence_unref(&fence); 846 847out_prevalid: 848 u_free(bo); 849 u_free(push); 850 851out_next: 852 if (chan->dma.ib_max) { 853 req->suffix0 = 0x00000000; 854 req->suffix1 = 0x00000000; 855 } else 856 if (nv_device(drm->device)->chipset >= 0x25) { 857 req->suffix0 = 0x00020000; 858 req->suffix1 = 0x00000000; 859 } else { 860 req->suffix0 = 0x20000000 | 861 (chan->push.vma.offset + ((chan->dma.cur + 2) << 2)); 862 req->suffix1 = 0x00000000; 863 } 864 865 return nouveau_abi16_put(abi16, ret); 866} 867 868static inline uint32_t 869domain_to_ttm(struct nouveau_bo *nvbo, uint32_t domain) 870{ 871 uint32_t flags = 0; 872 873 if (domain & NOUVEAU_GEM_DOMAIN_VRAM) 874 flags |= TTM_PL_FLAG_VRAM; 875 if (domain & NOUVEAU_GEM_DOMAIN_GART) 876 flags |= TTM_PL_FLAG_TT; 877 878 return flags; 879} 880 881int 882nouveau_gem_ioctl_cpu_prep(struct drm_device *dev, void *data, 883 struct drm_file *file_priv) 884{ 885 struct drm_nouveau_gem_cpu_prep *req = data; 886 struct drm_gem_object *gem; 887 struct nouveau_bo *nvbo; 888 bool no_wait = !!(req->flags & NOUVEAU_GEM_CPU_PREP_NOWAIT); 889 int ret = -EINVAL; 890 891 gem = drm_gem_object_lookup(dev, file_priv, req->handle); 892 if (!gem) 893 return -ENOENT; 894 nvbo = nouveau_gem_object(gem); 895 896 spin_lock(&nvbo->bo.bdev->fence_lock); 897 ret = ttm_bo_wait(&nvbo->bo, true, true, no_wait); 898 spin_unlock(&nvbo->bo.bdev->fence_lock); 899 drm_gem_object_unreference_unlocked(gem); 900 return ret; 901} 902 903int 904nouveau_gem_ioctl_cpu_fini(struct drm_device *dev, void *data, 905 struct drm_file *file_priv) 906{ 907 return 0; 908} 909 910int 911nouveau_gem_ioctl_info(struct drm_device *dev, void *data, 912 struct drm_file *file_priv) 913{ 914 struct drm_nouveau_gem_info *req = data; 915 struct drm_gem_object *gem; 916 int ret; 917 918 gem = drm_gem_object_lookup(dev, file_priv, req->handle); 919 if (!gem) 920 return -ENOENT; 921 922 ret = nouveau_gem_info(file_priv, gem, req); 923 drm_gem_object_unreference_unlocked(gem); 924 return ret; 925} 926 927