nouveau_gem.c revision 6796cb16c088905bf3af40548fda68c09e6f6ee5
1/* 2 * Copyright (C) 2008 Ben Skeggs. 3 * All Rights Reserved. 4 * 5 * Permission is hereby granted, free of charge, to any person obtaining 6 * a copy of this software and associated documentation files (the 7 * "Software"), to deal in the Software without restriction, including 8 * without limitation the rights to use, copy, modify, merge, publish, 9 * distribute, sublicense, and/or sell copies of the Software, and to 10 * permit persons to whom the Software is furnished to do so, subject to 11 * the following conditions: 12 * 13 * The above copyright notice and this permission notice (including the 14 * next paragraph) shall be included in all copies or substantial 15 * portions of the Software. 16 * 17 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 18 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 19 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. 20 * IN NO EVENT SHALL THE COPYRIGHT OWNER(S) AND/OR ITS SUPPLIERS BE 21 * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION 22 * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION 23 * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 24 * 25 */ 26 27#include <subdev/fb.h> 28 29#include "nouveau_drm.h" 30#include "nouveau_dma.h" 31#include "nouveau_fence.h" 32#include "nouveau_abi16.h" 33 34#include "nouveau_ttm.h" 35#include "nouveau_gem.h" 36 37void 38nouveau_gem_object_del(struct drm_gem_object *gem) 39{ 40 struct nouveau_bo *nvbo = nouveau_gem_object(gem); 41 struct ttm_buffer_object *bo = &nvbo->bo; 42 43 if (gem->import_attach) 44 drm_prime_gem_destroy(gem, nvbo->bo.sg); 45 46 drm_gem_object_release(gem); 47 48 /* reset filp so nouveau_bo_del_ttm() can test for it */ 49 gem->filp = NULL; 50 ttm_bo_unref(&bo); 51} 52 53int 54nouveau_gem_object_open(struct drm_gem_object *gem, struct drm_file *file_priv) 55{ 56 struct nouveau_cli *cli = nouveau_cli(file_priv); 57 struct nouveau_bo *nvbo = nouveau_gem_object(gem); 58 struct nouveau_vma *vma; 59 int ret; 60 61 if (!cli->base.vm) 62 return 0; 63 64 ret = ttm_bo_reserve(&nvbo->bo, false, false, false, 0); 65 if (ret) 66 return ret; 67 68 vma = nouveau_bo_vma_find(nvbo, cli->base.vm); 69 if (!vma) { 70 vma = kzalloc(sizeof(*vma), GFP_KERNEL); 71 if (!vma) { 72 ret = -ENOMEM; 73 goto out; 74 } 75 76 ret = nouveau_bo_vma_add(nvbo, cli->base.vm, vma); 77 if (ret) { 78 kfree(vma); 79 goto out; 80 } 81 } else { 82 vma->refcount++; 83 } 84 85out: 86 ttm_bo_unreserve(&nvbo->bo); 87 return ret; 88} 89 90static void 91nouveau_gem_object_delete(void *data) 92{ 93 struct nouveau_vma *vma = data; 94 nouveau_vm_unmap(vma); 95 nouveau_vm_put(vma); 96 kfree(vma); 97} 98 99static void 100nouveau_gem_object_unmap(struct nouveau_bo *nvbo, struct nouveau_vma *vma) 101{ 102 const bool mapped = nvbo->bo.mem.mem_type != TTM_PL_SYSTEM; 103 struct nouveau_fence *fence = NULL; 104 105 list_del(&vma->head); 106 107 if (mapped) { 108 spin_lock(&nvbo->bo.bdev->fence_lock); 109 fence = nouveau_fence_ref(nvbo->bo.sync_obj); 110 spin_unlock(&nvbo->bo.bdev->fence_lock); 111 } 112 113 if (fence) { 114 nouveau_fence_work(fence, nouveau_gem_object_delete, vma); 115 } else { 116 if (mapped) 117 nouveau_vm_unmap(vma); 118 nouveau_vm_put(vma); 119 kfree(vma); 120 } 121 nouveau_fence_unref(&fence); 122} 123 124void 125nouveau_gem_object_close(struct drm_gem_object *gem, struct drm_file *file_priv) 126{ 127 struct nouveau_cli *cli = nouveau_cli(file_priv); 128 struct nouveau_bo *nvbo = nouveau_gem_object(gem); 129 struct nouveau_vma *vma; 130 int ret; 131 132 if (!cli->base.vm) 133 return; 134 135 ret = ttm_bo_reserve(&nvbo->bo, false, false, false, 0); 136 if (ret) 137 return; 138 139 vma = nouveau_bo_vma_find(nvbo, cli->base.vm); 140 if (vma) { 141 if (--vma->refcount == 0) 142 nouveau_gem_object_unmap(nvbo, vma); 143 } 144 ttm_bo_unreserve(&nvbo->bo); 145} 146 147int 148nouveau_gem_new(struct drm_device *dev, int size, int align, uint32_t domain, 149 uint32_t tile_mode, uint32_t tile_flags, 150 struct nouveau_bo **pnvbo) 151{ 152 struct nouveau_drm *drm = nouveau_drm(dev); 153 struct nouveau_bo *nvbo; 154 u32 flags = 0; 155 int ret; 156 157 if (domain & NOUVEAU_GEM_DOMAIN_VRAM) 158 flags |= TTM_PL_FLAG_VRAM; 159 if (domain & NOUVEAU_GEM_DOMAIN_GART) 160 flags |= TTM_PL_FLAG_TT; 161 if (!flags || domain & NOUVEAU_GEM_DOMAIN_CPU) 162 flags |= TTM_PL_FLAG_SYSTEM; 163 164 ret = nouveau_bo_new(dev, size, align, flags, tile_mode, 165 tile_flags, NULL, pnvbo); 166 if (ret) 167 return ret; 168 nvbo = *pnvbo; 169 170 /* we restrict allowed domains on nv50+ to only the types 171 * that were requested at creation time. not possibly on 172 * earlier chips without busting the ABI. 173 */ 174 nvbo->valid_domains = NOUVEAU_GEM_DOMAIN_VRAM | 175 NOUVEAU_GEM_DOMAIN_GART; 176 if (nv_device(drm->device)->card_type >= NV_50) 177 nvbo->valid_domains &= domain; 178 179 /* Initialize the embedded gem-object. We return a single gem-reference 180 * to the caller, instead of a normal nouveau_bo ttm reference. */ 181 ret = drm_gem_object_init(dev, &nvbo->gem, nvbo->bo.mem.size); 182 if (ret) { 183 nouveau_bo_ref(NULL, pnvbo); 184 return -ENOMEM; 185 } 186 187 nvbo->bo.persistent_swap_storage = nvbo->gem.filp; 188 return 0; 189} 190 191static int 192nouveau_gem_info(struct drm_file *file_priv, struct drm_gem_object *gem, 193 struct drm_nouveau_gem_info *rep) 194{ 195 struct nouveau_cli *cli = nouveau_cli(file_priv); 196 struct nouveau_bo *nvbo = nouveau_gem_object(gem); 197 struct nouveau_vma *vma; 198 199 if (nvbo->bo.mem.mem_type == TTM_PL_TT) 200 rep->domain = NOUVEAU_GEM_DOMAIN_GART; 201 else 202 rep->domain = NOUVEAU_GEM_DOMAIN_VRAM; 203 204 rep->offset = nvbo->bo.offset; 205 if (cli->base.vm) { 206 vma = nouveau_bo_vma_find(nvbo, cli->base.vm); 207 if (!vma) 208 return -EINVAL; 209 210 rep->offset = vma->offset; 211 } 212 213 rep->size = nvbo->bo.mem.num_pages << PAGE_SHIFT; 214 rep->map_handle = drm_vma_node_offset_addr(&nvbo->bo.vma_node); 215 rep->tile_mode = nvbo->tile_mode; 216 rep->tile_flags = nvbo->tile_flags; 217 return 0; 218} 219 220int 221nouveau_gem_ioctl_new(struct drm_device *dev, void *data, 222 struct drm_file *file_priv) 223{ 224 struct nouveau_drm *drm = nouveau_drm(dev); 225 struct nouveau_cli *cli = nouveau_cli(file_priv); 226 struct nouveau_fb *pfb = nouveau_fb(drm->device); 227 struct drm_nouveau_gem_new *req = data; 228 struct nouveau_bo *nvbo = NULL; 229 int ret = 0; 230 231 drm->ttm.bdev.dev_mapping = drm->dev->anon_inode->i_mapping; 232 233 if (!pfb->memtype_valid(pfb, req->info.tile_flags)) { 234 NV_ERROR(cli, "bad page flags: 0x%08x\n", req->info.tile_flags); 235 return -EINVAL; 236 } 237 238 ret = nouveau_gem_new(dev, req->info.size, req->align, 239 req->info.domain, req->info.tile_mode, 240 req->info.tile_flags, &nvbo); 241 if (ret) 242 return ret; 243 244 ret = drm_gem_handle_create(file_priv, &nvbo->gem, &req->info.handle); 245 if (ret == 0) { 246 ret = nouveau_gem_info(file_priv, &nvbo->gem, &req->info); 247 if (ret) 248 drm_gem_handle_delete(file_priv, req->info.handle); 249 } 250 251 /* drop reference from allocate - handle holds it now */ 252 drm_gem_object_unreference_unlocked(&nvbo->gem); 253 return ret; 254} 255 256static int 257nouveau_gem_set_domain(struct drm_gem_object *gem, uint32_t read_domains, 258 uint32_t write_domains, uint32_t valid_domains) 259{ 260 struct nouveau_bo *nvbo = nouveau_gem_object(gem); 261 struct ttm_buffer_object *bo = &nvbo->bo; 262 uint32_t domains = valid_domains & nvbo->valid_domains & 263 (write_domains ? write_domains : read_domains); 264 uint32_t pref_flags = 0, valid_flags = 0; 265 266 if (!domains) 267 return -EINVAL; 268 269 if (valid_domains & NOUVEAU_GEM_DOMAIN_VRAM) 270 valid_flags |= TTM_PL_FLAG_VRAM; 271 272 if (valid_domains & NOUVEAU_GEM_DOMAIN_GART) 273 valid_flags |= TTM_PL_FLAG_TT; 274 275 if ((domains & NOUVEAU_GEM_DOMAIN_VRAM) && 276 bo->mem.mem_type == TTM_PL_VRAM) 277 pref_flags |= TTM_PL_FLAG_VRAM; 278 279 else if ((domains & NOUVEAU_GEM_DOMAIN_GART) && 280 bo->mem.mem_type == TTM_PL_TT) 281 pref_flags |= TTM_PL_FLAG_TT; 282 283 else if (domains & NOUVEAU_GEM_DOMAIN_VRAM) 284 pref_flags |= TTM_PL_FLAG_VRAM; 285 286 else 287 pref_flags |= TTM_PL_FLAG_TT; 288 289 nouveau_bo_placement_set(nvbo, pref_flags, valid_flags); 290 291 return 0; 292} 293 294struct validate_op { 295 struct list_head vram_list; 296 struct list_head gart_list; 297 struct list_head both_list; 298 struct ww_acquire_ctx ticket; 299}; 300 301static void 302validate_fini_list(struct list_head *list, struct nouveau_fence *fence, 303 struct ww_acquire_ctx *ticket) 304{ 305 struct list_head *entry, *tmp; 306 struct nouveau_bo *nvbo; 307 308 list_for_each_safe(entry, tmp, list) { 309 nvbo = list_entry(entry, struct nouveau_bo, entry); 310 311 if (likely(fence)) 312 nouveau_bo_fence(nvbo, fence); 313 314 if (unlikely(nvbo->validate_mapped)) { 315 ttm_bo_kunmap(&nvbo->kmap); 316 nvbo->validate_mapped = false; 317 } 318 319 list_del(&nvbo->entry); 320 nvbo->reserved_by = NULL; 321 ttm_bo_unreserve_ticket(&nvbo->bo, ticket); 322 drm_gem_object_unreference_unlocked(&nvbo->gem); 323 } 324} 325 326static void 327validate_fini_no_ticket(struct validate_op *op, struct nouveau_fence *fence) 328{ 329 validate_fini_list(&op->vram_list, fence, &op->ticket); 330 validate_fini_list(&op->gart_list, fence, &op->ticket); 331 validate_fini_list(&op->both_list, fence, &op->ticket); 332} 333 334static void 335validate_fini(struct validate_op *op, struct nouveau_fence *fence) 336{ 337 validate_fini_no_ticket(op, fence); 338 ww_acquire_fini(&op->ticket); 339} 340 341static int 342validate_init(struct nouveau_channel *chan, struct drm_file *file_priv, 343 struct drm_nouveau_gem_pushbuf_bo *pbbo, 344 int nr_buffers, struct validate_op *op) 345{ 346 struct nouveau_cli *cli = nouveau_cli(file_priv); 347 struct drm_device *dev = chan->drm->dev; 348 int trycnt = 0; 349 int ret, i; 350 struct nouveau_bo *res_bo = NULL; 351 352 ww_acquire_init(&op->ticket, &reservation_ww_class); 353retry: 354 if (++trycnt > 100000) { 355 NV_ERROR(cli, "%s failed and gave up.\n", __func__); 356 return -EINVAL; 357 } 358 359 for (i = 0; i < nr_buffers; i++) { 360 struct drm_nouveau_gem_pushbuf_bo *b = &pbbo[i]; 361 struct drm_gem_object *gem; 362 struct nouveau_bo *nvbo; 363 364 gem = drm_gem_object_lookup(dev, file_priv, b->handle); 365 if (!gem) { 366 NV_ERROR(cli, "Unknown handle 0x%08x\n", b->handle); 367 ww_acquire_done(&op->ticket); 368 validate_fini(op, NULL); 369 return -ENOENT; 370 } 371 nvbo = nouveau_gem_object(gem); 372 if (nvbo == res_bo) { 373 res_bo = NULL; 374 drm_gem_object_unreference_unlocked(gem); 375 continue; 376 } 377 378 if (nvbo->reserved_by && nvbo->reserved_by == file_priv) { 379 NV_ERROR(cli, "multiple instances of buffer %d on " 380 "validation list\n", b->handle); 381 drm_gem_object_unreference_unlocked(gem); 382 ww_acquire_done(&op->ticket); 383 validate_fini(op, NULL); 384 return -EINVAL; 385 } 386 387 ret = ttm_bo_reserve(&nvbo->bo, true, false, true, &op->ticket); 388 if (ret) { 389 validate_fini_no_ticket(op, NULL); 390 if (unlikely(ret == -EDEADLK)) { 391 ret = ttm_bo_reserve_slowpath(&nvbo->bo, true, 392 &op->ticket); 393 if (!ret) 394 res_bo = nvbo; 395 } 396 if (unlikely(ret)) { 397 ww_acquire_done(&op->ticket); 398 ww_acquire_fini(&op->ticket); 399 drm_gem_object_unreference_unlocked(gem); 400 if (ret != -ERESTARTSYS) 401 NV_ERROR(cli, "fail reserve\n"); 402 return ret; 403 } 404 } 405 406 b->user_priv = (uint64_t)(unsigned long)nvbo; 407 nvbo->reserved_by = file_priv; 408 nvbo->pbbo_index = i; 409 if ((b->valid_domains & NOUVEAU_GEM_DOMAIN_VRAM) && 410 (b->valid_domains & NOUVEAU_GEM_DOMAIN_GART)) 411 list_add_tail(&nvbo->entry, &op->both_list); 412 else 413 if (b->valid_domains & NOUVEAU_GEM_DOMAIN_VRAM) 414 list_add_tail(&nvbo->entry, &op->vram_list); 415 else 416 if (b->valid_domains & NOUVEAU_GEM_DOMAIN_GART) 417 list_add_tail(&nvbo->entry, &op->gart_list); 418 else { 419 NV_ERROR(cli, "invalid valid domains: 0x%08x\n", 420 b->valid_domains); 421 list_add_tail(&nvbo->entry, &op->both_list); 422 ww_acquire_done(&op->ticket); 423 validate_fini(op, NULL); 424 return -EINVAL; 425 } 426 if (nvbo == res_bo) 427 goto retry; 428 } 429 430 ww_acquire_done(&op->ticket); 431 return 0; 432} 433 434static int 435validate_sync(struct nouveau_channel *chan, struct nouveau_bo *nvbo) 436{ 437 struct nouveau_fence *fence = NULL; 438 int ret = 0; 439 440 spin_lock(&nvbo->bo.bdev->fence_lock); 441 fence = nouveau_fence_ref(nvbo->bo.sync_obj); 442 spin_unlock(&nvbo->bo.bdev->fence_lock); 443 444 if (fence) { 445 ret = nouveau_fence_sync(fence, chan); 446 nouveau_fence_unref(&fence); 447 } 448 449 return ret; 450} 451 452static int 453validate_list(struct nouveau_channel *chan, struct nouveau_cli *cli, 454 struct list_head *list, struct drm_nouveau_gem_pushbuf_bo *pbbo, 455 uint64_t user_pbbo_ptr) 456{ 457 struct nouveau_drm *drm = chan->drm; 458 struct drm_nouveau_gem_pushbuf_bo __user *upbbo = 459 (void __force __user *)(uintptr_t)user_pbbo_ptr; 460 struct nouveau_bo *nvbo; 461 int ret, relocs = 0; 462 463 list_for_each_entry(nvbo, list, entry) { 464 struct drm_nouveau_gem_pushbuf_bo *b = &pbbo[nvbo->pbbo_index]; 465 466 ret = nouveau_gem_set_domain(&nvbo->gem, b->read_domains, 467 b->write_domains, 468 b->valid_domains); 469 if (unlikely(ret)) { 470 NV_ERROR(cli, "fail set_domain\n"); 471 return ret; 472 } 473 474 ret = nouveau_bo_validate(nvbo, true, false); 475 if (unlikely(ret)) { 476 if (ret != -ERESTARTSYS) 477 NV_ERROR(cli, "fail ttm_validate\n"); 478 return ret; 479 } 480 481 ret = validate_sync(chan, nvbo); 482 if (unlikely(ret)) { 483 NV_ERROR(cli, "fail post-validate sync\n"); 484 return ret; 485 } 486 487 if (nv_device(drm->device)->card_type < NV_50) { 488 if (nvbo->bo.offset == b->presumed.offset && 489 ((nvbo->bo.mem.mem_type == TTM_PL_VRAM && 490 b->presumed.domain & NOUVEAU_GEM_DOMAIN_VRAM) || 491 (nvbo->bo.mem.mem_type == TTM_PL_TT && 492 b->presumed.domain & NOUVEAU_GEM_DOMAIN_GART))) 493 continue; 494 495 if (nvbo->bo.mem.mem_type == TTM_PL_TT) 496 b->presumed.domain = NOUVEAU_GEM_DOMAIN_GART; 497 else 498 b->presumed.domain = NOUVEAU_GEM_DOMAIN_VRAM; 499 b->presumed.offset = nvbo->bo.offset; 500 b->presumed.valid = 0; 501 relocs++; 502 503 if (copy_to_user(&upbbo[nvbo->pbbo_index].presumed, 504 &b->presumed, sizeof(b->presumed))) 505 return -EFAULT; 506 } 507 } 508 509 return relocs; 510} 511 512static int 513nouveau_gem_pushbuf_validate(struct nouveau_channel *chan, 514 struct drm_file *file_priv, 515 struct drm_nouveau_gem_pushbuf_bo *pbbo, 516 uint64_t user_buffers, int nr_buffers, 517 struct validate_op *op, int *apply_relocs) 518{ 519 struct nouveau_cli *cli = nouveau_cli(file_priv); 520 int ret, relocs = 0; 521 522 INIT_LIST_HEAD(&op->vram_list); 523 INIT_LIST_HEAD(&op->gart_list); 524 INIT_LIST_HEAD(&op->both_list); 525 526 if (nr_buffers == 0) 527 return 0; 528 529 ret = validate_init(chan, file_priv, pbbo, nr_buffers, op); 530 if (unlikely(ret)) { 531 if (ret != -ERESTARTSYS) 532 NV_ERROR(cli, "validate_init\n"); 533 return ret; 534 } 535 536 ret = validate_list(chan, cli, &op->vram_list, pbbo, user_buffers); 537 if (unlikely(ret < 0)) { 538 if (ret != -ERESTARTSYS) 539 NV_ERROR(cli, "validate vram_list\n"); 540 validate_fini(op, NULL); 541 return ret; 542 } 543 relocs += ret; 544 545 ret = validate_list(chan, cli, &op->gart_list, pbbo, user_buffers); 546 if (unlikely(ret < 0)) { 547 if (ret != -ERESTARTSYS) 548 NV_ERROR(cli, "validate gart_list\n"); 549 validate_fini(op, NULL); 550 return ret; 551 } 552 relocs += ret; 553 554 ret = validate_list(chan, cli, &op->both_list, pbbo, user_buffers); 555 if (unlikely(ret < 0)) { 556 if (ret != -ERESTARTSYS) 557 NV_ERROR(cli, "validate both_list\n"); 558 validate_fini(op, NULL); 559 return ret; 560 } 561 relocs += ret; 562 563 *apply_relocs = relocs; 564 return 0; 565} 566 567static inline void 568u_free(void *addr) 569{ 570 if (!is_vmalloc_addr(addr)) 571 kfree(addr); 572 else 573 vfree(addr); 574} 575 576static inline void * 577u_memcpya(uint64_t user, unsigned nmemb, unsigned size) 578{ 579 void *mem; 580 void __user *userptr = (void __force __user *)(uintptr_t)user; 581 582 size *= nmemb; 583 584 mem = kmalloc(size, GFP_KERNEL | __GFP_NOWARN); 585 if (!mem) 586 mem = vmalloc(size); 587 if (!mem) 588 return ERR_PTR(-ENOMEM); 589 590 if (copy_from_user(mem, userptr, size)) { 591 u_free(mem); 592 return ERR_PTR(-EFAULT); 593 } 594 595 return mem; 596} 597 598static int 599nouveau_gem_pushbuf_reloc_apply(struct nouveau_cli *cli, 600 struct drm_nouveau_gem_pushbuf *req, 601 struct drm_nouveau_gem_pushbuf_bo *bo) 602{ 603 struct drm_nouveau_gem_pushbuf_reloc *reloc = NULL; 604 int ret = 0; 605 unsigned i; 606 607 reloc = u_memcpya(req->relocs, req->nr_relocs, sizeof(*reloc)); 608 if (IS_ERR(reloc)) 609 return PTR_ERR(reloc); 610 611 for (i = 0; i < req->nr_relocs; i++) { 612 struct drm_nouveau_gem_pushbuf_reloc *r = &reloc[i]; 613 struct drm_nouveau_gem_pushbuf_bo *b; 614 struct nouveau_bo *nvbo; 615 uint32_t data; 616 617 if (unlikely(r->bo_index > req->nr_buffers)) { 618 NV_ERROR(cli, "reloc bo index invalid\n"); 619 ret = -EINVAL; 620 break; 621 } 622 623 b = &bo[r->bo_index]; 624 if (b->presumed.valid) 625 continue; 626 627 if (unlikely(r->reloc_bo_index > req->nr_buffers)) { 628 NV_ERROR(cli, "reloc container bo index invalid\n"); 629 ret = -EINVAL; 630 break; 631 } 632 nvbo = (void *)(unsigned long)bo[r->reloc_bo_index].user_priv; 633 634 if (unlikely(r->reloc_bo_offset + 4 > 635 nvbo->bo.mem.num_pages << PAGE_SHIFT)) { 636 NV_ERROR(cli, "reloc outside of bo\n"); 637 ret = -EINVAL; 638 break; 639 } 640 641 if (!nvbo->kmap.virtual) { 642 ret = ttm_bo_kmap(&nvbo->bo, 0, nvbo->bo.mem.num_pages, 643 &nvbo->kmap); 644 if (ret) { 645 NV_ERROR(cli, "failed kmap for reloc\n"); 646 break; 647 } 648 nvbo->validate_mapped = true; 649 } 650 651 if (r->flags & NOUVEAU_GEM_RELOC_LOW) 652 data = b->presumed.offset + r->data; 653 else 654 if (r->flags & NOUVEAU_GEM_RELOC_HIGH) 655 data = (b->presumed.offset + r->data) >> 32; 656 else 657 data = r->data; 658 659 if (r->flags & NOUVEAU_GEM_RELOC_OR) { 660 if (b->presumed.domain == NOUVEAU_GEM_DOMAIN_GART) 661 data |= r->tor; 662 else 663 data |= r->vor; 664 } 665 666 spin_lock(&nvbo->bo.bdev->fence_lock); 667 ret = ttm_bo_wait(&nvbo->bo, false, false, false); 668 spin_unlock(&nvbo->bo.bdev->fence_lock); 669 if (ret) { 670 NV_ERROR(cli, "reloc wait_idle failed: %d\n", ret); 671 break; 672 } 673 674 nouveau_bo_wr32(nvbo, r->reloc_bo_offset >> 2, data); 675 } 676 677 u_free(reloc); 678 return ret; 679} 680 681int 682nouveau_gem_ioctl_pushbuf(struct drm_device *dev, void *data, 683 struct drm_file *file_priv) 684{ 685 struct nouveau_abi16 *abi16 = nouveau_abi16_get(file_priv, dev); 686 struct nouveau_cli *cli = nouveau_cli(file_priv); 687 struct nouveau_abi16_chan *temp; 688 struct nouveau_drm *drm = nouveau_drm(dev); 689 struct drm_nouveau_gem_pushbuf *req = data; 690 struct drm_nouveau_gem_pushbuf_push *push; 691 struct drm_nouveau_gem_pushbuf_bo *bo; 692 struct nouveau_channel *chan = NULL; 693 struct validate_op op; 694 struct nouveau_fence *fence = NULL; 695 int i, j, ret = 0, do_reloc = 0; 696 697 if (unlikely(!abi16)) 698 return -ENOMEM; 699 700 list_for_each_entry(temp, &abi16->channels, head) { 701 if (temp->chan->handle == (NVDRM_CHAN | req->channel)) { 702 chan = temp->chan; 703 break; 704 } 705 } 706 707 if (!chan) 708 return nouveau_abi16_put(abi16, -ENOENT); 709 710 req->vram_available = drm->gem.vram_available; 711 req->gart_available = drm->gem.gart_available; 712 if (unlikely(req->nr_push == 0)) 713 goto out_next; 714 715 if (unlikely(req->nr_push > NOUVEAU_GEM_MAX_PUSH)) { 716 NV_ERROR(cli, "pushbuf push count exceeds limit: %d max %d\n", 717 req->nr_push, NOUVEAU_GEM_MAX_PUSH); 718 return nouveau_abi16_put(abi16, -EINVAL); 719 } 720 721 if (unlikely(req->nr_buffers > NOUVEAU_GEM_MAX_BUFFERS)) { 722 NV_ERROR(cli, "pushbuf bo count exceeds limit: %d max %d\n", 723 req->nr_buffers, NOUVEAU_GEM_MAX_BUFFERS); 724 return nouveau_abi16_put(abi16, -EINVAL); 725 } 726 727 if (unlikely(req->nr_relocs > NOUVEAU_GEM_MAX_RELOCS)) { 728 NV_ERROR(cli, "pushbuf reloc count exceeds limit: %d max %d\n", 729 req->nr_relocs, NOUVEAU_GEM_MAX_RELOCS); 730 return nouveau_abi16_put(abi16, -EINVAL); 731 } 732 733 push = u_memcpya(req->push, req->nr_push, sizeof(*push)); 734 if (IS_ERR(push)) 735 return nouveau_abi16_put(abi16, PTR_ERR(push)); 736 737 bo = u_memcpya(req->buffers, req->nr_buffers, sizeof(*bo)); 738 if (IS_ERR(bo)) { 739 u_free(push); 740 return nouveau_abi16_put(abi16, PTR_ERR(bo)); 741 } 742 743 /* Ensure all push buffers are on validate list */ 744 for (i = 0; i < req->nr_push; i++) { 745 if (push[i].bo_index >= req->nr_buffers) { 746 NV_ERROR(cli, "push %d buffer not in list\n", i); 747 ret = -EINVAL; 748 goto out_prevalid; 749 } 750 } 751 752 /* Validate buffer list */ 753 ret = nouveau_gem_pushbuf_validate(chan, file_priv, bo, req->buffers, 754 req->nr_buffers, &op, &do_reloc); 755 if (ret) { 756 if (ret != -ERESTARTSYS) 757 NV_ERROR(cli, "validate: %d\n", ret); 758 goto out_prevalid; 759 } 760 761 /* Apply any relocations that are required */ 762 if (do_reloc) { 763 ret = nouveau_gem_pushbuf_reloc_apply(cli, req, bo); 764 if (ret) { 765 NV_ERROR(cli, "reloc apply: %d\n", ret); 766 goto out; 767 } 768 } 769 770 if (chan->dma.ib_max) { 771 ret = nouveau_dma_wait(chan, req->nr_push + 1, 16); 772 if (ret) { 773 NV_ERROR(cli, "nv50cal_space: %d\n", ret); 774 goto out; 775 } 776 777 for (i = 0; i < req->nr_push; i++) { 778 struct nouveau_bo *nvbo = (void *)(unsigned long) 779 bo[push[i].bo_index].user_priv; 780 781 nv50_dma_push(chan, nvbo, push[i].offset, 782 push[i].length); 783 } 784 } else 785 if (nv_device(drm->device)->chipset >= 0x25) { 786 ret = RING_SPACE(chan, req->nr_push * 2); 787 if (ret) { 788 NV_ERROR(cli, "cal_space: %d\n", ret); 789 goto out; 790 } 791 792 for (i = 0; i < req->nr_push; i++) { 793 struct nouveau_bo *nvbo = (void *)(unsigned long) 794 bo[push[i].bo_index].user_priv; 795 796 OUT_RING(chan, (nvbo->bo.offset + push[i].offset) | 2); 797 OUT_RING(chan, 0); 798 } 799 } else { 800 ret = RING_SPACE(chan, req->nr_push * (2 + NOUVEAU_DMA_SKIPS)); 801 if (ret) { 802 NV_ERROR(cli, "jmp_space: %d\n", ret); 803 goto out; 804 } 805 806 for (i = 0; i < req->nr_push; i++) { 807 struct nouveau_bo *nvbo = (void *)(unsigned long) 808 bo[push[i].bo_index].user_priv; 809 uint32_t cmd; 810 811 cmd = chan->push.vma.offset + ((chan->dma.cur + 2) << 2); 812 cmd |= 0x20000000; 813 if (unlikely(cmd != req->suffix0)) { 814 if (!nvbo->kmap.virtual) { 815 ret = ttm_bo_kmap(&nvbo->bo, 0, 816 nvbo->bo.mem. 817 num_pages, 818 &nvbo->kmap); 819 if (ret) { 820 WIND_RING(chan); 821 goto out; 822 } 823 nvbo->validate_mapped = true; 824 } 825 826 nouveau_bo_wr32(nvbo, (push[i].offset + 827 push[i].length - 8) / 4, cmd); 828 } 829 830 OUT_RING(chan, 0x20000000 | 831 (nvbo->bo.offset + push[i].offset)); 832 OUT_RING(chan, 0); 833 for (j = 0; j < NOUVEAU_DMA_SKIPS; j++) 834 OUT_RING(chan, 0); 835 } 836 } 837 838 ret = nouveau_fence_new(chan, false, &fence); 839 if (ret) { 840 NV_ERROR(cli, "error fencing pushbuf: %d\n", ret); 841 WIND_RING(chan); 842 goto out; 843 } 844 845out: 846 validate_fini(&op, fence); 847 nouveau_fence_unref(&fence); 848 849out_prevalid: 850 u_free(bo); 851 u_free(push); 852 853out_next: 854 if (chan->dma.ib_max) { 855 req->suffix0 = 0x00000000; 856 req->suffix1 = 0x00000000; 857 } else 858 if (nv_device(drm->device)->chipset >= 0x25) { 859 req->suffix0 = 0x00020000; 860 req->suffix1 = 0x00000000; 861 } else { 862 req->suffix0 = 0x20000000 | 863 (chan->push.vma.offset + ((chan->dma.cur + 2) << 2)); 864 req->suffix1 = 0x00000000; 865 } 866 867 return nouveau_abi16_put(abi16, ret); 868} 869 870static inline uint32_t 871domain_to_ttm(struct nouveau_bo *nvbo, uint32_t domain) 872{ 873 uint32_t flags = 0; 874 875 if (domain & NOUVEAU_GEM_DOMAIN_VRAM) 876 flags |= TTM_PL_FLAG_VRAM; 877 if (domain & NOUVEAU_GEM_DOMAIN_GART) 878 flags |= TTM_PL_FLAG_TT; 879 880 return flags; 881} 882 883int 884nouveau_gem_ioctl_cpu_prep(struct drm_device *dev, void *data, 885 struct drm_file *file_priv) 886{ 887 struct drm_nouveau_gem_cpu_prep *req = data; 888 struct drm_gem_object *gem; 889 struct nouveau_bo *nvbo; 890 bool no_wait = !!(req->flags & NOUVEAU_GEM_CPU_PREP_NOWAIT); 891 int ret = -EINVAL; 892 893 gem = drm_gem_object_lookup(dev, file_priv, req->handle); 894 if (!gem) 895 return -ENOENT; 896 nvbo = nouveau_gem_object(gem); 897 898 spin_lock(&nvbo->bo.bdev->fence_lock); 899 ret = ttm_bo_wait(&nvbo->bo, true, true, no_wait); 900 spin_unlock(&nvbo->bo.bdev->fence_lock); 901 drm_gem_object_unreference_unlocked(gem); 902 return ret; 903} 904 905int 906nouveau_gem_ioctl_cpu_fini(struct drm_device *dev, void *data, 907 struct drm_file *file_priv) 908{ 909 return 0; 910} 911 912int 913nouveau_gem_ioctl_info(struct drm_device *dev, void *data, 914 struct drm_file *file_priv) 915{ 916 struct drm_nouveau_gem_info *req = data; 917 struct drm_gem_object *gem; 918 int ret; 919 920 gem = drm_gem_object_lookup(dev, file_priv, req->handle); 921 if (!gem) 922 return -ENOENT; 923 924 ret = nouveau_gem_info(file_priv, gem, req); 925 drm_gem_object_unreference_unlocked(gem); 926 return ret; 927} 928 929