nouveau_gem.c revision 9242829a87e970773628f30522d2278dd91890ec
1/* 2 * Copyright (C) 2008 Ben Skeggs. 3 * All Rights Reserved. 4 * 5 * Permission is hereby granted, free of charge, to any person obtaining 6 * a copy of this software and associated documentation files (the 7 * "Software"), to deal in the Software without restriction, including 8 * without limitation the rights to use, copy, modify, merge, publish, 9 * distribute, sublicense, and/or sell copies of the Software, and to 10 * permit persons to whom the Software is furnished to do so, subject to 11 * the following conditions: 12 * 13 * The above copyright notice and this permission notice (including the 14 * next paragraph) shall be included in all copies or substantial 15 * portions of the Software. 16 * 17 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 18 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 19 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. 20 * IN NO EVENT SHALL THE COPYRIGHT OWNER(S) AND/OR ITS SUPPLIERS BE 21 * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION 22 * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION 23 * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 24 * 25 */ 26 27#include "nouveau_drm.h" 28#include "nouveau_dma.h" 29#include "nouveau_fence.h" 30#include "nouveau_abi16.h" 31 32#include "nouveau_ttm.h" 33#include "nouveau_gem.h" 34 35void 36nouveau_gem_object_del(struct drm_gem_object *gem) 37{ 38 struct nouveau_bo *nvbo = nouveau_gem_object(gem); 39 struct ttm_buffer_object *bo = &nvbo->bo; 40 41 if (gem->import_attach) 42 drm_prime_gem_destroy(gem, nvbo->bo.sg); 43 44 drm_gem_object_release(gem); 45 46 /* reset filp so nouveau_bo_del_ttm() can test for it */ 47 gem->filp = NULL; 48 ttm_bo_unref(&bo); 49} 50 51int 52nouveau_gem_object_open(struct drm_gem_object *gem, struct drm_file *file_priv) 53{ 54 struct nouveau_cli *cli = nouveau_cli(file_priv); 55 struct nouveau_bo *nvbo = nouveau_gem_object(gem); 56 struct nouveau_vma *vma; 57 int ret; 58 59 if (!cli->vm) 60 return 0; 61 62 ret = ttm_bo_reserve(&nvbo->bo, false, false, false, NULL); 63 if (ret) 64 return ret; 65 66 vma = nouveau_bo_vma_find(nvbo, cli->vm); 67 if (!vma) { 68 vma = kzalloc(sizeof(*vma), GFP_KERNEL); 69 if (!vma) { 70 ret = -ENOMEM; 71 goto out; 72 } 73 74 ret = nouveau_bo_vma_add(nvbo, cli->vm, vma); 75 if (ret) { 76 kfree(vma); 77 goto out; 78 } 79 } else { 80 vma->refcount++; 81 } 82 83out: 84 ttm_bo_unreserve(&nvbo->bo); 85 return ret; 86} 87 88static void 89nouveau_gem_object_delete(void *data) 90{ 91 struct nouveau_vma *vma = data; 92 nouveau_vm_unmap(vma); 93 nouveau_vm_put(vma); 94 kfree(vma); 95} 96 97static void 98nouveau_gem_object_unmap(struct nouveau_bo *nvbo, struct nouveau_vma *vma) 99{ 100 const bool mapped = nvbo->bo.mem.mem_type != TTM_PL_SYSTEM; 101 struct fence *fence = NULL; 102 103 list_del(&vma->head); 104 105 if (mapped) 106 fence = reservation_object_get_excl(nvbo->bo.resv); 107 108 if (fence) { 109 nouveau_fence_work(fence, nouveau_gem_object_delete, vma); 110 } else { 111 if (mapped) 112 nouveau_vm_unmap(vma); 113 nouveau_vm_put(vma); 114 kfree(vma); 115 } 116} 117 118void 119nouveau_gem_object_close(struct drm_gem_object *gem, struct drm_file *file_priv) 120{ 121 struct nouveau_cli *cli = nouveau_cli(file_priv); 122 struct nouveau_bo *nvbo = nouveau_gem_object(gem); 123 struct nouveau_vma *vma; 124 int ret; 125 126 if (!cli->vm) 127 return; 128 129 ret = ttm_bo_reserve(&nvbo->bo, false, false, false, NULL); 130 if (ret) 131 return; 132 133 vma = nouveau_bo_vma_find(nvbo, cli->vm); 134 if (vma) { 135 if (--vma->refcount == 0) 136 nouveau_gem_object_unmap(nvbo, vma); 137 } 138 ttm_bo_unreserve(&nvbo->bo); 139} 140 141int 142nouveau_gem_new(struct drm_device *dev, int size, int align, uint32_t domain, 143 uint32_t tile_mode, uint32_t tile_flags, 144 struct nouveau_bo **pnvbo) 145{ 146 struct nouveau_drm *drm = nouveau_drm(dev); 147 struct nouveau_bo *nvbo; 148 u32 flags = 0; 149 int ret; 150 151 if (domain & NOUVEAU_GEM_DOMAIN_VRAM) 152 flags |= TTM_PL_FLAG_VRAM; 153 if (domain & NOUVEAU_GEM_DOMAIN_GART) 154 flags |= TTM_PL_FLAG_TT; 155 if (!flags || domain & NOUVEAU_GEM_DOMAIN_CPU) 156 flags |= TTM_PL_FLAG_SYSTEM; 157 158 ret = nouveau_bo_new(dev, size, align, flags, tile_mode, 159 tile_flags, NULL, pnvbo); 160 if (ret) 161 return ret; 162 nvbo = *pnvbo; 163 164 /* we restrict allowed domains on nv50+ to only the types 165 * that were requested at creation time. not possibly on 166 * earlier chips without busting the ABI. 167 */ 168 nvbo->valid_domains = NOUVEAU_GEM_DOMAIN_VRAM | 169 NOUVEAU_GEM_DOMAIN_GART; 170 if (drm->device.info.family >= NV_DEVICE_INFO_V0_TESLA) 171 nvbo->valid_domains &= domain; 172 173 /* Initialize the embedded gem-object. We return a single gem-reference 174 * to the caller, instead of a normal nouveau_bo ttm reference. */ 175 ret = drm_gem_object_init(dev, &nvbo->gem, nvbo->bo.mem.size); 176 if (ret) { 177 nouveau_bo_ref(NULL, pnvbo); 178 return -ENOMEM; 179 } 180 181 nvbo->bo.persistent_swap_storage = nvbo->gem.filp; 182 return 0; 183} 184 185static int 186nouveau_gem_info(struct drm_file *file_priv, struct drm_gem_object *gem, 187 struct drm_nouveau_gem_info *rep) 188{ 189 struct nouveau_cli *cli = nouveau_cli(file_priv); 190 struct nouveau_bo *nvbo = nouveau_gem_object(gem); 191 struct nouveau_vma *vma; 192 193 if (nvbo->bo.mem.mem_type == TTM_PL_TT) 194 rep->domain = NOUVEAU_GEM_DOMAIN_GART; 195 else 196 rep->domain = NOUVEAU_GEM_DOMAIN_VRAM; 197 198 rep->offset = nvbo->bo.offset; 199 if (cli->vm) { 200 vma = nouveau_bo_vma_find(nvbo, cli->vm); 201 if (!vma) 202 return -EINVAL; 203 204 rep->offset = vma->offset; 205 } 206 207 rep->size = nvbo->bo.mem.num_pages << PAGE_SHIFT; 208 rep->map_handle = drm_vma_node_offset_addr(&nvbo->bo.vma_node); 209 rep->tile_mode = nvbo->tile_mode; 210 rep->tile_flags = nvbo->tile_flags; 211 return 0; 212} 213 214int 215nouveau_gem_ioctl_new(struct drm_device *dev, void *data, 216 struct drm_file *file_priv) 217{ 218 struct nouveau_drm *drm = nouveau_drm(dev); 219 struct nouveau_cli *cli = nouveau_cli(file_priv); 220 struct nouveau_fb *pfb = nvkm_fb(&drm->device); 221 struct drm_nouveau_gem_new *req = data; 222 struct nouveau_bo *nvbo = NULL; 223 int ret = 0; 224 225 if (!pfb->memtype_valid(pfb, req->info.tile_flags)) { 226 NV_PRINTK(error, cli, "bad page flags: 0x%08x\n", req->info.tile_flags); 227 return -EINVAL; 228 } 229 230 ret = nouveau_gem_new(dev, req->info.size, req->align, 231 req->info.domain, req->info.tile_mode, 232 req->info.tile_flags, &nvbo); 233 if (ret) 234 return ret; 235 236 ret = drm_gem_handle_create(file_priv, &nvbo->gem, &req->info.handle); 237 if (ret == 0) { 238 ret = nouveau_gem_info(file_priv, &nvbo->gem, &req->info); 239 if (ret) 240 drm_gem_handle_delete(file_priv, req->info.handle); 241 } 242 243 /* drop reference from allocate - handle holds it now */ 244 drm_gem_object_unreference_unlocked(&nvbo->gem); 245 return ret; 246} 247 248static int 249nouveau_gem_set_domain(struct drm_gem_object *gem, uint32_t read_domains, 250 uint32_t write_domains, uint32_t valid_domains) 251{ 252 struct nouveau_bo *nvbo = nouveau_gem_object(gem); 253 struct ttm_buffer_object *bo = &nvbo->bo; 254 uint32_t domains = valid_domains & nvbo->valid_domains & 255 (write_domains ? write_domains : read_domains); 256 uint32_t pref_flags = 0, valid_flags = 0; 257 258 if (!domains) 259 return -EINVAL; 260 261 if (valid_domains & NOUVEAU_GEM_DOMAIN_VRAM) 262 valid_flags |= TTM_PL_FLAG_VRAM; 263 264 if (valid_domains & NOUVEAU_GEM_DOMAIN_GART) 265 valid_flags |= TTM_PL_FLAG_TT; 266 267 if ((domains & NOUVEAU_GEM_DOMAIN_VRAM) && 268 bo->mem.mem_type == TTM_PL_VRAM) 269 pref_flags |= TTM_PL_FLAG_VRAM; 270 271 else if ((domains & NOUVEAU_GEM_DOMAIN_GART) && 272 bo->mem.mem_type == TTM_PL_TT) 273 pref_flags |= TTM_PL_FLAG_TT; 274 275 else if (domains & NOUVEAU_GEM_DOMAIN_VRAM) 276 pref_flags |= TTM_PL_FLAG_VRAM; 277 278 else 279 pref_flags |= TTM_PL_FLAG_TT; 280 281 nouveau_bo_placement_set(nvbo, pref_flags, valid_flags); 282 283 return 0; 284} 285 286struct validate_op { 287 struct list_head list; 288 struct ww_acquire_ctx ticket; 289}; 290 291static void 292validate_fini_no_ticket(struct validate_op *op, struct nouveau_fence *fence) 293{ 294 struct nouveau_bo *nvbo; 295 296 while (!list_empty(&op->list)) { 297 nvbo = list_entry(op->list.next, struct nouveau_bo, entry); 298 299 if (likely(fence)) 300 nouveau_bo_fence(nvbo, fence); 301 302 if (unlikely(nvbo->validate_mapped)) { 303 ttm_bo_kunmap(&nvbo->kmap); 304 nvbo->validate_mapped = false; 305 } 306 307 list_del(&nvbo->entry); 308 nvbo->reserved_by = NULL; 309 ttm_bo_unreserve_ticket(&nvbo->bo, &op->ticket); 310 drm_gem_object_unreference_unlocked(&nvbo->gem); 311 } 312} 313 314static void 315validate_fini(struct validate_op *op, struct nouveau_fence *fence) 316{ 317 validate_fini_no_ticket(op, fence); 318 ww_acquire_fini(&op->ticket); 319} 320 321static int 322validate_init(struct nouveau_channel *chan, struct drm_file *file_priv, 323 struct drm_nouveau_gem_pushbuf_bo *pbbo, 324 int nr_buffers, struct validate_op *op) 325{ 326 struct nouveau_cli *cli = nouveau_cli(file_priv); 327 struct drm_device *dev = chan->drm->dev; 328 int trycnt = 0; 329 int ret, i; 330 struct nouveau_bo *res_bo = NULL; 331 LIST_HEAD(gart_list); 332 LIST_HEAD(vram_list); 333 LIST_HEAD(both_list); 334 335 ww_acquire_init(&op->ticket, &reservation_ww_class); 336retry: 337 if (++trycnt > 100000) { 338 NV_PRINTK(error, cli, "%s failed and gave up.\n", __func__); 339 return -EINVAL; 340 } 341 342 for (i = 0; i < nr_buffers; i++) { 343 struct drm_nouveau_gem_pushbuf_bo *b = &pbbo[i]; 344 struct drm_gem_object *gem; 345 struct nouveau_bo *nvbo; 346 347 gem = drm_gem_object_lookup(dev, file_priv, b->handle); 348 if (!gem) { 349 NV_PRINTK(error, cli, "Unknown handle 0x%08x\n", b->handle); 350 ret = -ENOENT; 351 break; 352 } 353 nvbo = nouveau_gem_object(gem); 354 if (nvbo == res_bo) { 355 res_bo = NULL; 356 drm_gem_object_unreference_unlocked(gem); 357 continue; 358 } 359 360 if (nvbo->reserved_by && nvbo->reserved_by == file_priv) { 361 NV_PRINTK(error, cli, "multiple instances of buffer %d on " 362 "validation list\n", b->handle); 363 drm_gem_object_unreference_unlocked(gem); 364 ret = -EINVAL; 365 break; 366 } 367 368 ret = ttm_bo_reserve(&nvbo->bo, true, false, true, &op->ticket); 369 if (ret) { 370 list_splice_tail_init(&vram_list, &op->list); 371 list_splice_tail_init(&gart_list, &op->list); 372 list_splice_tail_init(&both_list, &op->list); 373 validate_fini_no_ticket(op, NULL); 374 if (unlikely(ret == -EDEADLK)) { 375 ret = ttm_bo_reserve_slowpath(&nvbo->bo, true, 376 &op->ticket); 377 if (!ret) 378 res_bo = nvbo; 379 } 380 if (unlikely(ret)) { 381 if (ret != -ERESTARTSYS) 382 NV_PRINTK(error, cli, "fail reserve\n"); 383 break; 384 } 385 } 386 387 b->user_priv = (uint64_t)(unsigned long)nvbo; 388 nvbo->reserved_by = file_priv; 389 nvbo->pbbo_index = i; 390 if ((b->valid_domains & NOUVEAU_GEM_DOMAIN_VRAM) && 391 (b->valid_domains & NOUVEAU_GEM_DOMAIN_GART)) 392 list_add_tail(&nvbo->entry, &both_list); 393 else 394 if (b->valid_domains & NOUVEAU_GEM_DOMAIN_VRAM) 395 list_add_tail(&nvbo->entry, &vram_list); 396 else 397 if (b->valid_domains & NOUVEAU_GEM_DOMAIN_GART) 398 list_add_tail(&nvbo->entry, &gart_list); 399 else { 400 NV_PRINTK(error, cli, "invalid valid domains: 0x%08x\n", 401 b->valid_domains); 402 list_add_tail(&nvbo->entry, &both_list); 403 ret = -EINVAL; 404 break; 405 } 406 if (nvbo == res_bo) 407 goto retry; 408 } 409 410 ww_acquire_done(&op->ticket); 411 list_splice_tail(&vram_list, &op->list); 412 list_splice_tail(&gart_list, &op->list); 413 list_splice_tail(&both_list, &op->list); 414 if (ret) 415 validate_fini(op, NULL); 416 return ret; 417 418} 419 420static int 421validate_list(struct nouveau_channel *chan, struct nouveau_cli *cli, 422 struct list_head *list, struct drm_nouveau_gem_pushbuf_bo *pbbo, 423 uint64_t user_pbbo_ptr) 424{ 425 struct nouveau_drm *drm = chan->drm; 426 struct drm_nouveau_gem_pushbuf_bo __user *upbbo = 427 (void __force __user *)(uintptr_t)user_pbbo_ptr; 428 struct nouveau_bo *nvbo; 429 int ret, relocs = 0; 430 431 list_for_each_entry(nvbo, list, entry) { 432 struct drm_nouveau_gem_pushbuf_bo *b = &pbbo[nvbo->pbbo_index]; 433 434 ret = nouveau_gem_set_domain(&nvbo->gem, b->read_domains, 435 b->write_domains, 436 b->valid_domains); 437 if (unlikely(ret)) { 438 NV_PRINTK(error, cli, "fail set_domain\n"); 439 return ret; 440 } 441 442 ret = nouveau_bo_validate(nvbo, true, false); 443 if (unlikely(ret)) { 444 if (ret != -ERESTARTSYS) 445 NV_PRINTK(error, cli, "fail ttm_validate\n"); 446 return ret; 447 } 448 449 ret = nouveau_fence_sync(nvbo, chan); 450 if (unlikely(ret)) { 451 if (ret != -ERESTARTSYS) 452 NV_PRINTK(error, cli, "fail post-validate sync\n"); 453 return ret; 454 } 455 456 if (drm->device.info.family < NV_DEVICE_INFO_V0_TESLA) { 457 if (nvbo->bo.offset == b->presumed.offset && 458 ((nvbo->bo.mem.mem_type == TTM_PL_VRAM && 459 b->presumed.domain & NOUVEAU_GEM_DOMAIN_VRAM) || 460 (nvbo->bo.mem.mem_type == TTM_PL_TT && 461 b->presumed.domain & NOUVEAU_GEM_DOMAIN_GART))) 462 continue; 463 464 if (nvbo->bo.mem.mem_type == TTM_PL_TT) 465 b->presumed.domain = NOUVEAU_GEM_DOMAIN_GART; 466 else 467 b->presumed.domain = NOUVEAU_GEM_DOMAIN_VRAM; 468 b->presumed.offset = nvbo->bo.offset; 469 b->presumed.valid = 0; 470 relocs++; 471 472 if (copy_to_user(&upbbo[nvbo->pbbo_index].presumed, 473 &b->presumed, sizeof(b->presumed))) 474 return -EFAULT; 475 } 476 } 477 478 return relocs; 479} 480 481static int 482nouveau_gem_pushbuf_validate(struct nouveau_channel *chan, 483 struct drm_file *file_priv, 484 struct drm_nouveau_gem_pushbuf_bo *pbbo, 485 uint64_t user_buffers, int nr_buffers, 486 struct validate_op *op, int *apply_relocs) 487{ 488 struct nouveau_cli *cli = nouveau_cli(file_priv); 489 int ret; 490 491 INIT_LIST_HEAD(&op->list); 492 493 if (nr_buffers == 0) 494 return 0; 495 496 ret = validate_init(chan, file_priv, pbbo, nr_buffers, op); 497 if (unlikely(ret)) { 498 if (ret != -ERESTARTSYS) 499 NV_PRINTK(error, cli, "validate_init\n"); 500 return ret; 501 } 502 503 ret = validate_list(chan, cli, &op->list, pbbo, user_buffers); 504 if (unlikely(ret < 0)) { 505 if (ret != -ERESTARTSYS) 506 NV_PRINTK(error, cli, "validating bo list\n"); 507 validate_fini(op, NULL); 508 return ret; 509 } 510 *apply_relocs = ret; 511 return 0; 512} 513 514static inline void 515u_free(void *addr) 516{ 517 if (!is_vmalloc_addr(addr)) 518 kfree(addr); 519 else 520 vfree(addr); 521} 522 523static inline void * 524u_memcpya(uint64_t user, unsigned nmemb, unsigned size) 525{ 526 void *mem; 527 void __user *userptr = (void __force __user *)(uintptr_t)user; 528 529 size *= nmemb; 530 531 mem = kmalloc(size, GFP_KERNEL | __GFP_NOWARN); 532 if (!mem) 533 mem = vmalloc(size); 534 if (!mem) 535 return ERR_PTR(-ENOMEM); 536 537 if (copy_from_user(mem, userptr, size)) { 538 u_free(mem); 539 return ERR_PTR(-EFAULT); 540 } 541 542 return mem; 543} 544 545static int 546nouveau_gem_pushbuf_reloc_apply(struct nouveau_cli *cli, 547 struct drm_nouveau_gem_pushbuf *req, 548 struct drm_nouveau_gem_pushbuf_bo *bo) 549{ 550 struct drm_nouveau_gem_pushbuf_reloc *reloc = NULL; 551 int ret = 0; 552 unsigned i; 553 554 reloc = u_memcpya(req->relocs, req->nr_relocs, sizeof(*reloc)); 555 if (IS_ERR(reloc)) 556 return PTR_ERR(reloc); 557 558 for (i = 0; i < req->nr_relocs; i++) { 559 struct drm_nouveau_gem_pushbuf_reloc *r = &reloc[i]; 560 struct drm_nouveau_gem_pushbuf_bo *b; 561 struct nouveau_bo *nvbo; 562 uint32_t data; 563 564 if (unlikely(r->bo_index > req->nr_buffers)) { 565 NV_PRINTK(error, cli, "reloc bo index invalid\n"); 566 ret = -EINVAL; 567 break; 568 } 569 570 b = &bo[r->bo_index]; 571 if (b->presumed.valid) 572 continue; 573 574 if (unlikely(r->reloc_bo_index > req->nr_buffers)) { 575 NV_PRINTK(error, cli, "reloc container bo index invalid\n"); 576 ret = -EINVAL; 577 break; 578 } 579 nvbo = (void *)(unsigned long)bo[r->reloc_bo_index].user_priv; 580 581 if (unlikely(r->reloc_bo_offset + 4 > 582 nvbo->bo.mem.num_pages << PAGE_SHIFT)) { 583 NV_PRINTK(error, cli, "reloc outside of bo\n"); 584 ret = -EINVAL; 585 break; 586 } 587 588 if (!nvbo->kmap.virtual) { 589 ret = ttm_bo_kmap(&nvbo->bo, 0, nvbo->bo.mem.num_pages, 590 &nvbo->kmap); 591 if (ret) { 592 NV_PRINTK(error, cli, "failed kmap for reloc\n"); 593 break; 594 } 595 nvbo->validate_mapped = true; 596 } 597 598 if (r->flags & NOUVEAU_GEM_RELOC_LOW) 599 data = b->presumed.offset + r->data; 600 else 601 if (r->flags & NOUVEAU_GEM_RELOC_HIGH) 602 data = (b->presumed.offset + r->data) >> 32; 603 else 604 data = r->data; 605 606 if (r->flags & NOUVEAU_GEM_RELOC_OR) { 607 if (b->presumed.domain == NOUVEAU_GEM_DOMAIN_GART) 608 data |= r->tor; 609 else 610 data |= r->vor; 611 } 612 613 ret = ttm_bo_wait(&nvbo->bo, false, false, false); 614 if (ret) { 615 NV_PRINTK(error, cli, "reloc wait_idle failed: %d\n", ret); 616 break; 617 } 618 619 nouveau_bo_wr32(nvbo, r->reloc_bo_offset >> 2, data); 620 } 621 622 u_free(reloc); 623 return ret; 624} 625 626int 627nouveau_gem_ioctl_pushbuf(struct drm_device *dev, void *data, 628 struct drm_file *file_priv) 629{ 630 struct nouveau_abi16 *abi16 = nouveau_abi16_get(file_priv, dev); 631 struct nouveau_cli *cli = nouveau_cli(file_priv); 632 struct nouveau_abi16_chan *temp; 633 struct nouveau_drm *drm = nouveau_drm(dev); 634 struct drm_nouveau_gem_pushbuf *req = data; 635 struct drm_nouveau_gem_pushbuf_push *push; 636 struct drm_nouveau_gem_pushbuf_bo *bo; 637 struct nouveau_channel *chan = NULL; 638 struct validate_op op; 639 struct nouveau_fence *fence = NULL; 640 int i, j, ret = 0, do_reloc = 0; 641 642 if (unlikely(!abi16)) 643 return -ENOMEM; 644 645 list_for_each_entry(temp, &abi16->channels, head) { 646 if (temp->chan->object->handle == (NVDRM_CHAN | req->channel)) { 647 chan = temp->chan; 648 break; 649 } 650 } 651 652 if (!chan) 653 return nouveau_abi16_put(abi16, -ENOENT); 654 655 req->vram_available = drm->gem.vram_available; 656 req->gart_available = drm->gem.gart_available; 657 if (unlikely(req->nr_push == 0)) 658 goto out_next; 659 660 if (unlikely(req->nr_push > NOUVEAU_GEM_MAX_PUSH)) { 661 NV_PRINTK(error, cli, "pushbuf push count exceeds limit: %d max %d\n", 662 req->nr_push, NOUVEAU_GEM_MAX_PUSH); 663 return nouveau_abi16_put(abi16, -EINVAL); 664 } 665 666 if (unlikely(req->nr_buffers > NOUVEAU_GEM_MAX_BUFFERS)) { 667 NV_PRINTK(error, cli, "pushbuf bo count exceeds limit: %d max %d\n", 668 req->nr_buffers, NOUVEAU_GEM_MAX_BUFFERS); 669 return nouveau_abi16_put(abi16, -EINVAL); 670 } 671 672 if (unlikely(req->nr_relocs > NOUVEAU_GEM_MAX_RELOCS)) { 673 NV_PRINTK(error, cli, "pushbuf reloc count exceeds limit: %d max %d\n", 674 req->nr_relocs, NOUVEAU_GEM_MAX_RELOCS); 675 return nouveau_abi16_put(abi16, -EINVAL); 676 } 677 678 push = u_memcpya(req->push, req->nr_push, sizeof(*push)); 679 if (IS_ERR(push)) 680 return nouveau_abi16_put(abi16, PTR_ERR(push)); 681 682 bo = u_memcpya(req->buffers, req->nr_buffers, sizeof(*bo)); 683 if (IS_ERR(bo)) { 684 u_free(push); 685 return nouveau_abi16_put(abi16, PTR_ERR(bo)); 686 } 687 688 /* Ensure all push buffers are on validate list */ 689 for (i = 0; i < req->nr_push; i++) { 690 if (push[i].bo_index >= req->nr_buffers) { 691 NV_PRINTK(error, cli, "push %d buffer not in list\n", i); 692 ret = -EINVAL; 693 goto out_prevalid; 694 } 695 } 696 697 /* Validate buffer list */ 698 ret = nouveau_gem_pushbuf_validate(chan, file_priv, bo, req->buffers, 699 req->nr_buffers, &op, &do_reloc); 700 if (ret) { 701 if (ret != -ERESTARTSYS) 702 NV_PRINTK(error, cli, "validate: %d\n", ret); 703 goto out_prevalid; 704 } 705 706 /* Apply any relocations that are required */ 707 if (do_reloc) { 708 ret = nouveau_gem_pushbuf_reloc_apply(cli, req, bo); 709 if (ret) { 710 NV_PRINTK(error, cli, "reloc apply: %d\n", ret); 711 goto out; 712 } 713 } 714 715 if (chan->dma.ib_max) { 716 ret = nouveau_dma_wait(chan, req->nr_push + 1, 16); 717 if (ret) { 718 NV_PRINTK(error, cli, "nv50cal_space: %d\n", ret); 719 goto out; 720 } 721 722 for (i = 0; i < req->nr_push; i++) { 723 struct nouveau_bo *nvbo = (void *)(unsigned long) 724 bo[push[i].bo_index].user_priv; 725 726 nv50_dma_push(chan, nvbo, push[i].offset, 727 push[i].length); 728 } 729 } else 730 if (drm->device.info.chipset >= 0x25) { 731 ret = RING_SPACE(chan, req->nr_push * 2); 732 if (ret) { 733 NV_PRINTK(error, cli, "cal_space: %d\n", ret); 734 goto out; 735 } 736 737 for (i = 0; i < req->nr_push; i++) { 738 struct nouveau_bo *nvbo = (void *)(unsigned long) 739 bo[push[i].bo_index].user_priv; 740 741 OUT_RING(chan, (nvbo->bo.offset + push[i].offset) | 2); 742 OUT_RING(chan, 0); 743 } 744 } else { 745 ret = RING_SPACE(chan, req->nr_push * (2 + NOUVEAU_DMA_SKIPS)); 746 if (ret) { 747 NV_PRINTK(error, cli, "jmp_space: %d\n", ret); 748 goto out; 749 } 750 751 for (i = 0; i < req->nr_push; i++) { 752 struct nouveau_bo *nvbo = (void *)(unsigned long) 753 bo[push[i].bo_index].user_priv; 754 uint32_t cmd; 755 756 cmd = chan->push.vma.offset + ((chan->dma.cur + 2) << 2); 757 cmd |= 0x20000000; 758 if (unlikely(cmd != req->suffix0)) { 759 if (!nvbo->kmap.virtual) { 760 ret = ttm_bo_kmap(&nvbo->bo, 0, 761 nvbo->bo.mem. 762 num_pages, 763 &nvbo->kmap); 764 if (ret) { 765 WIND_RING(chan); 766 goto out; 767 } 768 nvbo->validate_mapped = true; 769 } 770 771 nouveau_bo_wr32(nvbo, (push[i].offset + 772 push[i].length - 8) / 4, cmd); 773 } 774 775 OUT_RING(chan, 0x20000000 | 776 (nvbo->bo.offset + push[i].offset)); 777 OUT_RING(chan, 0); 778 for (j = 0; j < NOUVEAU_DMA_SKIPS; j++) 779 OUT_RING(chan, 0); 780 } 781 } 782 783 ret = nouveau_fence_new(chan, false, &fence); 784 if (ret) { 785 NV_PRINTK(error, cli, "error fencing pushbuf: %d\n", ret); 786 WIND_RING(chan); 787 goto out; 788 } 789 790out: 791 validate_fini(&op, fence); 792 nouveau_fence_unref(&fence); 793 794out_prevalid: 795 u_free(bo); 796 u_free(push); 797 798out_next: 799 if (chan->dma.ib_max) { 800 req->suffix0 = 0x00000000; 801 req->suffix1 = 0x00000000; 802 } else 803 if (drm->device.info.chipset >= 0x25) { 804 req->suffix0 = 0x00020000; 805 req->suffix1 = 0x00000000; 806 } else { 807 req->suffix0 = 0x20000000 | 808 (chan->push.vma.offset + ((chan->dma.cur + 2) << 2)); 809 req->suffix1 = 0x00000000; 810 } 811 812 return nouveau_abi16_put(abi16, ret); 813} 814 815static inline uint32_t 816domain_to_ttm(struct nouveau_bo *nvbo, uint32_t domain) 817{ 818 uint32_t flags = 0; 819 820 if (domain & NOUVEAU_GEM_DOMAIN_VRAM) 821 flags |= TTM_PL_FLAG_VRAM; 822 if (domain & NOUVEAU_GEM_DOMAIN_GART) 823 flags |= TTM_PL_FLAG_TT; 824 825 return flags; 826} 827 828int 829nouveau_gem_ioctl_cpu_prep(struct drm_device *dev, void *data, 830 struct drm_file *file_priv) 831{ 832 struct drm_nouveau_gem_cpu_prep *req = data; 833 struct drm_gem_object *gem; 834 struct nouveau_bo *nvbo; 835 bool no_wait = !!(req->flags & NOUVEAU_GEM_CPU_PREP_NOWAIT); 836 bool write = !!(req->flags & NOUVEAU_GEM_CPU_PREP_WRITE); 837 int ret; 838 839 gem = drm_gem_object_lookup(dev, file_priv, req->handle); 840 if (!gem) 841 return -ENOENT; 842 nvbo = nouveau_gem_object(gem); 843 844 if (no_wait) 845 ret = reservation_object_test_signaled_rcu(nvbo->bo.resv, write) ? 0 : -EBUSY; 846 else { 847 long lret; 848 849 lret = reservation_object_wait_timeout_rcu(nvbo->bo.resv, write, true, 30 * HZ); 850 if (!lret) 851 ret = -EBUSY; 852 else if (lret > 0) 853 ret = 0; 854 else 855 ret = lret; 856 } 857 drm_gem_object_unreference_unlocked(gem); 858 859 return ret; 860} 861 862int 863nouveau_gem_ioctl_cpu_fini(struct drm_device *dev, void *data, 864 struct drm_file *file_priv) 865{ 866 return 0; 867} 868 869int 870nouveau_gem_ioctl_info(struct drm_device *dev, void *data, 871 struct drm_file *file_priv) 872{ 873 struct drm_nouveau_gem_info *req = data; 874 struct drm_gem_object *gem; 875 int ret; 876 877 gem = drm_gem_object_lookup(dev, file_priv, req->handle); 878 if (!gem) 879 return -ENOENT; 880 881 ret = nouveau_gem_info(file_priv, gem, req); 882 drm_gem_object_unreference_unlocked(gem); 883 return ret; 884} 885 886