1/* 2 * Copyright 2012 Red Hat Inc. 3 * 4 * Permission is hereby granted, free of charge, to any person obtaining a 5 * copy of this software and associated documentation files (the "Software"), 6 * to deal in the Software without restriction, including without limitation 7 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 8 * and/or sell copies of the Software, and to permit persons to whom the 9 * Software is furnished to do so, subject to the following conditions: 10 * 11 * The above copyright notice and this permission notice shall be included in 12 * all copies or substantial portions of the Software. 13 * 14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR 18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 20 * OTHER DEALINGS IN THE SOFTWARE. 21 * 22 * Authors: Ben Skeggs 23 */ 24 25#include <core/client.h> 26#include <core/handle.h> 27#include <core/namedb.h> 28#include <core/gpuobj.h> 29#include <core/engctx.h> 30#include <core/event.h> 31#include <nvif/unpack.h> 32#include <nvif/class.h> 33#include <core/enum.h> 34 35#include <subdev/timer.h> 36#include <subdev/bar.h> 37#include <subdev/fb.h> 38#include <subdev/vm.h> 39 40#include <engine/dmaobj.h> 41 42#include "nve0.h" 43 44#define _(a,b) { (a), ((1ULL << (a)) | (b)) } 45static const struct { 46 u64 subdev; 47 u64 mask; 48} fifo_engine[] = { 49 _(NVDEV_ENGINE_GR , (1ULL << NVDEV_ENGINE_SW) | 50 (1ULL << NVDEV_ENGINE_COPY2)), 51 _(NVDEV_ENGINE_VP , 0), 52 _(NVDEV_ENGINE_PPP , 0), 53 _(NVDEV_ENGINE_BSP , 0), 54 _(NVDEV_ENGINE_COPY0 , 0), 55 _(NVDEV_ENGINE_COPY1 , 0), 56 _(NVDEV_ENGINE_VENC , 0), 57}; 58#undef _ 59#define FIFO_ENGINE_NR ARRAY_SIZE(fifo_engine) 60 61struct nve0_fifo_engn { 62 struct nouveau_gpuobj *runlist[2]; 63 int cur_runlist; 64 wait_queue_head_t wait; 65}; 66 67struct nve0_fifo_priv { 68 struct nouveau_fifo base; 69 70 struct work_struct fault; 71 u64 mask; 72 73 struct nve0_fifo_engn engine[FIFO_ENGINE_NR]; 74 struct { 75 struct nouveau_gpuobj *mem; 76 struct nouveau_vma bar; 77 } user; 78 int spoon_nr; 79}; 80 81struct nve0_fifo_base { 82 struct nouveau_fifo_base base; 83 struct nouveau_gpuobj *pgd; 84 struct nouveau_vm *vm; 85}; 86 87struct nve0_fifo_chan { 88 struct nouveau_fifo_chan base; 89 u32 engine; 90 enum { 91 STOPPED, 92 RUNNING, 93 KILLED 94 } state; 95}; 96 97/******************************************************************************* 98 * FIFO channel objects 99 ******************************************************************************/ 100 101static void 102nve0_fifo_runlist_update(struct nve0_fifo_priv *priv, u32 engine) 103{ 104 struct nouveau_bar *bar = nouveau_bar(priv); 105 struct nve0_fifo_engn *engn = &priv->engine[engine]; 106 struct nouveau_gpuobj *cur; 107 int i, p; 108 109 mutex_lock(&nv_subdev(priv)->mutex); 110 cur = engn->runlist[engn->cur_runlist]; 111 engn->cur_runlist = !engn->cur_runlist; 112 113 for (i = 0, p = 0; i < priv->base.max; i++) { 114 struct nve0_fifo_chan *chan = (void *)priv->base.channel[i]; 115 if (chan && chan->state == RUNNING && chan->engine == engine) { 116 nv_wo32(cur, p + 0, i); 117 nv_wo32(cur, p + 4, 0x00000000); 118 p += 8; 119 } 120 } 121 bar->flush(bar); 122 123 nv_wr32(priv, 0x002270, cur->addr >> 12); 124 nv_wr32(priv, 0x002274, (engine << 20) | (p >> 3)); 125 126 if (wait_event_timeout(engn->wait, !(nv_rd32(priv, 0x002284 + 127 (engine * 0x08)) & 0x00100000), 128 msecs_to_jiffies(2000)) == 0) 129 nv_error(priv, "runlist %d update timeout\n", engine); 130 mutex_unlock(&nv_subdev(priv)->mutex); 131} 132 133static int 134nve0_fifo_context_attach(struct nouveau_object *parent, 135 struct nouveau_object *object) 136{ 137 struct nouveau_bar *bar = nouveau_bar(parent); 138 struct nve0_fifo_base *base = (void *)parent->parent; 139 struct nouveau_engctx *ectx = (void *)object; 140 u32 addr; 141 int ret; 142 143 switch (nv_engidx(object->engine)) { 144 case NVDEV_ENGINE_SW : 145 return 0; 146 case NVDEV_ENGINE_COPY0: 147 case NVDEV_ENGINE_COPY1: 148 case NVDEV_ENGINE_COPY2: 149 nv_engctx(ectx)->addr = nv_gpuobj(base)->addr >> 12; 150 return 0; 151 case NVDEV_ENGINE_GR : addr = 0x0210; break; 152 case NVDEV_ENGINE_BSP : addr = 0x0270; break; 153 case NVDEV_ENGINE_VP : addr = 0x0250; break; 154 case NVDEV_ENGINE_PPP : addr = 0x0260; break; 155 default: 156 return -EINVAL; 157 } 158 159 if (!ectx->vma.node) { 160 ret = nouveau_gpuobj_map_vm(nv_gpuobj(ectx), base->vm, 161 NV_MEM_ACCESS_RW, &ectx->vma); 162 if (ret) 163 return ret; 164 165 nv_engctx(ectx)->addr = nv_gpuobj(base)->addr >> 12; 166 } 167 168 nv_wo32(base, addr + 0x00, lower_32_bits(ectx->vma.offset) | 4); 169 nv_wo32(base, addr + 0x04, upper_32_bits(ectx->vma.offset)); 170 bar->flush(bar); 171 return 0; 172} 173 174static int 175nve0_fifo_context_detach(struct nouveau_object *parent, bool suspend, 176 struct nouveau_object *object) 177{ 178 struct nouveau_bar *bar = nouveau_bar(parent); 179 struct nve0_fifo_priv *priv = (void *)parent->engine; 180 struct nve0_fifo_base *base = (void *)parent->parent; 181 struct nve0_fifo_chan *chan = (void *)parent; 182 u32 addr; 183 184 switch (nv_engidx(object->engine)) { 185 case NVDEV_ENGINE_SW : return 0; 186 case NVDEV_ENGINE_COPY0: 187 case NVDEV_ENGINE_COPY1: 188 case NVDEV_ENGINE_COPY2: addr = 0x0000; break; 189 case NVDEV_ENGINE_GR : addr = 0x0210; break; 190 case NVDEV_ENGINE_BSP : addr = 0x0270; break; 191 case NVDEV_ENGINE_VP : addr = 0x0250; break; 192 case NVDEV_ENGINE_PPP : addr = 0x0260; break; 193 default: 194 return -EINVAL; 195 } 196 197 nv_wr32(priv, 0x002634, chan->base.chid); 198 if (!nv_wait(priv, 0x002634, 0xffffffff, chan->base.chid)) { 199 nv_error(priv, "channel %d [%s] kick timeout\n", 200 chan->base.chid, nouveau_client_name(chan)); 201 if (suspend) 202 return -EBUSY; 203 } 204 205 if (addr) { 206 nv_wo32(base, addr + 0x00, 0x00000000); 207 nv_wo32(base, addr + 0x04, 0x00000000); 208 bar->flush(bar); 209 } 210 211 return 0; 212} 213 214static int 215nve0_fifo_chan_ctor(struct nouveau_object *parent, 216 struct nouveau_object *engine, 217 struct nouveau_oclass *oclass, void *data, u32 size, 218 struct nouveau_object **pobject) 219{ 220 union { 221 struct kepler_channel_gpfifo_a_v0 v0; 222 } *args = data; 223 struct nouveau_bar *bar = nouveau_bar(parent); 224 struct nve0_fifo_priv *priv = (void *)engine; 225 struct nve0_fifo_base *base = (void *)parent; 226 struct nve0_fifo_chan *chan; 227 u64 usermem, ioffset, ilength; 228 int ret, i; 229 230 nv_ioctl(parent, "create channel gpfifo size %d\n", size); 231 if (nvif_unpack(args->v0, 0, 0, false)) { 232 nv_ioctl(parent, "create channel gpfifo vers %d pushbuf %08x " 233 "ioffset %016llx ilength %08x engine %08x\n", 234 args->v0.version, args->v0.pushbuf, args->v0.ioffset, 235 args->v0.ilength, args->v0.engine); 236 } else 237 return ret; 238 239 for (i = 0; i < FIFO_ENGINE_NR; i++) { 240 if (args->v0.engine & (1 << i)) { 241 if (nouveau_engine(parent, fifo_engine[i].subdev)) { 242 args->v0.engine = (1 << i); 243 break; 244 } 245 } 246 } 247 248 if (i == FIFO_ENGINE_NR) { 249 nv_error(priv, "unsupported engines 0x%08x\n", args->v0.engine); 250 return -ENODEV; 251 } 252 253 ret = nouveau_fifo_channel_create(parent, engine, oclass, 1, 254 priv->user.bar.offset, 0x200, 255 args->v0.pushbuf, 256 fifo_engine[i].mask, &chan); 257 *pobject = nv_object(chan); 258 if (ret) 259 return ret; 260 261 args->v0.chid = chan->base.chid; 262 263 nv_parent(chan)->context_attach = nve0_fifo_context_attach; 264 nv_parent(chan)->context_detach = nve0_fifo_context_detach; 265 chan->engine = i; 266 267 usermem = chan->base.chid * 0x200; 268 ioffset = args->v0.ioffset; 269 ilength = order_base_2(args->v0.ilength / 8); 270 271 for (i = 0; i < 0x200; i += 4) 272 nv_wo32(priv->user.mem, usermem + i, 0x00000000); 273 274 nv_wo32(base, 0x08, lower_32_bits(priv->user.mem->addr + usermem)); 275 nv_wo32(base, 0x0c, upper_32_bits(priv->user.mem->addr + usermem)); 276 nv_wo32(base, 0x10, 0x0000face); 277 nv_wo32(base, 0x30, 0xfffff902); 278 nv_wo32(base, 0x48, lower_32_bits(ioffset)); 279 nv_wo32(base, 0x4c, upper_32_bits(ioffset) | (ilength << 16)); 280 nv_wo32(base, 0x84, 0x20400000); 281 nv_wo32(base, 0x94, 0x30000001); 282 nv_wo32(base, 0x9c, 0x00000100); 283 nv_wo32(base, 0xac, 0x0000001f); 284 nv_wo32(base, 0xe8, chan->base.chid); 285 nv_wo32(base, 0xb8, 0xf8000000); 286 nv_wo32(base, 0xf8, 0x10003080); /* 0x002310 */ 287 nv_wo32(base, 0xfc, 0x10000010); /* 0x002350 */ 288 bar->flush(bar); 289 return 0; 290} 291 292static int 293nve0_fifo_chan_init(struct nouveau_object *object) 294{ 295 struct nouveau_gpuobj *base = nv_gpuobj(object->parent); 296 struct nve0_fifo_priv *priv = (void *)object->engine; 297 struct nve0_fifo_chan *chan = (void *)object; 298 u32 chid = chan->base.chid; 299 int ret; 300 301 ret = nouveau_fifo_channel_init(&chan->base); 302 if (ret) 303 return ret; 304 305 nv_mask(priv, 0x800004 + (chid * 8), 0x000f0000, chan->engine << 16); 306 nv_wr32(priv, 0x800000 + (chid * 8), 0x80000000 | base->addr >> 12); 307 308 if (chan->state == STOPPED && (chan->state = RUNNING) == RUNNING) { 309 nv_mask(priv, 0x800004 + (chid * 8), 0x00000400, 0x00000400); 310 nve0_fifo_runlist_update(priv, chan->engine); 311 nv_mask(priv, 0x800004 + (chid * 8), 0x00000400, 0x00000400); 312 } 313 314 return 0; 315} 316 317static int 318nve0_fifo_chan_fini(struct nouveau_object *object, bool suspend) 319{ 320 struct nve0_fifo_priv *priv = (void *)object->engine; 321 struct nve0_fifo_chan *chan = (void *)object; 322 u32 chid = chan->base.chid; 323 324 if (chan->state == RUNNING && (chan->state = STOPPED) == STOPPED) { 325 nv_mask(priv, 0x800004 + (chid * 8), 0x00000800, 0x00000800); 326 nve0_fifo_runlist_update(priv, chan->engine); 327 } 328 329 nv_wr32(priv, 0x800000 + (chid * 8), 0x00000000); 330 return nouveau_fifo_channel_fini(&chan->base, suspend); 331} 332 333static struct nouveau_ofuncs 334nve0_fifo_ofuncs = { 335 .ctor = nve0_fifo_chan_ctor, 336 .dtor = _nouveau_fifo_channel_dtor, 337 .init = nve0_fifo_chan_init, 338 .fini = nve0_fifo_chan_fini, 339 .map = _nouveau_fifo_channel_map, 340 .rd32 = _nouveau_fifo_channel_rd32, 341 .wr32 = _nouveau_fifo_channel_wr32, 342 .ntfy = _nouveau_fifo_channel_ntfy 343}; 344 345static struct nouveau_oclass 346nve0_fifo_sclass[] = { 347 { KEPLER_CHANNEL_GPFIFO_A, &nve0_fifo_ofuncs }, 348 {} 349}; 350 351/******************************************************************************* 352 * FIFO context - instmem heap and vm setup 353 ******************************************************************************/ 354 355static int 356nve0_fifo_context_ctor(struct nouveau_object *parent, 357 struct nouveau_object *engine, 358 struct nouveau_oclass *oclass, void *data, u32 size, 359 struct nouveau_object **pobject) 360{ 361 struct nve0_fifo_base *base; 362 int ret; 363 364 ret = nouveau_fifo_context_create(parent, engine, oclass, NULL, 0x1000, 365 0x1000, NVOBJ_FLAG_ZERO_ALLOC, &base); 366 *pobject = nv_object(base); 367 if (ret) 368 return ret; 369 370 ret = nouveau_gpuobj_new(nv_object(base), NULL, 0x10000, 0x1000, 0, 371 &base->pgd); 372 if (ret) 373 return ret; 374 375 nv_wo32(base, 0x0200, lower_32_bits(base->pgd->addr)); 376 nv_wo32(base, 0x0204, upper_32_bits(base->pgd->addr)); 377 nv_wo32(base, 0x0208, 0xffffffff); 378 nv_wo32(base, 0x020c, 0x000000ff); 379 380 ret = nouveau_vm_ref(nouveau_client(parent)->vm, &base->vm, base->pgd); 381 if (ret) 382 return ret; 383 384 return 0; 385} 386 387static void 388nve0_fifo_context_dtor(struct nouveau_object *object) 389{ 390 struct nve0_fifo_base *base = (void *)object; 391 nouveau_vm_ref(NULL, &base->vm, base->pgd); 392 nouveau_gpuobj_ref(NULL, &base->pgd); 393 nouveau_fifo_context_destroy(&base->base); 394} 395 396static struct nouveau_oclass 397nve0_fifo_cclass = { 398 .handle = NV_ENGCTX(FIFO, 0xe0), 399 .ofuncs = &(struct nouveau_ofuncs) { 400 .ctor = nve0_fifo_context_ctor, 401 .dtor = nve0_fifo_context_dtor, 402 .init = _nouveau_fifo_context_init, 403 .fini = _nouveau_fifo_context_fini, 404 .rd32 = _nouveau_fifo_context_rd32, 405 .wr32 = _nouveau_fifo_context_wr32, 406 }, 407}; 408 409/******************************************************************************* 410 * PFIFO engine 411 ******************************************************************************/ 412 413static inline int 414nve0_fifo_engidx(struct nve0_fifo_priv *priv, u32 engn) 415{ 416 switch (engn) { 417 case NVDEV_ENGINE_GR : 418 case NVDEV_ENGINE_COPY2: engn = 0; break; 419 case NVDEV_ENGINE_BSP : engn = 1; break; 420 case NVDEV_ENGINE_PPP : engn = 2; break; 421 case NVDEV_ENGINE_VP : engn = 3; break; 422 case NVDEV_ENGINE_COPY0: engn = 4; break; 423 case NVDEV_ENGINE_COPY1: engn = 5; break; 424 case NVDEV_ENGINE_VENC : engn = 6; break; 425 default: 426 return -1; 427 } 428 429 return engn; 430} 431 432static inline struct nouveau_engine * 433nve0_fifo_engine(struct nve0_fifo_priv *priv, u32 engn) 434{ 435 if (engn >= ARRAY_SIZE(fifo_engine)) 436 return NULL; 437 return nouveau_engine(priv, fifo_engine[engn].subdev); 438} 439 440static void 441nve0_fifo_recover_work(struct work_struct *work) 442{ 443 struct nve0_fifo_priv *priv = container_of(work, typeof(*priv), fault); 444 struct nouveau_object *engine; 445 unsigned long flags; 446 u32 engn, engm = 0; 447 u64 mask, todo; 448 449 spin_lock_irqsave(&priv->base.lock, flags); 450 mask = priv->mask; 451 priv->mask = 0ULL; 452 spin_unlock_irqrestore(&priv->base.lock, flags); 453 454 for (todo = mask; engn = __ffs64(todo), todo; todo &= ~(1 << engn)) 455 engm |= 1 << nve0_fifo_engidx(priv, engn); 456 nv_mask(priv, 0x002630, engm, engm); 457 458 for (todo = mask; engn = __ffs64(todo), todo; todo &= ~(1 << engn)) { 459 if ((engine = (void *)nouveau_engine(priv, engn))) { 460 nv_ofuncs(engine)->fini(engine, false); 461 WARN_ON(nv_ofuncs(engine)->init(engine)); 462 } 463 nve0_fifo_runlist_update(priv, nve0_fifo_engidx(priv, engn)); 464 } 465 466 nv_wr32(priv, 0x00262c, engm); 467 nv_mask(priv, 0x002630, engm, 0x00000000); 468} 469 470static void 471nve0_fifo_recover(struct nve0_fifo_priv *priv, struct nouveau_engine *engine, 472 struct nve0_fifo_chan *chan) 473{ 474 struct nouveau_object *engobj = nv_object(engine); 475 u32 chid = chan->base.chid; 476 unsigned long flags; 477 478 nv_error(priv, "%s engine fault on channel %d, recovering...\n", 479 nv_subdev(engine)->name, chid); 480 481 nv_mask(priv, 0x800004 + (chid * 0x08), 0x00000800, 0x00000800); 482 chan->state = KILLED; 483 484 spin_lock_irqsave(&priv->base.lock, flags); 485 priv->mask |= 1ULL << nv_engidx(engobj); 486 spin_unlock_irqrestore(&priv->base.lock, flags); 487 schedule_work(&priv->fault); 488} 489 490static int 491nve0_fifo_swmthd(struct nve0_fifo_priv *priv, u32 chid, u32 mthd, u32 data) 492{ 493 struct nve0_fifo_chan *chan = NULL; 494 struct nouveau_handle *bind; 495 unsigned long flags; 496 int ret = -EINVAL; 497 498 spin_lock_irqsave(&priv->base.lock, flags); 499 if (likely(chid >= priv->base.min && chid <= priv->base.max)) 500 chan = (void *)priv->base.channel[chid]; 501 if (unlikely(!chan)) 502 goto out; 503 504 bind = nouveau_namedb_get_class(nv_namedb(chan), 0x906e); 505 if (likely(bind)) { 506 if (!mthd || !nv_call(bind->object, mthd, data)) 507 ret = 0; 508 nouveau_namedb_put(bind); 509 } 510 511out: 512 spin_unlock_irqrestore(&priv->base.lock, flags); 513 return ret; 514} 515 516static const struct nouveau_enum 517nve0_fifo_bind_reason[] = { 518 { 0x01, "BIND_NOT_UNBOUND" }, 519 { 0x02, "SNOOP_WITHOUT_BAR1" }, 520 { 0x03, "UNBIND_WHILE_RUNNING" }, 521 { 0x05, "INVALID_RUNLIST" }, 522 { 0x06, "INVALID_CTX_TGT" }, 523 { 0x0b, "UNBIND_WHILE_PARKED" }, 524 {} 525}; 526 527static void 528nve0_fifo_intr_bind(struct nve0_fifo_priv *priv) 529{ 530 u32 intr = nv_rd32(priv, 0x00252c); 531 u32 code = intr & 0x000000ff; 532 const struct nouveau_enum *en; 533 char enunk[6] = ""; 534 535 en = nouveau_enum_find(nve0_fifo_bind_reason, code); 536 if (!en) 537 snprintf(enunk, sizeof(enunk), "UNK%02x", code); 538 539 nv_error(priv, "BIND_ERROR [ %s ]\n", en ? en->name : enunk); 540} 541 542static const struct nouveau_enum 543nve0_fifo_sched_reason[] = { 544 { 0x0a, "CTXSW_TIMEOUT" }, 545 {} 546}; 547 548static void 549nve0_fifo_intr_sched_ctxsw(struct nve0_fifo_priv *priv) 550{ 551 struct nouveau_engine *engine; 552 struct nve0_fifo_chan *chan; 553 u32 engn; 554 555 for (engn = 0; engn < ARRAY_SIZE(fifo_engine); engn++) { 556 u32 stat = nv_rd32(priv, 0x002640 + (engn * 0x04)); 557 u32 busy = (stat & 0x80000000); 558 u32 next = (stat & 0x07ff0000) >> 16; 559 u32 chsw = (stat & 0x00008000); 560 u32 save = (stat & 0x00004000); 561 u32 load = (stat & 0x00002000); 562 u32 prev = (stat & 0x000007ff); 563 u32 chid = load ? next : prev; 564 (void)save; 565 566 if (busy && chsw) { 567 if (!(chan = (void *)priv->base.channel[chid])) 568 continue; 569 if (!(engine = nve0_fifo_engine(priv, engn))) 570 continue; 571 nve0_fifo_recover(priv, engine, chan); 572 } 573 } 574} 575 576static void 577nve0_fifo_intr_sched(struct nve0_fifo_priv *priv) 578{ 579 u32 intr = nv_rd32(priv, 0x00254c); 580 u32 code = intr & 0x000000ff; 581 const struct nouveau_enum *en; 582 char enunk[6] = ""; 583 584 en = nouveau_enum_find(nve0_fifo_sched_reason, code); 585 if (!en) 586 snprintf(enunk, sizeof(enunk), "UNK%02x", code); 587 588 nv_error(priv, "SCHED_ERROR [ %s ]\n", en ? en->name : enunk); 589 590 switch (code) { 591 case 0x0a: 592 nve0_fifo_intr_sched_ctxsw(priv); 593 break; 594 default: 595 break; 596 } 597} 598 599static void 600nve0_fifo_intr_chsw(struct nve0_fifo_priv *priv) 601{ 602 u32 stat = nv_rd32(priv, 0x00256c); 603 nv_error(priv, "CHSW_ERROR 0x%08x\n", stat); 604 nv_wr32(priv, 0x00256c, stat); 605} 606 607static void 608nve0_fifo_intr_dropped_fault(struct nve0_fifo_priv *priv) 609{ 610 u32 stat = nv_rd32(priv, 0x00259c); 611 nv_error(priv, "DROPPED_MMU_FAULT 0x%08x\n", stat); 612} 613 614static const struct nouveau_enum 615nve0_fifo_fault_engine[] = { 616 { 0x00, "GR", NULL, NVDEV_ENGINE_GR }, 617 { 0x03, "IFB", NULL, NVDEV_ENGINE_IFB }, 618 { 0x04, "BAR1", NULL, NVDEV_SUBDEV_BAR }, 619 { 0x05, "BAR3", NULL, NVDEV_SUBDEV_INSTMEM }, 620 { 0x07, "PBDMA0", NULL, NVDEV_ENGINE_FIFO }, 621 { 0x08, "PBDMA1", NULL, NVDEV_ENGINE_FIFO }, 622 { 0x09, "PBDMA2", NULL, NVDEV_ENGINE_FIFO }, 623 { 0x10, "MSVLD", NULL, NVDEV_ENGINE_BSP }, 624 { 0x11, "MSPPP", NULL, NVDEV_ENGINE_PPP }, 625 { 0x13, "PERF" }, 626 { 0x14, "MSPDEC", NULL, NVDEV_ENGINE_VP }, 627 { 0x15, "CE0", NULL, NVDEV_ENGINE_COPY0 }, 628 { 0x16, "CE1", NULL, NVDEV_ENGINE_COPY1 }, 629 { 0x17, "PMU" }, 630 { 0x19, "MSENC", NULL, NVDEV_ENGINE_VENC }, 631 { 0x1b, "CE2", NULL, NVDEV_ENGINE_COPY2 }, 632 {} 633}; 634 635static const struct nouveau_enum 636nve0_fifo_fault_reason[] = { 637 { 0x00, "PDE" }, 638 { 0x01, "PDE_SIZE" }, 639 { 0x02, "PTE" }, 640 { 0x03, "VA_LIMIT_VIOLATION" }, 641 { 0x04, "UNBOUND_INST_BLOCK" }, 642 { 0x05, "PRIV_VIOLATION" }, 643 { 0x06, "RO_VIOLATION" }, 644 { 0x07, "WO_VIOLATION" }, 645 { 0x08, "PITCH_MASK_VIOLATION" }, 646 { 0x09, "WORK_CREATION" }, 647 { 0x0a, "UNSUPPORTED_APERTURE" }, 648 { 0x0b, "COMPRESSION_FAILURE" }, 649 { 0x0c, "UNSUPPORTED_KIND" }, 650 { 0x0d, "REGION_VIOLATION" }, 651 { 0x0e, "BOTH_PTES_VALID" }, 652 { 0x0f, "INFO_TYPE_POISONED" }, 653 {} 654}; 655 656static const struct nouveau_enum 657nve0_fifo_fault_hubclient[] = { 658 { 0x00, "VIP" }, 659 { 0x01, "CE0" }, 660 { 0x02, "CE1" }, 661 { 0x03, "DNISO" }, 662 { 0x04, "FE" }, 663 { 0x05, "FECS" }, 664 { 0x06, "HOST" }, 665 { 0x07, "HOST_CPU" }, 666 { 0x08, "HOST_CPU_NB" }, 667 { 0x09, "ISO" }, 668 { 0x0a, "MMU" }, 669 { 0x0b, "MSPDEC" }, 670 { 0x0c, "MSPPP" }, 671 { 0x0d, "MSVLD" }, 672 { 0x0e, "NISO" }, 673 { 0x0f, "P2P" }, 674 { 0x10, "PD" }, 675 { 0x11, "PERF" }, 676 { 0x12, "PMU" }, 677 { 0x13, "RASTERTWOD" }, 678 { 0x14, "SCC" }, 679 { 0x15, "SCC_NB" }, 680 { 0x16, "SEC" }, 681 { 0x17, "SSYNC" }, 682 { 0x18, "GR_COPY" }, 683 { 0x19, "CE2" }, 684 { 0x1a, "XV" }, 685 { 0x1b, "MMU_NB" }, 686 { 0x1c, "MSENC" }, 687 { 0x1d, "DFALCON" }, 688 { 0x1e, "SKED" }, 689 { 0x1f, "AFALCON" }, 690 {} 691}; 692 693static const struct nouveau_enum 694nve0_fifo_fault_gpcclient[] = { 695 { 0x00, "L1_0" }, { 0x01, "T1_0" }, { 0x02, "PE_0" }, 696 { 0x03, "L1_1" }, { 0x04, "T1_1" }, { 0x05, "PE_1" }, 697 { 0x06, "L1_2" }, { 0x07, "T1_2" }, { 0x08, "PE_2" }, 698 { 0x09, "L1_3" }, { 0x0a, "T1_3" }, { 0x0b, "PE_3" }, 699 { 0x0c, "RAST" }, 700 { 0x0d, "GCC" }, 701 { 0x0e, "GPCCS" }, 702 { 0x0f, "PROP_0" }, 703 { 0x10, "PROP_1" }, 704 { 0x11, "PROP_2" }, 705 { 0x12, "PROP_3" }, 706 { 0x13, "L1_4" }, { 0x14, "T1_4" }, { 0x15, "PE_4" }, 707 { 0x16, "L1_5" }, { 0x17, "T1_5" }, { 0x18, "PE_5" }, 708 { 0x19, "L1_6" }, { 0x1a, "T1_6" }, { 0x1b, "PE_6" }, 709 { 0x1c, "L1_7" }, { 0x1d, "T1_7" }, { 0x1e, "PE_7" }, 710 { 0x1f, "GPM" }, 711 { 0x20, "LTP_UTLB_0" }, 712 { 0x21, "LTP_UTLB_1" }, 713 { 0x22, "LTP_UTLB_2" }, 714 { 0x23, "LTP_UTLB_3" }, 715 { 0x24, "GPC_RGG_UTLB" }, 716 {} 717}; 718 719static void 720nve0_fifo_intr_fault(struct nve0_fifo_priv *priv, int unit) 721{ 722 u32 inst = nv_rd32(priv, 0x002800 + (unit * 0x10)); 723 u32 valo = nv_rd32(priv, 0x002804 + (unit * 0x10)); 724 u32 vahi = nv_rd32(priv, 0x002808 + (unit * 0x10)); 725 u32 stat = nv_rd32(priv, 0x00280c + (unit * 0x10)); 726 u32 gpc = (stat & 0x1f000000) >> 24; 727 u32 client = (stat & 0x00001f00) >> 8; 728 u32 write = (stat & 0x00000080); 729 u32 hub = (stat & 0x00000040); 730 u32 reason = (stat & 0x0000000f); 731 struct nouveau_object *engctx = NULL, *object; 732 struct nouveau_engine *engine = NULL; 733 const struct nouveau_enum *er, *eu, *ec; 734 char erunk[6] = ""; 735 char euunk[6] = ""; 736 char ecunk[6] = ""; 737 char gpcid[3] = ""; 738 739 er = nouveau_enum_find(nve0_fifo_fault_reason, reason); 740 if (!er) 741 snprintf(erunk, sizeof(erunk), "UNK%02X", reason); 742 743 eu = nouveau_enum_find(nve0_fifo_fault_engine, unit); 744 if (eu) { 745 switch (eu->data2) { 746 case NVDEV_SUBDEV_BAR: 747 nv_mask(priv, 0x001704, 0x00000000, 0x00000000); 748 break; 749 case NVDEV_SUBDEV_INSTMEM: 750 nv_mask(priv, 0x001714, 0x00000000, 0x00000000); 751 break; 752 case NVDEV_ENGINE_IFB: 753 nv_mask(priv, 0x001718, 0x00000000, 0x00000000); 754 break; 755 default: 756 engine = nouveau_engine(priv, eu->data2); 757 if (engine) 758 engctx = nouveau_engctx_get(engine, inst); 759 break; 760 } 761 } else { 762 snprintf(euunk, sizeof(euunk), "UNK%02x", unit); 763 } 764 765 if (hub) { 766 ec = nouveau_enum_find(nve0_fifo_fault_hubclient, client); 767 } else { 768 ec = nouveau_enum_find(nve0_fifo_fault_gpcclient, client); 769 snprintf(gpcid, sizeof(gpcid), "%d", gpc); 770 } 771 772 if (!ec) 773 snprintf(ecunk, sizeof(ecunk), "UNK%02x", client); 774 775 nv_error(priv, "%s fault at 0x%010llx [%s] from %s/%s%s%s%s on " 776 "channel 0x%010llx [%s]\n", write ? "write" : "read", 777 (u64)vahi << 32 | valo, er ? er->name : erunk, 778 eu ? eu->name : euunk, hub ? "" : "GPC", gpcid, hub ? "" : "/", 779 ec ? ec->name : ecunk, (u64)inst << 12, 780 nouveau_client_name(engctx)); 781 782 object = engctx; 783 while (object) { 784 switch (nv_mclass(object)) { 785 case KEPLER_CHANNEL_GPFIFO_A: 786 nve0_fifo_recover(priv, engine, (void *)object); 787 break; 788 } 789 object = object->parent; 790 } 791 792 nouveau_engctx_put(engctx); 793} 794 795static const struct nouveau_bitfield nve0_fifo_pbdma_intr[] = { 796 { 0x00000001, "MEMREQ" }, 797 { 0x00000002, "MEMACK_TIMEOUT" }, 798 { 0x00000004, "MEMACK_EXTRA" }, 799 { 0x00000008, "MEMDAT_TIMEOUT" }, 800 { 0x00000010, "MEMDAT_EXTRA" }, 801 { 0x00000020, "MEMFLUSH" }, 802 { 0x00000040, "MEMOP" }, 803 { 0x00000080, "LBCONNECT" }, 804 { 0x00000100, "LBREQ" }, 805 { 0x00000200, "LBACK_TIMEOUT" }, 806 { 0x00000400, "LBACK_EXTRA" }, 807 { 0x00000800, "LBDAT_TIMEOUT" }, 808 { 0x00001000, "LBDAT_EXTRA" }, 809 { 0x00002000, "GPFIFO" }, 810 { 0x00004000, "GPPTR" }, 811 { 0x00008000, "GPENTRY" }, 812 { 0x00010000, "GPCRC" }, 813 { 0x00020000, "PBPTR" }, 814 { 0x00040000, "PBENTRY" }, 815 { 0x00080000, "PBCRC" }, 816 { 0x00100000, "XBARCONNECT" }, 817 { 0x00200000, "METHOD" }, 818 { 0x00400000, "METHODCRC" }, 819 { 0x00800000, "DEVICE" }, 820 { 0x02000000, "SEMAPHORE" }, 821 { 0x04000000, "ACQUIRE" }, 822 { 0x08000000, "PRI" }, 823 { 0x20000000, "NO_CTXSW_SEG" }, 824 { 0x40000000, "PBSEG" }, 825 { 0x80000000, "SIGNATURE" }, 826 {} 827}; 828 829static void 830nve0_fifo_intr_pbdma(struct nve0_fifo_priv *priv, int unit) 831{ 832 u32 stat = nv_rd32(priv, 0x040108 + (unit * 0x2000)); 833 u32 addr = nv_rd32(priv, 0x0400c0 + (unit * 0x2000)); 834 u32 data = nv_rd32(priv, 0x0400c4 + (unit * 0x2000)); 835 u32 chid = nv_rd32(priv, 0x040120 + (unit * 0x2000)) & 0xfff; 836 u32 subc = (addr & 0x00070000) >> 16; 837 u32 mthd = (addr & 0x00003ffc); 838 u32 show = stat; 839 840 if (stat & 0x00800000) { 841 if (!nve0_fifo_swmthd(priv, chid, mthd, data)) 842 show &= ~0x00800000; 843 } 844 845 if (show) { 846 nv_error(priv, "PBDMA%d:", unit); 847 nouveau_bitfield_print(nve0_fifo_pbdma_intr, show); 848 pr_cont("\n"); 849 nv_error(priv, 850 "PBDMA%d: ch %d [%s] subc %d mthd 0x%04x data 0x%08x\n", 851 unit, chid, 852 nouveau_client_name_for_fifo_chid(&priv->base, chid), 853 subc, mthd, data); 854 } 855 856 nv_wr32(priv, 0x0400c0 + (unit * 0x2000), 0x80600008); 857 nv_wr32(priv, 0x040108 + (unit * 0x2000), stat); 858} 859 860static void 861nve0_fifo_intr_runlist(struct nve0_fifo_priv *priv) 862{ 863 u32 mask = nv_rd32(priv, 0x002a00); 864 while (mask) { 865 u32 engn = __ffs(mask); 866 wake_up(&priv->engine[engn].wait); 867 nv_wr32(priv, 0x002a00, 1 << engn); 868 mask &= ~(1 << engn); 869 } 870} 871 872static void 873nve0_fifo_intr_engine(struct nve0_fifo_priv *priv) 874{ 875 nouveau_fifo_uevent(&priv->base); 876} 877 878static void 879nve0_fifo_intr(struct nouveau_subdev *subdev) 880{ 881 struct nve0_fifo_priv *priv = (void *)subdev; 882 u32 mask = nv_rd32(priv, 0x002140); 883 u32 stat = nv_rd32(priv, 0x002100) & mask; 884 885 if (stat & 0x00000001) { 886 nve0_fifo_intr_bind(priv); 887 nv_wr32(priv, 0x002100, 0x00000001); 888 stat &= ~0x00000001; 889 } 890 891 if (stat & 0x00000010) { 892 nv_error(priv, "PIO_ERROR\n"); 893 nv_wr32(priv, 0x002100, 0x00000010); 894 stat &= ~0x00000010; 895 } 896 897 if (stat & 0x00000100) { 898 nve0_fifo_intr_sched(priv); 899 nv_wr32(priv, 0x002100, 0x00000100); 900 stat &= ~0x00000100; 901 } 902 903 if (stat & 0x00010000) { 904 nve0_fifo_intr_chsw(priv); 905 nv_wr32(priv, 0x002100, 0x00010000); 906 stat &= ~0x00010000; 907 } 908 909 if (stat & 0x00800000) { 910 nv_error(priv, "FB_FLUSH_TIMEOUT\n"); 911 nv_wr32(priv, 0x002100, 0x00800000); 912 stat &= ~0x00800000; 913 } 914 915 if (stat & 0x01000000) { 916 nv_error(priv, "LB_ERROR\n"); 917 nv_wr32(priv, 0x002100, 0x01000000); 918 stat &= ~0x01000000; 919 } 920 921 if (stat & 0x08000000) { 922 nve0_fifo_intr_dropped_fault(priv); 923 nv_wr32(priv, 0x002100, 0x08000000); 924 stat &= ~0x08000000; 925 } 926 927 if (stat & 0x10000000) { 928 u32 mask = nv_rd32(priv, 0x00259c); 929 while (mask) { 930 u32 unit = __ffs(mask); 931 nve0_fifo_intr_fault(priv, unit); 932 nv_wr32(priv, 0x00259c, (1 << unit)); 933 mask &= ~(1 << unit); 934 } 935 stat &= ~0x10000000; 936 } 937 938 if (stat & 0x20000000) { 939 u32 mask = nv_rd32(priv, 0x0025a0); 940 while (mask) { 941 u32 unit = __ffs(mask); 942 nve0_fifo_intr_pbdma(priv, unit); 943 nv_wr32(priv, 0x0025a0, (1 << unit)); 944 mask &= ~(1 << unit); 945 } 946 stat &= ~0x20000000; 947 } 948 949 if (stat & 0x40000000) { 950 nve0_fifo_intr_runlist(priv); 951 stat &= ~0x40000000; 952 } 953 954 if (stat & 0x80000000) { 955 nv_wr32(priv, 0x002100, 0x80000000); 956 nve0_fifo_intr_engine(priv); 957 stat &= ~0x80000000; 958 } 959 960 if (stat) { 961 nv_error(priv, "INTR 0x%08x\n", stat); 962 nv_mask(priv, 0x002140, stat, 0x00000000); 963 nv_wr32(priv, 0x002100, stat); 964 } 965} 966 967static void 968nve0_fifo_uevent_init(struct nvkm_event *event, int type, int index) 969{ 970 struct nouveau_fifo *fifo = container_of(event, typeof(*fifo), uevent); 971 nv_mask(fifo, 0x002140, 0x80000000, 0x80000000); 972} 973 974static void 975nve0_fifo_uevent_fini(struct nvkm_event *event, int type, int index) 976{ 977 struct nouveau_fifo *fifo = container_of(event, typeof(*fifo), uevent); 978 nv_mask(fifo, 0x002140, 0x80000000, 0x00000000); 979} 980 981static const struct nvkm_event_func 982nve0_fifo_uevent_func = { 983 .ctor = nouveau_fifo_uevent_ctor, 984 .init = nve0_fifo_uevent_init, 985 .fini = nve0_fifo_uevent_fini, 986}; 987 988int 989nve0_fifo_fini(struct nouveau_object *object, bool suspend) 990{ 991 struct nve0_fifo_priv *priv = (void *)object; 992 int ret; 993 994 ret = nouveau_fifo_fini(&priv->base, suspend); 995 if (ret) 996 return ret; 997 998 /* allow mmu fault interrupts, even when we're not using fifo */ 999 nv_mask(priv, 0x002140, 0x10000000, 0x10000000); 1000 return 0; 1001} 1002 1003int 1004nve0_fifo_init(struct nouveau_object *object) 1005{ 1006 struct nve0_fifo_priv *priv = (void *)object; 1007 int ret, i; 1008 1009 ret = nouveau_fifo_init(&priv->base); 1010 if (ret) 1011 return ret; 1012 1013 /* enable all available PBDMA units */ 1014 nv_wr32(priv, 0x000204, 0xffffffff); 1015 priv->spoon_nr = hweight32(nv_rd32(priv, 0x000204)); 1016 nv_debug(priv, "%d PBDMA unit(s)\n", priv->spoon_nr); 1017 1018 /* PBDMA[n] */ 1019 for (i = 0; i < priv->spoon_nr; i++) { 1020 nv_mask(priv, 0x04013c + (i * 0x2000), 0x10000100, 0x00000000); 1021 nv_wr32(priv, 0x040108 + (i * 0x2000), 0xffffffff); /* INTR */ 1022 nv_wr32(priv, 0x04010c + (i * 0x2000), 0xfffffeff); /* INTREN */ 1023 } 1024 1025 nv_wr32(priv, 0x002254, 0x10000000 | priv->user.bar.offset >> 12); 1026 1027 nv_wr32(priv, 0x002100, 0xffffffff); 1028 nv_wr32(priv, 0x002140, 0x7fffffff); 1029 return 0; 1030} 1031 1032void 1033nve0_fifo_dtor(struct nouveau_object *object) 1034{ 1035 struct nve0_fifo_priv *priv = (void *)object; 1036 int i; 1037 1038 nouveau_gpuobj_unmap(&priv->user.bar); 1039 nouveau_gpuobj_ref(NULL, &priv->user.mem); 1040 1041 for (i = 0; i < FIFO_ENGINE_NR; i++) { 1042 nouveau_gpuobj_ref(NULL, &priv->engine[i].runlist[1]); 1043 nouveau_gpuobj_ref(NULL, &priv->engine[i].runlist[0]); 1044 } 1045 1046 nouveau_fifo_destroy(&priv->base); 1047} 1048 1049int 1050nve0_fifo_ctor(struct nouveau_object *parent, struct nouveau_object *engine, 1051 struct nouveau_oclass *oclass, void *data, u32 size, 1052 struct nouveau_object **pobject) 1053{ 1054 struct nve0_fifo_impl *impl = (void *)oclass; 1055 struct nve0_fifo_priv *priv; 1056 int ret, i; 1057 1058 ret = nouveau_fifo_create(parent, engine, oclass, 0, 1059 impl->channels - 1, &priv); 1060 *pobject = nv_object(priv); 1061 if (ret) 1062 return ret; 1063 1064 INIT_WORK(&priv->fault, nve0_fifo_recover_work); 1065 1066 for (i = 0; i < FIFO_ENGINE_NR; i++) { 1067 ret = nouveau_gpuobj_new(nv_object(priv), NULL, 0x8000, 0x1000, 1068 0, &priv->engine[i].runlist[0]); 1069 if (ret) 1070 return ret; 1071 1072 ret = nouveau_gpuobj_new(nv_object(priv), NULL, 0x8000, 0x1000, 1073 0, &priv->engine[i].runlist[1]); 1074 if (ret) 1075 return ret; 1076 1077 init_waitqueue_head(&priv->engine[i].wait); 1078 } 1079 1080 ret = nouveau_gpuobj_new(nv_object(priv), NULL, impl->channels * 0x200, 1081 0x1000, NVOBJ_FLAG_ZERO_ALLOC, &priv->user.mem); 1082 if (ret) 1083 return ret; 1084 1085 ret = nouveau_gpuobj_map(priv->user.mem, NV_MEM_ACCESS_RW, 1086 &priv->user.bar); 1087 if (ret) 1088 return ret; 1089 1090 ret = nvkm_event_init(&nve0_fifo_uevent_func, 1, 1, &priv->base.uevent); 1091 if (ret) 1092 return ret; 1093 1094 nv_subdev(priv)->unit = 0x00000100; 1095 nv_subdev(priv)->intr = nve0_fifo_intr; 1096 nv_engine(priv)->cclass = &nve0_fifo_cclass; 1097 nv_engine(priv)->sclass = nve0_fifo_sclass; 1098 return 0; 1099} 1100 1101struct nouveau_oclass * 1102nve0_fifo_oclass = &(struct nve0_fifo_impl) { 1103 .base.handle = NV_ENGINE(FIFO, 0xe0), 1104 .base.ofuncs = &(struct nouveau_ofuncs) { 1105 .ctor = nve0_fifo_ctor, 1106 .dtor = nve0_fifo_dtor, 1107 .init = nve0_fifo_init, 1108 .fini = nve0_fifo_fini, 1109 }, 1110 .channels = 4096, 1111}.base; 1112