16ee738610f41b59733f63718f0bdbcba7d3a3f12Ben Skeggs/* 26ee738610f41b59733f63718f0bdbcba7d3a3f12Ben Skeggs * Copyright (C) 2007 Ben Skeggs. 36ee738610f41b59733f63718f0bdbcba7d3a3f12Ben Skeggs * All Rights Reserved. 46ee738610f41b59733f63718f0bdbcba7d3a3f12Ben Skeggs * 56ee738610f41b59733f63718f0bdbcba7d3a3f12Ben Skeggs * Permission is hereby granted, free of charge, to any person obtaining 66ee738610f41b59733f63718f0bdbcba7d3a3f12Ben Skeggs * a copy of this software and associated documentation files (the 76ee738610f41b59733f63718f0bdbcba7d3a3f12Ben Skeggs * "Software"), to deal in the Software without restriction, including 86ee738610f41b59733f63718f0bdbcba7d3a3f12Ben Skeggs * without limitation the rights to use, copy, modify, merge, publish, 96ee738610f41b59733f63718f0bdbcba7d3a3f12Ben Skeggs * distribute, sublicense, and/or sell copies of the Software, and to 106ee738610f41b59733f63718f0bdbcba7d3a3f12Ben Skeggs * permit persons to whom the Software is furnished to do so, subject to 116ee738610f41b59733f63718f0bdbcba7d3a3f12Ben Skeggs * the following conditions: 126ee738610f41b59733f63718f0bdbcba7d3a3f12Ben Skeggs * 136ee738610f41b59733f63718f0bdbcba7d3a3f12Ben Skeggs * The above copyright notice and this permission notice (including the 146ee738610f41b59733f63718f0bdbcba7d3a3f12Ben Skeggs * next paragraph) shall be included in all copies or substantial 156ee738610f41b59733f63718f0bdbcba7d3a3f12Ben Skeggs * portions of the Software. 166ee738610f41b59733f63718f0bdbcba7d3a3f12Ben Skeggs * 176ee738610f41b59733f63718f0bdbcba7d3a3f12Ben Skeggs * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 186ee738610f41b59733f63718f0bdbcba7d3a3f12Ben Skeggs * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 196ee738610f41b59733f63718f0bdbcba7d3a3f12Ben Skeggs * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. 206ee738610f41b59733f63718f0bdbcba7d3a3f12Ben Skeggs * IN NO EVENT SHALL THE COPYRIGHT OWNER(S) AND/OR ITS SUPPLIERS BE 216ee738610f41b59733f63718f0bdbcba7d3a3f12Ben Skeggs * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION 226ee738610f41b59733f63718f0bdbcba7d3a3f12Ben Skeggs * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION 236ee738610f41b59733f63718f0bdbcba7d3a3f12Ben Skeggs * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 246ee738610f41b59733f63718f0bdbcba7d3a3f12Ben Skeggs * 256ee738610f41b59733f63718f0bdbcba7d3a3f12Ben Skeggs */ 266ee738610f41b59733f63718f0bdbcba7d3a3f12Ben Skeggs 27ebb945a94bba2ce8dff7b0942ff2b3f2a52a0a69Ben Skeggs#include "nouveau_drm.h" 28ebb945a94bba2ce8dff7b0942ff2b3f2a52a0a69Ben Skeggs#include "nouveau_dma.h" 2975c99da6cdd0ec9d70747f9ced3e85b2dd955cdfBen Skeggs 306ee738610f41b59733f63718f0bdbcba7d3a3f12Ben Skeggsvoid 316ee738610f41b59733f63718f0bdbcba7d3a3f12Ben SkeggsOUT_RINGp(struct nouveau_channel *chan, const void *data, unsigned nr_dwords) 326ee738610f41b59733f63718f0bdbcba7d3a3f12Ben Skeggs{ 336ee738610f41b59733f63718f0bdbcba7d3a3f12Ben Skeggs bool is_iomem; 34ebb945a94bba2ce8dff7b0942ff2b3f2a52a0a69Ben Skeggs u32 *mem = ttm_kmap_obj_virtual(&chan->push.buffer->kmap, &is_iomem); 356ee738610f41b59733f63718f0bdbcba7d3a3f12Ben Skeggs mem = &mem[chan->dma.cur]; 366ee738610f41b59733f63718f0bdbcba7d3a3f12Ben Skeggs if (is_iomem) 376ee738610f41b59733f63718f0bdbcba7d3a3f12Ben Skeggs memcpy_toio((void __force __iomem *)mem, data, nr_dwords * 4); 386ee738610f41b59733f63718f0bdbcba7d3a3f12Ben Skeggs else 396ee738610f41b59733f63718f0bdbcba7d3a3f12Ben Skeggs memcpy(mem, data, nr_dwords * 4); 406ee738610f41b59733f63718f0bdbcba7d3a3f12Ben Skeggs chan->dma.cur += nr_dwords; 416ee738610f41b59733f63718f0bdbcba7d3a3f12Ben Skeggs} 426ee738610f41b59733f63718f0bdbcba7d3a3f12Ben Skeggs 43ba59953d281747b1f7518a60f0ba8ff671cd0d65Ben Skeggs/* Fetch and adjust GPU GET pointer 44ba59953d281747b1f7518a60f0ba8ff671cd0d65Ben Skeggs * 45ba59953d281747b1f7518a60f0ba8ff671cd0d65Ben Skeggs * Returns: 46ba59953d281747b1f7518a60f0ba8ff671cd0d65Ben Skeggs * value >= 0, the adjusted GET pointer 47ba59953d281747b1f7518a60f0ba8ff671cd0d65Ben Skeggs * -EINVAL if GET pointer currently outside main push buffer 48ba59953d281747b1f7518a60f0ba8ff671cd0d65Ben Skeggs * -EBUSY if timeout exceeded 49ba59953d281747b1f7518a60f0ba8ff671cd0d65Ben Skeggs */ 50ba59953d281747b1f7518a60f0ba8ff671cd0d65Ben Skeggsstatic inline int 514e03b4af6dd3cff445fc0455805b43b101647bfcFrancisco JerezREAD_GET(struct nouveau_channel *chan, uint64_t *prev_get, int *timeout) 526ee738610f41b59733f63718f0bdbcba7d3a3f12Ben Skeggs{ 534e03b4af6dd3cff445fc0455805b43b101647bfcFrancisco Jerez uint64_t val; 546ee738610f41b59733f63718f0bdbcba7d3a3f12Ben Skeggs 55967e7bde8739fe3b215f7537e8f1f39c044902afBen Skeggs val = nvif_rd32(chan, chan->user_get); 564e03b4af6dd3cff445fc0455805b43b101647bfcFrancisco Jerez if (chan->user_get_hi) 57967e7bde8739fe3b215f7537e8f1f39c044902afBen Skeggs val |= (uint64_t)nvif_rd32(chan, chan->user_get_hi) << 32; 58ba59953d281747b1f7518a60f0ba8ff671cd0d65Ben Skeggs 59ba59953d281747b1f7518a60f0ba8ff671cd0d65Ben Skeggs /* reset counter as long as GET is still advancing, this is 60ba59953d281747b1f7518a60f0ba8ff671cd0d65Ben Skeggs * to avoid misdetecting a GPU lockup if the GPU happens to 61ba59953d281747b1f7518a60f0ba8ff671cd0d65Ben Skeggs * just be processing an operation that takes a long time 62ba59953d281747b1f7518a60f0ba8ff671cd0d65Ben Skeggs */ 63ba59953d281747b1f7518a60f0ba8ff671cd0d65Ben Skeggs if (val != *prev_get) { 64ba59953d281747b1f7518a60f0ba8ff671cd0d65Ben Skeggs *prev_get = val; 65ba59953d281747b1f7518a60f0ba8ff671cd0d65Ben Skeggs *timeout = 0; 66ba59953d281747b1f7518a60f0ba8ff671cd0d65Ben Skeggs } 67ba59953d281747b1f7518a60f0ba8ff671cd0d65Ben Skeggs 68ba59953d281747b1f7518a60f0ba8ff671cd0d65Ben Skeggs if ((++*timeout & 0xff) == 0) { 69ebb945a94bba2ce8dff7b0942ff2b3f2a52a0a69Ben Skeggs udelay(1); 70ba59953d281747b1f7518a60f0ba8ff671cd0d65Ben Skeggs if (*timeout > 100000) 71ba59953d281747b1f7518a60f0ba8ff671cd0d65Ben Skeggs return -EBUSY; 726ee738610f41b59733f63718f0bdbcba7d3a3f12Ben Skeggs } 736ee738610f41b59733f63718f0bdbcba7d3a3f12Ben Skeggs 74ebb945a94bba2ce8dff7b0942ff2b3f2a52a0a69Ben Skeggs if (val < chan->push.vma.offset || 75ebb945a94bba2ce8dff7b0942ff2b3f2a52a0a69Ben Skeggs val > chan->push.vma.offset + (chan->dma.max << 2)) 76ba59953d281747b1f7518a60f0ba8ff671cd0d65Ben Skeggs return -EINVAL; 77ba59953d281747b1f7518a60f0ba8ff671cd0d65Ben Skeggs 78ebb945a94bba2ce8dff7b0942ff2b3f2a52a0a69Ben Skeggs return (val - chan->push.vma.offset) >> 2; 796ee738610f41b59733f63718f0bdbcba7d3a3f12Ben Skeggs} 806ee738610f41b59733f63718f0bdbcba7d3a3f12Ben Skeggs 819a391ad8a2cdd7e5be9b6aabb56f4a46683ba377Ben Skeggsvoid 829a391ad8a2cdd7e5be9b6aabb56f4a46683ba377Ben Skeggsnv50_dma_push(struct nouveau_channel *chan, struct nouveau_bo *bo, 83a1606a9596e54da90ad6209071b357a4c1b0fa82Ben Skeggs int delta, int length) 849a391ad8a2cdd7e5be9b6aabb56f4a46683ba377Ben Skeggs{ 850ad72863ea426d46b2786cba9430e122a40aad0bBen Skeggs struct nouveau_cli *cli = (void *)nvif_client(&chan->device->base); 86ebb945a94bba2ce8dff7b0942ff2b3f2a52a0a69Ben Skeggs struct nouveau_bo *pb = chan->push.buffer; 879f9f51fcb92ba3c1f395e0908407c8c1f5305a31Ben Skeggs struct nouveau_vma *vma; 889a391ad8a2cdd7e5be9b6aabb56f4a46683ba377Ben Skeggs int ip = (chan->dma.ib_put * 2) + chan->dma.ib_base; 899f9f51fcb92ba3c1f395e0908407c8c1f5305a31Ben Skeggs u64 offset; 909f9f51fcb92ba3c1f395e0908407c8c1f5305a31Ben Skeggs 910ad72863ea426d46b2786cba9430e122a40aad0bBen Skeggs vma = nouveau_bo_vma_find(bo, cli->vm); 929f9f51fcb92ba3c1f395e0908407c8c1f5305a31Ben Skeggs BUG_ON(!vma); 939f9f51fcb92ba3c1f395e0908407c8c1f5305a31Ben Skeggs offset = vma->offset + delta; 949a391ad8a2cdd7e5be9b6aabb56f4a46683ba377Ben Skeggs 959a391ad8a2cdd7e5be9b6aabb56f4a46683ba377Ben Skeggs BUG_ON(chan->dma.ib_free < 1); 96ebb945a94bba2ce8dff7b0942ff2b3f2a52a0a69Ben Skeggs 97d87897d4c853a5c136f60efa858d0d4cb3740f57Ben Skeggs nouveau_bo_wr32(pb, ip++, lower_32_bits(offset)); 98a1606a9596e54da90ad6209071b357a4c1b0fa82Ben Skeggs nouveau_bo_wr32(pb, ip++, upper_32_bits(offset) | length << 8); 999a391ad8a2cdd7e5be9b6aabb56f4a46683ba377Ben Skeggs 1009a391ad8a2cdd7e5be9b6aabb56f4a46683ba377Ben Skeggs chan->dma.ib_put = (chan->dma.ib_put + 1) & chan->dma.ib_max; 101ce48fa93a6f5cadd4141a921dfb4129c8850374eMaarten Maathuis 10285b2331b3437a55c21cce3ee1ea0bd12301ecb56Daniel Vetter mb(); 103ce48fa93a6f5cadd4141a921dfb4129c8850374eMaarten Maathuis /* Flush writes. */ 104ce48fa93a6f5cadd4141a921dfb4129c8850374eMaarten Maathuis nouveau_bo_rd32(pb, 0); 105ce48fa93a6f5cadd4141a921dfb4129c8850374eMaarten Maathuis 106967e7bde8739fe3b215f7537e8f1f39c044902afBen Skeggs nvif_wr32(chan, 0x8c, chan->dma.ib_put); 1079a391ad8a2cdd7e5be9b6aabb56f4a46683ba377Ben Skeggs chan->dma.ib_free--; 1089a391ad8a2cdd7e5be9b6aabb56f4a46683ba377Ben Skeggs} 1099a391ad8a2cdd7e5be9b6aabb56f4a46683ba377Ben Skeggs 1109a391ad8a2cdd7e5be9b6aabb56f4a46683ba377Ben Skeggsstatic int 1119a391ad8a2cdd7e5be9b6aabb56f4a46683ba377Ben Skeggsnv50_dma_push_wait(struct nouveau_channel *chan, int count) 1129a391ad8a2cdd7e5be9b6aabb56f4a46683ba377Ben Skeggs{ 1139a391ad8a2cdd7e5be9b6aabb56f4a46683ba377Ben Skeggs uint32_t cnt = 0, prev_get = 0; 1149a391ad8a2cdd7e5be9b6aabb56f4a46683ba377Ben Skeggs 1159a391ad8a2cdd7e5be9b6aabb56f4a46683ba377Ben Skeggs while (chan->dma.ib_free < count) { 116967e7bde8739fe3b215f7537e8f1f39c044902afBen Skeggs uint32_t get = nvif_rd32(chan, 0x88); 1179a391ad8a2cdd7e5be9b6aabb56f4a46683ba377Ben Skeggs if (get != prev_get) { 1189a391ad8a2cdd7e5be9b6aabb56f4a46683ba377Ben Skeggs prev_get = get; 1199a391ad8a2cdd7e5be9b6aabb56f4a46683ba377Ben Skeggs cnt = 0; 1209a391ad8a2cdd7e5be9b6aabb56f4a46683ba377Ben Skeggs } 1219a391ad8a2cdd7e5be9b6aabb56f4a46683ba377Ben Skeggs 1229a391ad8a2cdd7e5be9b6aabb56f4a46683ba377Ben Skeggs if ((++cnt & 0xff) == 0) { 1239a391ad8a2cdd7e5be9b6aabb56f4a46683ba377Ben Skeggs DRM_UDELAY(1); 1249a391ad8a2cdd7e5be9b6aabb56f4a46683ba377Ben Skeggs if (cnt > 100000) 1259a391ad8a2cdd7e5be9b6aabb56f4a46683ba377Ben Skeggs return -EBUSY; 1269a391ad8a2cdd7e5be9b6aabb56f4a46683ba377Ben Skeggs } 1279a391ad8a2cdd7e5be9b6aabb56f4a46683ba377Ben Skeggs 1289a391ad8a2cdd7e5be9b6aabb56f4a46683ba377Ben Skeggs chan->dma.ib_free = get - chan->dma.ib_put; 1299a391ad8a2cdd7e5be9b6aabb56f4a46683ba377Ben Skeggs if (chan->dma.ib_free <= 0) 13062841ab726def838472dfaaee571a0d30a2ce1e0Ben Skeggs chan->dma.ib_free += chan->dma.ib_max; 1319a391ad8a2cdd7e5be9b6aabb56f4a46683ba377Ben Skeggs } 1329a391ad8a2cdd7e5be9b6aabb56f4a46683ba377Ben Skeggs 1339a391ad8a2cdd7e5be9b6aabb56f4a46683ba377Ben Skeggs return 0; 1349a391ad8a2cdd7e5be9b6aabb56f4a46683ba377Ben Skeggs} 1359a391ad8a2cdd7e5be9b6aabb56f4a46683ba377Ben Skeggs 1369a391ad8a2cdd7e5be9b6aabb56f4a46683ba377Ben Skeggsstatic int 1379a391ad8a2cdd7e5be9b6aabb56f4a46683ba377Ben Skeggsnv50_dma_wait(struct nouveau_channel *chan, int slots, int count) 1389a391ad8a2cdd7e5be9b6aabb56f4a46683ba377Ben Skeggs{ 1394e03b4af6dd3cff445fc0455805b43b101647bfcFrancisco Jerez uint64_t prev_get = 0; 1404e03b4af6dd3cff445fc0455805b43b101647bfcFrancisco Jerez int ret, cnt = 0; 1419a391ad8a2cdd7e5be9b6aabb56f4a46683ba377Ben Skeggs 1429a391ad8a2cdd7e5be9b6aabb56f4a46683ba377Ben Skeggs ret = nv50_dma_push_wait(chan, slots + 1); 1439a391ad8a2cdd7e5be9b6aabb56f4a46683ba377Ben Skeggs if (unlikely(ret)) 1449a391ad8a2cdd7e5be9b6aabb56f4a46683ba377Ben Skeggs return ret; 1459a391ad8a2cdd7e5be9b6aabb56f4a46683ba377Ben Skeggs 1469a391ad8a2cdd7e5be9b6aabb56f4a46683ba377Ben Skeggs while (chan->dma.free < count) { 1479a391ad8a2cdd7e5be9b6aabb56f4a46683ba377Ben Skeggs int get = READ_GET(chan, &prev_get, &cnt); 1489a391ad8a2cdd7e5be9b6aabb56f4a46683ba377Ben Skeggs if (unlikely(get < 0)) { 1499a391ad8a2cdd7e5be9b6aabb56f4a46683ba377Ben Skeggs if (get == -EINVAL) 1509a391ad8a2cdd7e5be9b6aabb56f4a46683ba377Ben Skeggs continue; 1519a391ad8a2cdd7e5be9b6aabb56f4a46683ba377Ben Skeggs 1529a391ad8a2cdd7e5be9b6aabb56f4a46683ba377Ben Skeggs return get; 1539a391ad8a2cdd7e5be9b6aabb56f4a46683ba377Ben Skeggs } 1549a391ad8a2cdd7e5be9b6aabb56f4a46683ba377Ben Skeggs 1559a391ad8a2cdd7e5be9b6aabb56f4a46683ba377Ben Skeggs if (get <= chan->dma.cur) { 1569a391ad8a2cdd7e5be9b6aabb56f4a46683ba377Ben Skeggs chan->dma.free = chan->dma.max - chan->dma.cur; 1579a391ad8a2cdd7e5be9b6aabb56f4a46683ba377Ben Skeggs if (chan->dma.free >= count) 1589a391ad8a2cdd7e5be9b6aabb56f4a46683ba377Ben Skeggs break; 1599a391ad8a2cdd7e5be9b6aabb56f4a46683ba377Ben Skeggs 1609a391ad8a2cdd7e5be9b6aabb56f4a46683ba377Ben Skeggs FIRE_RING(chan); 1619a391ad8a2cdd7e5be9b6aabb56f4a46683ba377Ben Skeggs do { 1629a391ad8a2cdd7e5be9b6aabb56f4a46683ba377Ben Skeggs get = READ_GET(chan, &prev_get, &cnt); 1639a391ad8a2cdd7e5be9b6aabb56f4a46683ba377Ben Skeggs if (unlikely(get < 0)) { 1649a391ad8a2cdd7e5be9b6aabb56f4a46683ba377Ben Skeggs if (get == -EINVAL) 1659a391ad8a2cdd7e5be9b6aabb56f4a46683ba377Ben Skeggs continue; 1669a391ad8a2cdd7e5be9b6aabb56f4a46683ba377Ben Skeggs return get; 1679a391ad8a2cdd7e5be9b6aabb56f4a46683ba377Ben Skeggs } 1689a391ad8a2cdd7e5be9b6aabb56f4a46683ba377Ben Skeggs } while (get == 0); 1699a391ad8a2cdd7e5be9b6aabb56f4a46683ba377Ben Skeggs chan->dma.cur = 0; 1709a391ad8a2cdd7e5be9b6aabb56f4a46683ba377Ben Skeggs chan->dma.put = 0; 1719a391ad8a2cdd7e5be9b6aabb56f4a46683ba377Ben Skeggs } 1729a391ad8a2cdd7e5be9b6aabb56f4a46683ba377Ben Skeggs 1739a391ad8a2cdd7e5be9b6aabb56f4a46683ba377Ben Skeggs chan->dma.free = get - chan->dma.cur - 1; 1749a391ad8a2cdd7e5be9b6aabb56f4a46683ba377Ben Skeggs } 1759a391ad8a2cdd7e5be9b6aabb56f4a46683ba377Ben Skeggs 1769a391ad8a2cdd7e5be9b6aabb56f4a46683ba377Ben Skeggs return 0; 1779a391ad8a2cdd7e5be9b6aabb56f4a46683ba377Ben Skeggs} 1789a391ad8a2cdd7e5be9b6aabb56f4a46683ba377Ben Skeggs 1796ee738610f41b59733f63718f0bdbcba7d3a3f12Ben Skeggsint 1809a391ad8a2cdd7e5be9b6aabb56f4a46683ba377Ben Skeggsnouveau_dma_wait(struct nouveau_channel *chan, int slots, int size) 1816ee738610f41b59733f63718f0bdbcba7d3a3f12Ben Skeggs{ 1824e03b4af6dd3cff445fc0455805b43b101647bfcFrancisco Jerez uint64_t prev_get = 0; 1834e03b4af6dd3cff445fc0455805b43b101647bfcFrancisco Jerez int cnt = 0, get; 1846ee738610f41b59733f63718f0bdbcba7d3a3f12Ben Skeggs 1859a391ad8a2cdd7e5be9b6aabb56f4a46683ba377Ben Skeggs if (chan->dma.ib_max) 1869a391ad8a2cdd7e5be9b6aabb56f4a46683ba377Ben Skeggs return nv50_dma_wait(chan, slots, size); 1879a391ad8a2cdd7e5be9b6aabb56f4a46683ba377Ben Skeggs 1886ee738610f41b59733f63718f0bdbcba7d3a3f12Ben Skeggs while (chan->dma.free < size) { 189ba59953d281747b1f7518a60f0ba8ff671cd0d65Ben Skeggs get = READ_GET(chan, &prev_get, &cnt); 190ba59953d281747b1f7518a60f0ba8ff671cd0d65Ben Skeggs if (unlikely(get == -EBUSY)) 191ba59953d281747b1f7518a60f0ba8ff671cd0d65Ben Skeggs return -EBUSY; 1926ee738610f41b59733f63718f0bdbcba7d3a3f12Ben Skeggs 1936ee738610f41b59733f63718f0bdbcba7d3a3f12Ben Skeggs /* loop until we have a usable GET pointer. the value 1946ee738610f41b59733f63718f0bdbcba7d3a3f12Ben Skeggs * we read from the GPU may be outside the main ring if 1956ee738610f41b59733f63718f0bdbcba7d3a3f12Ben Skeggs * PFIFO is processing a buffer called from the main ring, 1966ee738610f41b59733f63718f0bdbcba7d3a3f12Ben Skeggs * discard these values until something sensible is seen. 1976ee738610f41b59733f63718f0bdbcba7d3a3f12Ben Skeggs * 1986ee738610f41b59733f63718f0bdbcba7d3a3f12Ben Skeggs * the other case we discard GET is while the GPU is fetching 1996ee738610f41b59733f63718f0bdbcba7d3a3f12Ben Skeggs * from the SKIPS area, so the code below doesn't have to deal 2006ee738610f41b59733f63718f0bdbcba7d3a3f12Ben Skeggs * with some fun corner cases. 2016ee738610f41b59733f63718f0bdbcba7d3a3f12Ben Skeggs */ 202ba59953d281747b1f7518a60f0ba8ff671cd0d65Ben Skeggs if (unlikely(get == -EINVAL) || get < NOUVEAU_DMA_SKIPS) 2036ee738610f41b59733f63718f0bdbcba7d3a3f12Ben Skeggs continue; 2046ee738610f41b59733f63718f0bdbcba7d3a3f12Ben Skeggs 2056ee738610f41b59733f63718f0bdbcba7d3a3f12Ben Skeggs if (get <= chan->dma.cur) { 2066ee738610f41b59733f63718f0bdbcba7d3a3f12Ben Skeggs /* engine is fetching behind us, or is completely 2076ee738610f41b59733f63718f0bdbcba7d3a3f12Ben Skeggs * idle (GET == PUT) so we have free space up until 2086ee738610f41b59733f63718f0bdbcba7d3a3f12Ben Skeggs * the end of the push buffer 2096ee738610f41b59733f63718f0bdbcba7d3a3f12Ben Skeggs * 2106ee738610f41b59733f63718f0bdbcba7d3a3f12Ben Skeggs * we can only hit that path once per call due to 2116ee738610f41b59733f63718f0bdbcba7d3a3f12Ben Skeggs * looping back to the beginning of the push buffer, 2126ee738610f41b59733f63718f0bdbcba7d3a3f12Ben Skeggs * we'll hit the fetching-ahead-of-us path from that 2136ee738610f41b59733f63718f0bdbcba7d3a3f12Ben Skeggs * point on. 2146ee738610f41b59733f63718f0bdbcba7d3a3f12Ben Skeggs * 2156ee738610f41b59733f63718f0bdbcba7d3a3f12Ben Skeggs * the *one* exception to that rule is if we read 2166ee738610f41b59733f63718f0bdbcba7d3a3f12Ben Skeggs * GET==PUT, in which case the below conditional will 2176ee738610f41b59733f63718f0bdbcba7d3a3f12Ben Skeggs * always succeed and break us out of the wait loop. 2186ee738610f41b59733f63718f0bdbcba7d3a3f12Ben Skeggs */ 2196ee738610f41b59733f63718f0bdbcba7d3a3f12Ben Skeggs chan->dma.free = chan->dma.max - chan->dma.cur; 2206ee738610f41b59733f63718f0bdbcba7d3a3f12Ben Skeggs if (chan->dma.free >= size) 2216ee738610f41b59733f63718f0bdbcba7d3a3f12Ben Skeggs break; 2226ee738610f41b59733f63718f0bdbcba7d3a3f12Ben Skeggs 2236ee738610f41b59733f63718f0bdbcba7d3a3f12Ben Skeggs /* not enough space left at the end of the push buffer, 2246ee738610f41b59733f63718f0bdbcba7d3a3f12Ben Skeggs * instruct the GPU to jump back to the start right 2256ee738610f41b59733f63718f0bdbcba7d3a3f12Ben Skeggs * after processing the currently pending commands. 2266ee738610f41b59733f63718f0bdbcba7d3a3f12Ben Skeggs */ 227ebb945a94bba2ce8dff7b0942ff2b3f2a52a0a69Ben Skeggs OUT_RING(chan, chan->push.vma.offset | 0x20000000); 228ba59953d281747b1f7518a60f0ba8ff671cd0d65Ben Skeggs 229ba59953d281747b1f7518a60f0ba8ff671cd0d65Ben Skeggs /* wait for GET to depart from the skips area. 230ba59953d281747b1f7518a60f0ba8ff671cd0d65Ben Skeggs * prevents writing GET==PUT and causing a race 231ba59953d281747b1f7518a60f0ba8ff671cd0d65Ben Skeggs * condition that causes us to think the GPU is 232ba59953d281747b1f7518a60f0ba8ff671cd0d65Ben Skeggs * idle when it's not. 233ba59953d281747b1f7518a60f0ba8ff671cd0d65Ben Skeggs */ 234ba59953d281747b1f7518a60f0ba8ff671cd0d65Ben Skeggs do { 235ba59953d281747b1f7518a60f0ba8ff671cd0d65Ben Skeggs get = READ_GET(chan, &prev_get, &cnt); 236ba59953d281747b1f7518a60f0ba8ff671cd0d65Ben Skeggs if (unlikely(get == -EBUSY)) 237ba59953d281747b1f7518a60f0ba8ff671cd0d65Ben Skeggs return -EBUSY; 238ba59953d281747b1f7518a60f0ba8ff671cd0d65Ben Skeggs if (unlikely(get == -EINVAL)) 239ba59953d281747b1f7518a60f0ba8ff671cd0d65Ben Skeggs continue; 240ba59953d281747b1f7518a60f0ba8ff671cd0d65Ben Skeggs } while (get <= NOUVEAU_DMA_SKIPS); 2416ee738610f41b59733f63718f0bdbcba7d3a3f12Ben Skeggs WRITE_PUT(NOUVEAU_DMA_SKIPS); 2426ee738610f41b59733f63718f0bdbcba7d3a3f12Ben Skeggs 2436ee738610f41b59733f63718f0bdbcba7d3a3f12Ben Skeggs /* we're now submitting commands at the start of 2446ee738610f41b59733f63718f0bdbcba7d3a3f12Ben Skeggs * the push buffer. 2456ee738610f41b59733f63718f0bdbcba7d3a3f12Ben Skeggs */ 2466ee738610f41b59733f63718f0bdbcba7d3a3f12Ben Skeggs chan->dma.cur = 2476ee738610f41b59733f63718f0bdbcba7d3a3f12Ben Skeggs chan->dma.put = NOUVEAU_DMA_SKIPS; 2486ee738610f41b59733f63718f0bdbcba7d3a3f12Ben Skeggs } 2496ee738610f41b59733f63718f0bdbcba7d3a3f12Ben Skeggs 2506ee738610f41b59733f63718f0bdbcba7d3a3f12Ben Skeggs /* engine fetching ahead of us, we have space up until the 2516ee738610f41b59733f63718f0bdbcba7d3a3f12Ben Skeggs * current GET pointer. the "- 1" is to ensure there's 2526ee738610f41b59733f63718f0bdbcba7d3a3f12Ben Skeggs * space left to emit a jump back to the beginning of the 2536ee738610f41b59733f63718f0bdbcba7d3a3f12Ben Skeggs * push buffer if we require it. we can never get GET == PUT 2546ee738610f41b59733f63718f0bdbcba7d3a3f12Ben Skeggs * here, so this is safe. 2556ee738610f41b59733f63718f0bdbcba7d3a3f12Ben Skeggs */ 2566ee738610f41b59733f63718f0bdbcba7d3a3f12Ben Skeggs chan->dma.free = get - chan->dma.cur - 1; 2576ee738610f41b59733f63718f0bdbcba7d3a3f12Ben Skeggs } 2586ee738610f41b59733f63718f0bdbcba7d3a3f12Ben Skeggs 2596ee738610f41b59733f63718f0bdbcba7d3a3f12Ben Skeggs return 0; 2606ee738610f41b59733f63718f0bdbcba7d3a3f12Ben Skeggs} 2616ee738610f41b59733f63718f0bdbcba7d3a3f12Ben Skeggs 262