nouveau_dma.c revision 967e7bde8739fe3b215f7537e8f1f39c044902af
16ee738610f41b59733f63718f0bdbcba7d3a3f12Ben Skeggs/* 26ee738610f41b59733f63718f0bdbcba7d3a3f12Ben Skeggs * Copyright (C) 2007 Ben Skeggs. 36ee738610f41b59733f63718f0bdbcba7d3a3f12Ben Skeggs * All Rights Reserved. 46ee738610f41b59733f63718f0bdbcba7d3a3f12Ben Skeggs * 56ee738610f41b59733f63718f0bdbcba7d3a3f12Ben Skeggs * Permission is hereby granted, free of charge, to any person obtaining 66ee738610f41b59733f63718f0bdbcba7d3a3f12Ben Skeggs * a copy of this software and associated documentation files (the 76ee738610f41b59733f63718f0bdbcba7d3a3f12Ben Skeggs * "Software"), to deal in the Software without restriction, including 86ee738610f41b59733f63718f0bdbcba7d3a3f12Ben Skeggs * without limitation the rights to use, copy, modify, merge, publish, 96ee738610f41b59733f63718f0bdbcba7d3a3f12Ben Skeggs * distribute, sublicense, and/or sell copies of the Software, and to 106ee738610f41b59733f63718f0bdbcba7d3a3f12Ben Skeggs * permit persons to whom the Software is furnished to do so, subject to 116ee738610f41b59733f63718f0bdbcba7d3a3f12Ben Skeggs * the following conditions: 126ee738610f41b59733f63718f0bdbcba7d3a3f12Ben Skeggs * 136ee738610f41b59733f63718f0bdbcba7d3a3f12Ben Skeggs * The above copyright notice and this permission notice (including the 146ee738610f41b59733f63718f0bdbcba7d3a3f12Ben Skeggs * next paragraph) shall be included in all copies or substantial 156ee738610f41b59733f63718f0bdbcba7d3a3f12Ben Skeggs * portions of the Software. 166ee738610f41b59733f63718f0bdbcba7d3a3f12Ben Skeggs * 176ee738610f41b59733f63718f0bdbcba7d3a3f12Ben Skeggs * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 186ee738610f41b59733f63718f0bdbcba7d3a3f12Ben Skeggs * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 196ee738610f41b59733f63718f0bdbcba7d3a3f12Ben Skeggs * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. 206ee738610f41b59733f63718f0bdbcba7d3a3f12Ben Skeggs * IN NO EVENT SHALL THE COPYRIGHT OWNER(S) AND/OR ITS SUPPLIERS BE 216ee738610f41b59733f63718f0bdbcba7d3a3f12Ben Skeggs * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION 226ee738610f41b59733f63718f0bdbcba7d3a3f12Ben Skeggs * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION 236ee738610f41b59733f63718f0bdbcba7d3a3f12Ben Skeggs * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 246ee738610f41b59733f63718f0bdbcba7d3a3f12Ben Skeggs * 256ee738610f41b59733f63718f0bdbcba7d3a3f12Ben Skeggs */ 266ee738610f41b59733f63718f0bdbcba7d3a3f12Ben Skeggs 27ebb945a94bba2ce8dff7b0942ff2b3f2a52a0a69Ben Skeggs#include <core/client.h> 289a391ad8a2cdd7e5be9b6aabb56f4a46683ba377Ben Skeggs 29ebb945a94bba2ce8dff7b0942ff2b3f2a52a0a69Ben Skeggs#include "nouveau_drm.h" 30ebb945a94bba2ce8dff7b0942ff2b3f2a52a0a69Ben Skeggs#include "nouveau_dma.h" 3175c99da6cdd0ec9d70747f9ced3e85b2dd955cdfBen Skeggs 326ee738610f41b59733f63718f0bdbcba7d3a3f12Ben Skeggsvoid 336ee738610f41b59733f63718f0bdbcba7d3a3f12Ben SkeggsOUT_RINGp(struct nouveau_channel *chan, const void *data, unsigned nr_dwords) 346ee738610f41b59733f63718f0bdbcba7d3a3f12Ben Skeggs{ 356ee738610f41b59733f63718f0bdbcba7d3a3f12Ben Skeggs bool is_iomem; 36ebb945a94bba2ce8dff7b0942ff2b3f2a52a0a69Ben Skeggs u32 *mem = ttm_kmap_obj_virtual(&chan->push.buffer->kmap, &is_iomem); 376ee738610f41b59733f63718f0bdbcba7d3a3f12Ben Skeggs mem = &mem[chan->dma.cur]; 386ee738610f41b59733f63718f0bdbcba7d3a3f12Ben Skeggs if (is_iomem) 396ee738610f41b59733f63718f0bdbcba7d3a3f12Ben Skeggs memcpy_toio((void __force __iomem *)mem, data, nr_dwords * 4); 406ee738610f41b59733f63718f0bdbcba7d3a3f12Ben Skeggs else 416ee738610f41b59733f63718f0bdbcba7d3a3f12Ben Skeggs memcpy(mem, data, nr_dwords * 4); 426ee738610f41b59733f63718f0bdbcba7d3a3f12Ben Skeggs chan->dma.cur += nr_dwords; 436ee738610f41b59733f63718f0bdbcba7d3a3f12Ben Skeggs} 446ee738610f41b59733f63718f0bdbcba7d3a3f12Ben Skeggs 45ba59953d281747b1f7518a60f0ba8ff671cd0d65Ben Skeggs/* Fetch and adjust GPU GET pointer 46ba59953d281747b1f7518a60f0ba8ff671cd0d65Ben Skeggs * 47ba59953d281747b1f7518a60f0ba8ff671cd0d65Ben Skeggs * Returns: 48ba59953d281747b1f7518a60f0ba8ff671cd0d65Ben Skeggs * value >= 0, the adjusted GET pointer 49ba59953d281747b1f7518a60f0ba8ff671cd0d65Ben Skeggs * -EINVAL if GET pointer currently outside main push buffer 50ba59953d281747b1f7518a60f0ba8ff671cd0d65Ben Skeggs * -EBUSY if timeout exceeded 51ba59953d281747b1f7518a60f0ba8ff671cd0d65Ben Skeggs */ 52ba59953d281747b1f7518a60f0ba8ff671cd0d65Ben Skeggsstatic inline int 534e03b4af6dd3cff445fc0455805b43b101647bfcFrancisco JerezREAD_GET(struct nouveau_channel *chan, uint64_t *prev_get, int *timeout) 546ee738610f41b59733f63718f0bdbcba7d3a3f12Ben Skeggs{ 554e03b4af6dd3cff445fc0455805b43b101647bfcFrancisco Jerez uint64_t val; 566ee738610f41b59733f63718f0bdbcba7d3a3f12Ben Skeggs 57967e7bde8739fe3b215f7537e8f1f39c044902afBen Skeggs val = nvif_rd32(chan, chan->user_get); 584e03b4af6dd3cff445fc0455805b43b101647bfcFrancisco Jerez if (chan->user_get_hi) 59967e7bde8739fe3b215f7537e8f1f39c044902afBen Skeggs val |= (uint64_t)nvif_rd32(chan, chan->user_get_hi) << 32; 60ba59953d281747b1f7518a60f0ba8ff671cd0d65Ben Skeggs 61ba59953d281747b1f7518a60f0ba8ff671cd0d65Ben Skeggs /* reset counter as long as GET is still advancing, this is 62ba59953d281747b1f7518a60f0ba8ff671cd0d65Ben Skeggs * to avoid misdetecting a GPU lockup if the GPU happens to 63ba59953d281747b1f7518a60f0ba8ff671cd0d65Ben Skeggs * just be processing an operation that takes a long time 64ba59953d281747b1f7518a60f0ba8ff671cd0d65Ben Skeggs */ 65ba59953d281747b1f7518a60f0ba8ff671cd0d65Ben Skeggs if (val != *prev_get) { 66ba59953d281747b1f7518a60f0ba8ff671cd0d65Ben Skeggs *prev_get = val; 67ba59953d281747b1f7518a60f0ba8ff671cd0d65Ben Skeggs *timeout = 0; 68ba59953d281747b1f7518a60f0ba8ff671cd0d65Ben Skeggs } 69ba59953d281747b1f7518a60f0ba8ff671cd0d65Ben Skeggs 70ba59953d281747b1f7518a60f0ba8ff671cd0d65Ben Skeggs if ((++*timeout & 0xff) == 0) { 71ebb945a94bba2ce8dff7b0942ff2b3f2a52a0a69Ben Skeggs udelay(1); 72ba59953d281747b1f7518a60f0ba8ff671cd0d65Ben Skeggs if (*timeout > 100000) 73ba59953d281747b1f7518a60f0ba8ff671cd0d65Ben Skeggs return -EBUSY; 746ee738610f41b59733f63718f0bdbcba7d3a3f12Ben Skeggs } 756ee738610f41b59733f63718f0bdbcba7d3a3f12Ben Skeggs 76ebb945a94bba2ce8dff7b0942ff2b3f2a52a0a69Ben Skeggs if (val < chan->push.vma.offset || 77ebb945a94bba2ce8dff7b0942ff2b3f2a52a0a69Ben Skeggs val > chan->push.vma.offset + (chan->dma.max << 2)) 78ba59953d281747b1f7518a60f0ba8ff671cd0d65Ben Skeggs return -EINVAL; 79ba59953d281747b1f7518a60f0ba8ff671cd0d65Ben Skeggs 80ebb945a94bba2ce8dff7b0942ff2b3f2a52a0a69Ben Skeggs return (val - chan->push.vma.offset) >> 2; 816ee738610f41b59733f63718f0bdbcba7d3a3f12Ben Skeggs} 826ee738610f41b59733f63718f0bdbcba7d3a3f12Ben Skeggs 839a391ad8a2cdd7e5be9b6aabb56f4a46683ba377Ben Skeggsvoid 849a391ad8a2cdd7e5be9b6aabb56f4a46683ba377Ben Skeggsnv50_dma_push(struct nouveau_channel *chan, struct nouveau_bo *bo, 85a1606a9596e54da90ad6209071b357a4c1b0fa82Ben Skeggs int delta, int length) 869a391ad8a2cdd7e5be9b6aabb56f4a46683ba377Ben Skeggs{ 87ebb945a94bba2ce8dff7b0942ff2b3f2a52a0a69Ben Skeggs struct nouveau_bo *pb = chan->push.buffer; 889f9f51fcb92ba3c1f395e0908407c8c1f5305a31Ben Skeggs struct nouveau_vma *vma; 899a391ad8a2cdd7e5be9b6aabb56f4a46683ba377Ben Skeggs int ip = (chan->dma.ib_put * 2) + chan->dma.ib_base; 909f9f51fcb92ba3c1f395e0908407c8c1f5305a31Ben Skeggs u64 offset; 919f9f51fcb92ba3c1f395e0908407c8c1f5305a31Ben Skeggs 92ebb945a94bba2ce8dff7b0942ff2b3f2a52a0a69Ben Skeggs vma = nouveau_bo_vma_find(bo, nv_client(chan->cli)->vm); 939f9f51fcb92ba3c1f395e0908407c8c1f5305a31Ben Skeggs BUG_ON(!vma); 949f9f51fcb92ba3c1f395e0908407c8c1f5305a31Ben Skeggs offset = vma->offset + delta; 959a391ad8a2cdd7e5be9b6aabb56f4a46683ba377Ben Skeggs 969a391ad8a2cdd7e5be9b6aabb56f4a46683ba377Ben Skeggs BUG_ON(chan->dma.ib_free < 1); 97ebb945a94bba2ce8dff7b0942ff2b3f2a52a0a69Ben Skeggs 98d87897d4c853a5c136f60efa858d0d4cb3740f57Ben Skeggs nouveau_bo_wr32(pb, ip++, lower_32_bits(offset)); 99a1606a9596e54da90ad6209071b357a4c1b0fa82Ben Skeggs nouveau_bo_wr32(pb, ip++, upper_32_bits(offset) | length << 8); 1009a391ad8a2cdd7e5be9b6aabb56f4a46683ba377Ben Skeggs 1019a391ad8a2cdd7e5be9b6aabb56f4a46683ba377Ben Skeggs chan->dma.ib_put = (chan->dma.ib_put + 1) & chan->dma.ib_max; 102ce48fa93a6f5cadd4141a921dfb4129c8850374eMaarten Maathuis 10385b2331b3437a55c21cce3ee1ea0bd12301ecb56Daniel Vetter mb(); 104ce48fa93a6f5cadd4141a921dfb4129c8850374eMaarten Maathuis /* Flush writes. */ 105ce48fa93a6f5cadd4141a921dfb4129c8850374eMaarten Maathuis nouveau_bo_rd32(pb, 0); 106ce48fa93a6f5cadd4141a921dfb4129c8850374eMaarten Maathuis 107967e7bde8739fe3b215f7537e8f1f39c044902afBen Skeggs nvif_wr32(chan, 0x8c, chan->dma.ib_put); 1089a391ad8a2cdd7e5be9b6aabb56f4a46683ba377Ben Skeggs chan->dma.ib_free--; 1099a391ad8a2cdd7e5be9b6aabb56f4a46683ba377Ben Skeggs} 1109a391ad8a2cdd7e5be9b6aabb56f4a46683ba377Ben Skeggs 1119a391ad8a2cdd7e5be9b6aabb56f4a46683ba377Ben Skeggsstatic int 1129a391ad8a2cdd7e5be9b6aabb56f4a46683ba377Ben Skeggsnv50_dma_push_wait(struct nouveau_channel *chan, int count) 1139a391ad8a2cdd7e5be9b6aabb56f4a46683ba377Ben Skeggs{ 1149a391ad8a2cdd7e5be9b6aabb56f4a46683ba377Ben Skeggs uint32_t cnt = 0, prev_get = 0; 1159a391ad8a2cdd7e5be9b6aabb56f4a46683ba377Ben Skeggs 1169a391ad8a2cdd7e5be9b6aabb56f4a46683ba377Ben Skeggs while (chan->dma.ib_free < count) { 117967e7bde8739fe3b215f7537e8f1f39c044902afBen Skeggs uint32_t get = nvif_rd32(chan, 0x88); 1189a391ad8a2cdd7e5be9b6aabb56f4a46683ba377Ben Skeggs if (get != prev_get) { 1199a391ad8a2cdd7e5be9b6aabb56f4a46683ba377Ben Skeggs prev_get = get; 1209a391ad8a2cdd7e5be9b6aabb56f4a46683ba377Ben Skeggs cnt = 0; 1219a391ad8a2cdd7e5be9b6aabb56f4a46683ba377Ben Skeggs } 1229a391ad8a2cdd7e5be9b6aabb56f4a46683ba377Ben Skeggs 1239a391ad8a2cdd7e5be9b6aabb56f4a46683ba377Ben Skeggs if ((++cnt & 0xff) == 0) { 1249a391ad8a2cdd7e5be9b6aabb56f4a46683ba377Ben Skeggs DRM_UDELAY(1); 1259a391ad8a2cdd7e5be9b6aabb56f4a46683ba377Ben Skeggs if (cnt > 100000) 1269a391ad8a2cdd7e5be9b6aabb56f4a46683ba377Ben Skeggs return -EBUSY; 1279a391ad8a2cdd7e5be9b6aabb56f4a46683ba377Ben Skeggs } 1289a391ad8a2cdd7e5be9b6aabb56f4a46683ba377Ben Skeggs 1299a391ad8a2cdd7e5be9b6aabb56f4a46683ba377Ben Skeggs chan->dma.ib_free = get - chan->dma.ib_put; 1309a391ad8a2cdd7e5be9b6aabb56f4a46683ba377Ben Skeggs if (chan->dma.ib_free <= 0) 13162841ab726def838472dfaaee571a0d30a2ce1e0Ben Skeggs chan->dma.ib_free += chan->dma.ib_max; 1329a391ad8a2cdd7e5be9b6aabb56f4a46683ba377Ben Skeggs } 1339a391ad8a2cdd7e5be9b6aabb56f4a46683ba377Ben Skeggs 1349a391ad8a2cdd7e5be9b6aabb56f4a46683ba377Ben Skeggs return 0; 1359a391ad8a2cdd7e5be9b6aabb56f4a46683ba377Ben Skeggs} 1369a391ad8a2cdd7e5be9b6aabb56f4a46683ba377Ben Skeggs 1379a391ad8a2cdd7e5be9b6aabb56f4a46683ba377Ben Skeggsstatic int 1389a391ad8a2cdd7e5be9b6aabb56f4a46683ba377Ben Skeggsnv50_dma_wait(struct nouveau_channel *chan, int slots, int count) 1399a391ad8a2cdd7e5be9b6aabb56f4a46683ba377Ben Skeggs{ 1404e03b4af6dd3cff445fc0455805b43b101647bfcFrancisco Jerez uint64_t prev_get = 0; 1414e03b4af6dd3cff445fc0455805b43b101647bfcFrancisco Jerez int ret, cnt = 0; 1429a391ad8a2cdd7e5be9b6aabb56f4a46683ba377Ben Skeggs 1439a391ad8a2cdd7e5be9b6aabb56f4a46683ba377Ben Skeggs ret = nv50_dma_push_wait(chan, slots + 1); 1449a391ad8a2cdd7e5be9b6aabb56f4a46683ba377Ben Skeggs if (unlikely(ret)) 1459a391ad8a2cdd7e5be9b6aabb56f4a46683ba377Ben Skeggs return ret; 1469a391ad8a2cdd7e5be9b6aabb56f4a46683ba377Ben Skeggs 1479a391ad8a2cdd7e5be9b6aabb56f4a46683ba377Ben Skeggs while (chan->dma.free < count) { 1489a391ad8a2cdd7e5be9b6aabb56f4a46683ba377Ben Skeggs int get = READ_GET(chan, &prev_get, &cnt); 1499a391ad8a2cdd7e5be9b6aabb56f4a46683ba377Ben Skeggs if (unlikely(get < 0)) { 1509a391ad8a2cdd7e5be9b6aabb56f4a46683ba377Ben Skeggs if (get == -EINVAL) 1519a391ad8a2cdd7e5be9b6aabb56f4a46683ba377Ben Skeggs continue; 1529a391ad8a2cdd7e5be9b6aabb56f4a46683ba377Ben Skeggs 1539a391ad8a2cdd7e5be9b6aabb56f4a46683ba377Ben Skeggs return get; 1549a391ad8a2cdd7e5be9b6aabb56f4a46683ba377Ben Skeggs } 1559a391ad8a2cdd7e5be9b6aabb56f4a46683ba377Ben Skeggs 1569a391ad8a2cdd7e5be9b6aabb56f4a46683ba377Ben Skeggs if (get <= chan->dma.cur) { 1579a391ad8a2cdd7e5be9b6aabb56f4a46683ba377Ben Skeggs chan->dma.free = chan->dma.max - chan->dma.cur; 1589a391ad8a2cdd7e5be9b6aabb56f4a46683ba377Ben Skeggs if (chan->dma.free >= count) 1599a391ad8a2cdd7e5be9b6aabb56f4a46683ba377Ben Skeggs break; 1609a391ad8a2cdd7e5be9b6aabb56f4a46683ba377Ben Skeggs 1619a391ad8a2cdd7e5be9b6aabb56f4a46683ba377Ben Skeggs FIRE_RING(chan); 1629a391ad8a2cdd7e5be9b6aabb56f4a46683ba377Ben Skeggs do { 1639a391ad8a2cdd7e5be9b6aabb56f4a46683ba377Ben Skeggs get = READ_GET(chan, &prev_get, &cnt); 1649a391ad8a2cdd7e5be9b6aabb56f4a46683ba377Ben Skeggs if (unlikely(get < 0)) { 1659a391ad8a2cdd7e5be9b6aabb56f4a46683ba377Ben Skeggs if (get == -EINVAL) 1669a391ad8a2cdd7e5be9b6aabb56f4a46683ba377Ben Skeggs continue; 1679a391ad8a2cdd7e5be9b6aabb56f4a46683ba377Ben Skeggs return get; 1689a391ad8a2cdd7e5be9b6aabb56f4a46683ba377Ben Skeggs } 1699a391ad8a2cdd7e5be9b6aabb56f4a46683ba377Ben Skeggs } while (get == 0); 1709a391ad8a2cdd7e5be9b6aabb56f4a46683ba377Ben Skeggs chan->dma.cur = 0; 1719a391ad8a2cdd7e5be9b6aabb56f4a46683ba377Ben Skeggs chan->dma.put = 0; 1729a391ad8a2cdd7e5be9b6aabb56f4a46683ba377Ben Skeggs } 1739a391ad8a2cdd7e5be9b6aabb56f4a46683ba377Ben Skeggs 1749a391ad8a2cdd7e5be9b6aabb56f4a46683ba377Ben Skeggs chan->dma.free = get - chan->dma.cur - 1; 1759a391ad8a2cdd7e5be9b6aabb56f4a46683ba377Ben Skeggs } 1769a391ad8a2cdd7e5be9b6aabb56f4a46683ba377Ben Skeggs 1779a391ad8a2cdd7e5be9b6aabb56f4a46683ba377Ben Skeggs return 0; 1789a391ad8a2cdd7e5be9b6aabb56f4a46683ba377Ben Skeggs} 1799a391ad8a2cdd7e5be9b6aabb56f4a46683ba377Ben Skeggs 1806ee738610f41b59733f63718f0bdbcba7d3a3f12Ben Skeggsint 1819a391ad8a2cdd7e5be9b6aabb56f4a46683ba377Ben Skeggsnouveau_dma_wait(struct nouveau_channel *chan, int slots, int size) 1826ee738610f41b59733f63718f0bdbcba7d3a3f12Ben Skeggs{ 1834e03b4af6dd3cff445fc0455805b43b101647bfcFrancisco Jerez uint64_t prev_get = 0; 1844e03b4af6dd3cff445fc0455805b43b101647bfcFrancisco Jerez int cnt = 0, get; 1856ee738610f41b59733f63718f0bdbcba7d3a3f12Ben Skeggs 1869a391ad8a2cdd7e5be9b6aabb56f4a46683ba377Ben Skeggs if (chan->dma.ib_max) 1879a391ad8a2cdd7e5be9b6aabb56f4a46683ba377Ben Skeggs return nv50_dma_wait(chan, slots, size); 1889a391ad8a2cdd7e5be9b6aabb56f4a46683ba377Ben Skeggs 1896ee738610f41b59733f63718f0bdbcba7d3a3f12Ben Skeggs while (chan->dma.free < size) { 190ba59953d281747b1f7518a60f0ba8ff671cd0d65Ben Skeggs get = READ_GET(chan, &prev_get, &cnt); 191ba59953d281747b1f7518a60f0ba8ff671cd0d65Ben Skeggs if (unlikely(get == -EBUSY)) 192ba59953d281747b1f7518a60f0ba8ff671cd0d65Ben Skeggs return -EBUSY; 1936ee738610f41b59733f63718f0bdbcba7d3a3f12Ben Skeggs 1946ee738610f41b59733f63718f0bdbcba7d3a3f12Ben Skeggs /* loop until we have a usable GET pointer. the value 1956ee738610f41b59733f63718f0bdbcba7d3a3f12Ben Skeggs * we read from the GPU may be outside the main ring if 1966ee738610f41b59733f63718f0bdbcba7d3a3f12Ben Skeggs * PFIFO is processing a buffer called from the main ring, 1976ee738610f41b59733f63718f0bdbcba7d3a3f12Ben Skeggs * discard these values until something sensible is seen. 1986ee738610f41b59733f63718f0bdbcba7d3a3f12Ben Skeggs * 1996ee738610f41b59733f63718f0bdbcba7d3a3f12Ben Skeggs * the other case we discard GET is while the GPU is fetching 2006ee738610f41b59733f63718f0bdbcba7d3a3f12Ben Skeggs * from the SKIPS area, so the code below doesn't have to deal 2016ee738610f41b59733f63718f0bdbcba7d3a3f12Ben Skeggs * with some fun corner cases. 2026ee738610f41b59733f63718f0bdbcba7d3a3f12Ben Skeggs */ 203ba59953d281747b1f7518a60f0ba8ff671cd0d65Ben Skeggs if (unlikely(get == -EINVAL) || get < NOUVEAU_DMA_SKIPS) 2046ee738610f41b59733f63718f0bdbcba7d3a3f12Ben Skeggs continue; 2056ee738610f41b59733f63718f0bdbcba7d3a3f12Ben Skeggs 2066ee738610f41b59733f63718f0bdbcba7d3a3f12Ben Skeggs if (get <= chan->dma.cur) { 2076ee738610f41b59733f63718f0bdbcba7d3a3f12Ben Skeggs /* engine is fetching behind us, or is completely 2086ee738610f41b59733f63718f0bdbcba7d3a3f12Ben Skeggs * idle (GET == PUT) so we have free space up until 2096ee738610f41b59733f63718f0bdbcba7d3a3f12Ben Skeggs * the end of the push buffer 2106ee738610f41b59733f63718f0bdbcba7d3a3f12Ben Skeggs * 2116ee738610f41b59733f63718f0bdbcba7d3a3f12Ben Skeggs * we can only hit that path once per call due to 2126ee738610f41b59733f63718f0bdbcba7d3a3f12Ben Skeggs * looping back to the beginning of the push buffer, 2136ee738610f41b59733f63718f0bdbcba7d3a3f12Ben Skeggs * we'll hit the fetching-ahead-of-us path from that 2146ee738610f41b59733f63718f0bdbcba7d3a3f12Ben Skeggs * point on. 2156ee738610f41b59733f63718f0bdbcba7d3a3f12Ben Skeggs * 2166ee738610f41b59733f63718f0bdbcba7d3a3f12Ben Skeggs * the *one* exception to that rule is if we read 2176ee738610f41b59733f63718f0bdbcba7d3a3f12Ben Skeggs * GET==PUT, in which case the below conditional will 2186ee738610f41b59733f63718f0bdbcba7d3a3f12Ben Skeggs * always succeed and break us out of the wait loop. 2196ee738610f41b59733f63718f0bdbcba7d3a3f12Ben Skeggs */ 2206ee738610f41b59733f63718f0bdbcba7d3a3f12Ben Skeggs chan->dma.free = chan->dma.max - chan->dma.cur; 2216ee738610f41b59733f63718f0bdbcba7d3a3f12Ben Skeggs if (chan->dma.free >= size) 2226ee738610f41b59733f63718f0bdbcba7d3a3f12Ben Skeggs break; 2236ee738610f41b59733f63718f0bdbcba7d3a3f12Ben Skeggs 2246ee738610f41b59733f63718f0bdbcba7d3a3f12Ben Skeggs /* not enough space left at the end of the push buffer, 2256ee738610f41b59733f63718f0bdbcba7d3a3f12Ben Skeggs * instruct the GPU to jump back to the start right 2266ee738610f41b59733f63718f0bdbcba7d3a3f12Ben Skeggs * after processing the currently pending commands. 2276ee738610f41b59733f63718f0bdbcba7d3a3f12Ben Skeggs */ 228ebb945a94bba2ce8dff7b0942ff2b3f2a52a0a69Ben Skeggs OUT_RING(chan, chan->push.vma.offset | 0x20000000); 229ba59953d281747b1f7518a60f0ba8ff671cd0d65Ben Skeggs 230ba59953d281747b1f7518a60f0ba8ff671cd0d65Ben Skeggs /* wait for GET to depart from the skips area. 231ba59953d281747b1f7518a60f0ba8ff671cd0d65Ben Skeggs * prevents writing GET==PUT and causing a race 232ba59953d281747b1f7518a60f0ba8ff671cd0d65Ben Skeggs * condition that causes us to think the GPU is 233ba59953d281747b1f7518a60f0ba8ff671cd0d65Ben Skeggs * idle when it's not. 234ba59953d281747b1f7518a60f0ba8ff671cd0d65Ben Skeggs */ 235ba59953d281747b1f7518a60f0ba8ff671cd0d65Ben Skeggs do { 236ba59953d281747b1f7518a60f0ba8ff671cd0d65Ben Skeggs get = READ_GET(chan, &prev_get, &cnt); 237ba59953d281747b1f7518a60f0ba8ff671cd0d65Ben Skeggs if (unlikely(get == -EBUSY)) 238ba59953d281747b1f7518a60f0ba8ff671cd0d65Ben Skeggs return -EBUSY; 239ba59953d281747b1f7518a60f0ba8ff671cd0d65Ben Skeggs if (unlikely(get == -EINVAL)) 240ba59953d281747b1f7518a60f0ba8ff671cd0d65Ben Skeggs continue; 241ba59953d281747b1f7518a60f0ba8ff671cd0d65Ben Skeggs } while (get <= NOUVEAU_DMA_SKIPS); 2426ee738610f41b59733f63718f0bdbcba7d3a3f12Ben Skeggs WRITE_PUT(NOUVEAU_DMA_SKIPS); 2436ee738610f41b59733f63718f0bdbcba7d3a3f12Ben Skeggs 2446ee738610f41b59733f63718f0bdbcba7d3a3f12Ben Skeggs /* we're now submitting commands at the start of 2456ee738610f41b59733f63718f0bdbcba7d3a3f12Ben Skeggs * the push buffer. 2466ee738610f41b59733f63718f0bdbcba7d3a3f12Ben Skeggs */ 2476ee738610f41b59733f63718f0bdbcba7d3a3f12Ben Skeggs chan->dma.cur = 2486ee738610f41b59733f63718f0bdbcba7d3a3f12Ben Skeggs chan->dma.put = NOUVEAU_DMA_SKIPS; 2496ee738610f41b59733f63718f0bdbcba7d3a3f12Ben Skeggs } 2506ee738610f41b59733f63718f0bdbcba7d3a3f12Ben Skeggs 2516ee738610f41b59733f63718f0bdbcba7d3a3f12Ben Skeggs /* engine fetching ahead of us, we have space up until the 2526ee738610f41b59733f63718f0bdbcba7d3a3f12Ben Skeggs * current GET pointer. the "- 1" is to ensure there's 2536ee738610f41b59733f63718f0bdbcba7d3a3f12Ben Skeggs * space left to emit a jump back to the beginning of the 2546ee738610f41b59733f63718f0bdbcba7d3a3f12Ben Skeggs * push buffer if we require it. we can never get GET == PUT 2556ee738610f41b59733f63718f0bdbcba7d3a3f12Ben Skeggs * here, so this is safe. 2566ee738610f41b59733f63718f0bdbcba7d3a3f12Ben Skeggs */ 2576ee738610f41b59733f63718f0bdbcba7d3a3f12Ben Skeggs chan->dma.free = get - chan->dma.cur - 1; 2586ee738610f41b59733f63718f0bdbcba7d3a3f12Ben Skeggs } 2596ee738610f41b59733f63718f0bdbcba7d3a3f12Ben Skeggs 2606ee738610f41b59733f63718f0bdbcba7d3a3f12Ben Skeggs return 0; 2616ee738610f41b59733f63718f0bdbcba7d3a3f12Ben Skeggs} 2626ee738610f41b59733f63718f0bdbcba7d3a3f12Ben Skeggs 263