[go: nahoru, domu]
1/*
2 * Copyright 2008 Advanced Micro Devices, Inc.
3 * Copyright 2008 Red Hat Inc.
4 * Copyright 2009 Jerome Glisse.
5 *
6 * Permission is hereby granted, free of charge, to any person obtaining a
7 * copy of this software and associated documentation files (the "Software"),
8 * to deal in the Software without restriction, including without limitation
9 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
10 * and/or sell copies of the Software, and to permit persons to whom the
11 * Software is furnished to do so, subject to the following conditions:
12 *
13 * The above copyright notice and this permission notice shall be included in
14 * all copies or substantial portions of the Software.
15 *
16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
19 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
20 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
21 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
22 * OTHER DEALINGS IN THE SOFTWARE.
23 *
24 * Authors: Dave Airlie
25 *          Alex Deucher
26 *          Jerome Glisse
27 */
28#include <linux/firmware.h>
29#include <linux/slab.h>
30#include <drm/drmP.h>
31#include "radeon.h"
32#include "radeon_asic.h"
33#include <drm/radeon_drm.h>
34#include "rv770d.h"
35#include "atom.h"
36#include "avivod.h"
37
38#define R700_PFP_UCODE_SIZE 848
39#define R700_PM4_UCODE_SIZE 1360
40
41static void rv770_gpu_init(struct radeon_device *rdev);
42void rv770_fini(struct radeon_device *rdev);
43static void rv770_pcie_gen2_enable(struct radeon_device *rdev);
44int evergreen_set_uvd_clocks(struct radeon_device *rdev, u32 vclk, u32 dclk);
45
46int rv770_set_uvd_clocks(struct radeon_device *rdev, u32 vclk, u32 dclk)
47{
48	unsigned fb_div = 0, vclk_div = 0, dclk_div = 0;
49	int r;
50
51	/* RV740 uses evergreen uvd clk programming */
52	if (rdev->family == CHIP_RV740)
53		return evergreen_set_uvd_clocks(rdev, vclk, dclk);
54
55	/* bypass vclk and dclk with bclk */
56	WREG32_P(CG_UPLL_FUNC_CNTL_2,
57		 VCLK_SRC_SEL(1) | DCLK_SRC_SEL(1),
58		 ~(VCLK_SRC_SEL_MASK | DCLK_SRC_SEL_MASK));
59
60	if (!vclk || !dclk) {
61		/* keep the Bypass mode, put PLL to sleep */
62		WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_SLEEP_MASK, ~UPLL_SLEEP_MASK);
63		return 0;
64	}
65
66	r = radeon_uvd_calc_upll_dividers(rdev, vclk, dclk, 50000, 160000,
67					  43663, 0x03FFFFFE, 1, 30, ~0,
68					  &fb_div, &vclk_div, &dclk_div);
69	if (r)
70		return r;
71
72	fb_div |= 1;
73	vclk_div -= 1;
74	dclk_div -= 1;
75
76	/* set UPLL_FB_DIV to 0x50000 */
77	WREG32_P(CG_UPLL_FUNC_CNTL_3, UPLL_FB_DIV(0x50000), ~UPLL_FB_DIV_MASK);
78
79	/* deassert UPLL_RESET and UPLL_SLEEP */
80	WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~(UPLL_RESET_MASK | UPLL_SLEEP_MASK));
81
82	/* assert BYPASS EN and FB_DIV[0] <- ??? why? */
83	WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_BYPASS_EN_MASK, ~UPLL_BYPASS_EN_MASK);
84	WREG32_P(CG_UPLL_FUNC_CNTL_3, UPLL_FB_DIV(1), ~UPLL_FB_DIV(1));
85
86	r = radeon_uvd_send_upll_ctlreq(rdev, CG_UPLL_FUNC_CNTL);
87	if (r)
88		return r;
89
90	/* assert PLL_RESET */
91	WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_RESET_MASK, ~UPLL_RESET_MASK);
92
93	/* set the required FB_DIV, REF_DIV, Post divder values */
94	WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_REF_DIV(1), ~UPLL_REF_DIV_MASK);
95	WREG32_P(CG_UPLL_FUNC_CNTL_2,
96		 UPLL_SW_HILEN(vclk_div >> 1) |
97		 UPLL_SW_LOLEN((vclk_div >> 1) + (vclk_div & 1)) |
98		 UPLL_SW_HILEN2(dclk_div >> 1) |
99		 UPLL_SW_LOLEN2((dclk_div >> 1) + (dclk_div & 1)),
100		 ~UPLL_SW_MASK);
101
102	WREG32_P(CG_UPLL_FUNC_CNTL_3, UPLL_FB_DIV(fb_div),
103		 ~UPLL_FB_DIV_MASK);
104
105	/* give the PLL some time to settle */
106	mdelay(15);
107
108	/* deassert PLL_RESET */
109	WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_RESET_MASK);
110
111	mdelay(15);
112
113	/* deassert BYPASS EN and FB_DIV[0] <- ??? why? */
114	WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_BYPASS_EN_MASK);
115	WREG32_P(CG_UPLL_FUNC_CNTL_3, 0, ~UPLL_FB_DIV(1));
116
117	r = radeon_uvd_send_upll_ctlreq(rdev, CG_UPLL_FUNC_CNTL);
118	if (r)
119		return r;
120
121	/* switch VCLK and DCLK selection */
122	WREG32_P(CG_UPLL_FUNC_CNTL_2,
123		 VCLK_SRC_SEL(2) | DCLK_SRC_SEL(2),
124		 ~(VCLK_SRC_SEL_MASK | DCLK_SRC_SEL_MASK));
125
126	mdelay(100);
127
128	return 0;
129}
130
131static const u32 r7xx_golden_registers[] =
132{
133	0x8d00, 0xffffffff, 0x0e0e0074,
134	0x8d04, 0xffffffff, 0x013a2b34,
135	0x9508, 0xffffffff, 0x00000002,
136	0x8b20, 0xffffffff, 0,
137	0x88c4, 0xffffffff, 0x000000c2,
138	0x28350, 0xffffffff, 0,
139	0x9058, 0xffffffff, 0x0fffc40f,
140	0x240c, 0xffffffff, 0x00000380,
141	0x733c, 0xffffffff, 0x00000002,
142	0x2650, 0x00040000, 0,
143	0x20bc, 0x00040000, 0,
144	0x7300, 0xffffffff, 0x001000f0
145};
146
147static const u32 r7xx_golden_dyn_gpr_registers[] =
148{
149	0x8db0, 0xffffffff, 0x98989898,
150	0x8db4, 0xffffffff, 0x98989898,
151	0x8db8, 0xffffffff, 0x98989898,
152	0x8dbc, 0xffffffff, 0x98989898,
153	0x8dc0, 0xffffffff, 0x98989898,
154	0x8dc4, 0xffffffff, 0x98989898,
155	0x8dc8, 0xffffffff, 0x98989898,
156	0x8dcc, 0xffffffff, 0x98989898,
157	0x88c4, 0xffffffff, 0x00000082
158};
159
160static const u32 rv770_golden_registers[] =
161{
162	0x562c, 0xffffffff, 0,
163	0x3f90, 0xffffffff, 0,
164	0x9148, 0xffffffff, 0,
165	0x3f94, 0xffffffff, 0,
166	0x914c, 0xffffffff, 0,
167	0x9698, 0x18000000, 0x18000000
168};
169
170static const u32 rv770ce_golden_registers[] =
171{
172	0x562c, 0xffffffff, 0,
173	0x3f90, 0xffffffff, 0x00cc0000,
174	0x9148, 0xffffffff, 0x00cc0000,
175	0x3f94, 0xffffffff, 0x00cc0000,
176	0x914c, 0xffffffff, 0x00cc0000,
177	0x9b7c, 0xffffffff, 0x00fa0000,
178	0x3f8c, 0xffffffff, 0x00fa0000,
179	0x9698, 0x18000000, 0x18000000
180};
181
182static const u32 rv770_mgcg_init[] =
183{
184	0x8bcc, 0xffffffff, 0x130300f9,
185	0x5448, 0xffffffff, 0x100,
186	0x55e4, 0xffffffff, 0x100,
187	0x160c, 0xffffffff, 0x100,
188	0x5644, 0xffffffff, 0x100,
189	0xc164, 0xffffffff, 0x100,
190	0x8a18, 0xffffffff, 0x100,
191	0x897c, 0xffffffff, 0x8000100,
192	0x8b28, 0xffffffff, 0x3c000100,
193	0x9144, 0xffffffff, 0x100,
194	0x9a1c, 0xffffffff, 0x10000,
195	0x9a50, 0xffffffff, 0x100,
196	0x9a1c, 0xffffffff, 0x10001,
197	0x9a50, 0xffffffff, 0x100,
198	0x9a1c, 0xffffffff, 0x10002,
199	0x9a50, 0xffffffff, 0x100,
200	0x9a1c, 0xffffffff, 0x10003,
201	0x9a50, 0xffffffff, 0x100,
202	0x9a1c, 0xffffffff, 0x0,
203	0x9870, 0xffffffff, 0x100,
204	0x8d58, 0xffffffff, 0x100,
205	0x9500, 0xffffffff, 0x0,
206	0x9510, 0xffffffff, 0x100,
207	0x9500, 0xffffffff, 0x1,
208	0x9510, 0xffffffff, 0x100,
209	0x9500, 0xffffffff, 0x2,
210	0x9510, 0xffffffff, 0x100,
211	0x9500, 0xffffffff, 0x3,
212	0x9510, 0xffffffff, 0x100,
213	0x9500, 0xffffffff, 0x4,
214	0x9510, 0xffffffff, 0x100,
215	0x9500, 0xffffffff, 0x5,
216	0x9510, 0xffffffff, 0x100,
217	0x9500, 0xffffffff, 0x6,
218	0x9510, 0xffffffff, 0x100,
219	0x9500, 0xffffffff, 0x7,
220	0x9510, 0xffffffff, 0x100,
221	0x9500, 0xffffffff, 0x8,
222	0x9510, 0xffffffff, 0x100,
223	0x9500, 0xffffffff, 0x9,
224	0x9510, 0xffffffff, 0x100,
225	0x9500, 0xffffffff, 0x8000,
226	0x9490, 0xffffffff, 0x0,
227	0x949c, 0xffffffff, 0x100,
228	0x9490, 0xffffffff, 0x1,
229	0x949c, 0xffffffff, 0x100,
230	0x9490, 0xffffffff, 0x2,
231	0x949c, 0xffffffff, 0x100,
232	0x9490, 0xffffffff, 0x3,
233	0x949c, 0xffffffff, 0x100,
234	0x9490, 0xffffffff, 0x4,
235	0x949c, 0xffffffff, 0x100,
236	0x9490, 0xffffffff, 0x5,
237	0x949c, 0xffffffff, 0x100,
238	0x9490, 0xffffffff, 0x6,
239	0x949c, 0xffffffff, 0x100,
240	0x9490, 0xffffffff, 0x7,
241	0x949c, 0xffffffff, 0x100,
242	0x9490, 0xffffffff, 0x8,
243	0x949c, 0xffffffff, 0x100,
244	0x9490, 0xffffffff, 0x9,
245	0x949c, 0xffffffff, 0x100,
246	0x9490, 0xffffffff, 0x8000,
247	0x9604, 0xffffffff, 0x0,
248	0x9654, 0xffffffff, 0x100,
249	0x9604, 0xffffffff, 0x1,
250	0x9654, 0xffffffff, 0x100,
251	0x9604, 0xffffffff, 0x2,
252	0x9654, 0xffffffff, 0x100,
253	0x9604, 0xffffffff, 0x3,
254	0x9654, 0xffffffff, 0x100,
255	0x9604, 0xffffffff, 0x4,
256	0x9654, 0xffffffff, 0x100,
257	0x9604, 0xffffffff, 0x5,
258	0x9654, 0xffffffff, 0x100,
259	0x9604, 0xffffffff, 0x6,
260	0x9654, 0xffffffff, 0x100,
261	0x9604, 0xffffffff, 0x7,
262	0x9654, 0xffffffff, 0x100,
263	0x9604, 0xffffffff, 0x8,
264	0x9654, 0xffffffff, 0x100,
265	0x9604, 0xffffffff, 0x9,
266	0x9654, 0xffffffff, 0x100,
267	0x9604, 0xffffffff, 0x80000000,
268	0x9030, 0xffffffff, 0x100,
269	0x9034, 0xffffffff, 0x100,
270	0x9038, 0xffffffff, 0x100,
271	0x903c, 0xffffffff, 0x100,
272	0x9040, 0xffffffff, 0x100,
273	0xa200, 0xffffffff, 0x100,
274	0xa204, 0xffffffff, 0x100,
275	0xa208, 0xffffffff, 0x100,
276	0xa20c, 0xffffffff, 0x100,
277	0x971c, 0xffffffff, 0x100,
278	0x915c, 0xffffffff, 0x00020001,
279	0x9160, 0xffffffff, 0x00040003,
280	0x916c, 0xffffffff, 0x00060005,
281	0x9170, 0xffffffff, 0x00080007,
282	0x9174, 0xffffffff, 0x000a0009,
283	0x9178, 0xffffffff, 0x000c000b,
284	0x917c, 0xffffffff, 0x000e000d,
285	0x9180, 0xffffffff, 0x0010000f,
286	0x918c, 0xffffffff, 0x00120011,
287	0x9190, 0xffffffff, 0x00140013,
288	0x9194, 0xffffffff, 0x00020001,
289	0x9198, 0xffffffff, 0x00040003,
290	0x919c, 0xffffffff, 0x00060005,
291	0x91a8, 0xffffffff, 0x00080007,
292	0x91ac, 0xffffffff, 0x000a0009,
293	0x91b0, 0xffffffff, 0x000c000b,
294	0x91b4, 0xffffffff, 0x000e000d,
295	0x91b8, 0xffffffff, 0x0010000f,
296	0x91c4, 0xffffffff, 0x00120011,
297	0x91c8, 0xffffffff, 0x00140013,
298	0x91cc, 0xffffffff, 0x00020001,
299	0x91d0, 0xffffffff, 0x00040003,
300	0x91d4, 0xffffffff, 0x00060005,
301	0x91e0, 0xffffffff, 0x00080007,
302	0x91e4, 0xffffffff, 0x000a0009,
303	0x91e8, 0xffffffff, 0x000c000b,
304	0x91ec, 0xffffffff, 0x00020001,
305	0x91f0, 0xffffffff, 0x00040003,
306	0x91f4, 0xffffffff, 0x00060005,
307	0x9200, 0xffffffff, 0x00080007,
308	0x9204, 0xffffffff, 0x000a0009,
309	0x9208, 0xffffffff, 0x000c000b,
310	0x920c, 0xffffffff, 0x000e000d,
311	0x9210, 0xffffffff, 0x0010000f,
312	0x921c, 0xffffffff, 0x00120011,
313	0x9220, 0xffffffff, 0x00140013,
314	0x9224, 0xffffffff, 0x00020001,
315	0x9228, 0xffffffff, 0x00040003,
316	0x922c, 0xffffffff, 0x00060005,
317	0x9238, 0xffffffff, 0x00080007,
318	0x923c, 0xffffffff, 0x000a0009,
319	0x9240, 0xffffffff, 0x000c000b,
320	0x9244, 0xffffffff, 0x000e000d,
321	0x9248, 0xffffffff, 0x0010000f,
322	0x9254, 0xffffffff, 0x00120011,
323	0x9258, 0xffffffff, 0x00140013,
324	0x925c, 0xffffffff, 0x00020001,
325	0x9260, 0xffffffff, 0x00040003,
326	0x9264, 0xffffffff, 0x00060005,
327	0x9270, 0xffffffff, 0x00080007,
328	0x9274, 0xffffffff, 0x000a0009,
329	0x9278, 0xffffffff, 0x000c000b,
330	0x927c, 0xffffffff, 0x000e000d,
331	0x9280, 0xffffffff, 0x0010000f,
332	0x928c, 0xffffffff, 0x00120011,
333	0x9290, 0xffffffff, 0x00140013,
334	0x9294, 0xffffffff, 0x00020001,
335	0x929c, 0xffffffff, 0x00040003,
336	0x92a0, 0xffffffff, 0x00060005,
337	0x92a4, 0xffffffff, 0x00080007
338};
339
340static const u32 rv710_golden_registers[] =
341{
342	0x3f90, 0x00ff0000, 0x00fc0000,
343	0x9148, 0x00ff0000, 0x00fc0000,
344	0x3f94, 0x00ff0000, 0x00fc0000,
345	0x914c, 0x00ff0000, 0x00fc0000,
346	0xb4c, 0x00000020, 0x00000020,
347	0xa180, 0xffffffff, 0x00003f3f
348};
349
350static const u32 rv710_mgcg_init[] =
351{
352	0x8bcc, 0xffffffff, 0x13030040,
353	0x5448, 0xffffffff, 0x100,
354	0x55e4, 0xffffffff, 0x100,
355	0x160c, 0xffffffff, 0x100,
356	0x5644, 0xffffffff, 0x100,
357	0xc164, 0xffffffff, 0x100,
358	0x8a18, 0xffffffff, 0x100,
359	0x897c, 0xffffffff, 0x8000100,
360	0x8b28, 0xffffffff, 0x3c000100,
361	0x9144, 0xffffffff, 0x100,
362	0x9a1c, 0xffffffff, 0x10000,
363	0x9a50, 0xffffffff, 0x100,
364	0x9a1c, 0xffffffff, 0x0,
365	0x9870, 0xffffffff, 0x100,
366	0x8d58, 0xffffffff, 0x100,
367	0x9500, 0xffffffff, 0x0,
368	0x9510, 0xffffffff, 0x100,
369	0x9500, 0xffffffff, 0x1,
370	0x9510, 0xffffffff, 0x100,
371	0x9500, 0xffffffff, 0x8000,
372	0x9490, 0xffffffff, 0x0,
373	0x949c, 0xffffffff, 0x100,
374	0x9490, 0xffffffff, 0x1,
375	0x949c, 0xffffffff, 0x100,
376	0x9490, 0xffffffff, 0x8000,
377	0x9604, 0xffffffff, 0x0,
378	0x9654, 0xffffffff, 0x100,
379	0x9604, 0xffffffff, 0x1,
380	0x9654, 0xffffffff, 0x100,
381	0x9604, 0xffffffff, 0x80000000,
382	0x9030, 0xffffffff, 0x100,
383	0x9034, 0xffffffff, 0x100,
384	0x9038, 0xffffffff, 0x100,
385	0x903c, 0xffffffff, 0x100,
386	0x9040, 0xffffffff, 0x100,
387	0xa200, 0xffffffff, 0x100,
388	0xa204, 0xffffffff, 0x100,
389	0xa208, 0xffffffff, 0x100,
390	0xa20c, 0xffffffff, 0x100,
391	0x971c, 0xffffffff, 0x100,
392	0x915c, 0xffffffff, 0x00020001,
393	0x9174, 0xffffffff, 0x00000003,
394	0x9178, 0xffffffff, 0x00050001,
395	0x917c, 0xffffffff, 0x00030002,
396	0x918c, 0xffffffff, 0x00000004,
397	0x9190, 0xffffffff, 0x00070006,
398	0x9194, 0xffffffff, 0x00050001,
399	0x9198, 0xffffffff, 0x00030002,
400	0x91a8, 0xffffffff, 0x00000004,
401	0x91ac, 0xffffffff, 0x00070006,
402	0x91e8, 0xffffffff, 0x00000001,
403	0x9294, 0xffffffff, 0x00000001,
404	0x929c, 0xffffffff, 0x00000002,
405	0x92a0, 0xffffffff, 0x00040003,
406	0x9150, 0xffffffff, 0x4d940000
407};
408
409static const u32 rv730_golden_registers[] =
410{
411	0x3f90, 0x00ff0000, 0x00f00000,
412	0x9148, 0x00ff0000, 0x00f00000,
413	0x3f94, 0x00ff0000, 0x00f00000,
414	0x914c, 0x00ff0000, 0x00f00000,
415	0x900c, 0xffffffff, 0x003b033f,
416	0xb4c, 0x00000020, 0x00000020,
417	0xa180, 0xffffffff, 0x00003f3f
418};
419
420static const u32 rv730_mgcg_init[] =
421{
422	0x8bcc, 0xffffffff, 0x130300f9,
423	0x5448, 0xffffffff, 0x100,
424	0x55e4, 0xffffffff, 0x100,
425	0x160c, 0xffffffff, 0x100,
426	0x5644, 0xffffffff, 0x100,
427	0xc164, 0xffffffff, 0x100,
428	0x8a18, 0xffffffff, 0x100,
429	0x897c, 0xffffffff, 0x8000100,
430	0x8b28, 0xffffffff, 0x3c000100,
431	0x9144, 0xffffffff, 0x100,
432	0x9a1c, 0xffffffff, 0x10000,
433	0x9a50, 0xffffffff, 0x100,
434	0x9a1c, 0xffffffff, 0x10001,
435	0x9a50, 0xffffffff, 0x100,
436	0x9a1c, 0xffffffff, 0x0,
437	0x9870, 0xffffffff, 0x100,
438	0x8d58, 0xffffffff, 0x100,
439	0x9500, 0xffffffff, 0x0,
440	0x9510, 0xffffffff, 0x100,
441	0x9500, 0xffffffff, 0x1,
442	0x9510, 0xffffffff, 0x100,
443	0x9500, 0xffffffff, 0x2,
444	0x9510, 0xffffffff, 0x100,
445	0x9500, 0xffffffff, 0x3,
446	0x9510, 0xffffffff, 0x100,
447	0x9500, 0xffffffff, 0x4,
448	0x9510, 0xffffffff, 0x100,
449	0x9500, 0xffffffff, 0x5,
450	0x9510, 0xffffffff, 0x100,
451	0x9500, 0xffffffff, 0x6,
452	0x9510, 0xffffffff, 0x100,
453	0x9500, 0xffffffff, 0x7,
454	0x9510, 0xffffffff, 0x100,
455	0x9500, 0xffffffff, 0x8000,
456	0x9490, 0xffffffff, 0x0,
457	0x949c, 0xffffffff, 0x100,
458	0x9490, 0xffffffff, 0x1,
459	0x949c, 0xffffffff, 0x100,
460	0x9490, 0xffffffff, 0x2,
461	0x949c, 0xffffffff, 0x100,
462	0x9490, 0xffffffff, 0x3,
463	0x949c, 0xffffffff, 0x100,
464	0x9490, 0xffffffff, 0x4,
465	0x949c, 0xffffffff, 0x100,
466	0x9490, 0xffffffff, 0x5,
467	0x949c, 0xffffffff, 0x100,
468	0x9490, 0xffffffff, 0x6,
469	0x949c, 0xffffffff, 0x100,
470	0x9490, 0xffffffff, 0x7,
471	0x949c, 0xffffffff, 0x100,
472	0x9490, 0xffffffff, 0x8000,
473	0x9604, 0xffffffff, 0x0,
474	0x9654, 0xffffffff, 0x100,
475	0x9604, 0xffffffff, 0x1,
476	0x9654, 0xffffffff, 0x100,
477	0x9604, 0xffffffff, 0x2,
478	0x9654, 0xffffffff, 0x100,
479	0x9604, 0xffffffff, 0x3,
480	0x9654, 0xffffffff, 0x100,
481	0x9604, 0xffffffff, 0x4,
482	0x9654, 0xffffffff, 0x100,
483	0x9604, 0xffffffff, 0x5,
484	0x9654, 0xffffffff, 0x100,
485	0x9604, 0xffffffff, 0x6,
486	0x9654, 0xffffffff, 0x100,
487	0x9604, 0xffffffff, 0x7,
488	0x9654, 0xffffffff, 0x100,
489	0x9604, 0xffffffff, 0x80000000,
490	0x9030, 0xffffffff, 0x100,
491	0x9034, 0xffffffff, 0x100,
492	0x9038, 0xffffffff, 0x100,
493	0x903c, 0xffffffff, 0x100,
494	0x9040, 0xffffffff, 0x100,
495	0xa200, 0xffffffff, 0x100,
496	0xa204, 0xffffffff, 0x100,
497	0xa208, 0xffffffff, 0x100,
498	0xa20c, 0xffffffff, 0x100,
499	0x971c, 0xffffffff, 0x100,
500	0x915c, 0xffffffff, 0x00020001,
501	0x916c, 0xffffffff, 0x00040003,
502	0x9170, 0xffffffff, 0x00000005,
503	0x9178, 0xffffffff, 0x00050001,
504	0x917c, 0xffffffff, 0x00030002,
505	0x918c, 0xffffffff, 0x00000004,
506	0x9190, 0xffffffff, 0x00070006,
507	0x9194, 0xffffffff, 0x00050001,
508	0x9198, 0xffffffff, 0x00030002,
509	0x91a8, 0xffffffff, 0x00000004,
510	0x91ac, 0xffffffff, 0x00070006,
511	0x91b0, 0xffffffff, 0x00050001,
512	0x91b4, 0xffffffff, 0x00030002,
513	0x91c4, 0xffffffff, 0x00000004,
514	0x91c8, 0xffffffff, 0x00070006,
515	0x91cc, 0xffffffff, 0x00050001,
516	0x91d0, 0xffffffff, 0x00030002,
517	0x91e0, 0xffffffff, 0x00000004,
518	0x91e4, 0xffffffff, 0x00070006,
519	0x91e8, 0xffffffff, 0x00000001,
520	0x91ec, 0xffffffff, 0x00050001,
521	0x91f0, 0xffffffff, 0x00030002,
522	0x9200, 0xffffffff, 0x00000004,
523	0x9204, 0xffffffff, 0x00070006,
524	0x9208, 0xffffffff, 0x00050001,
525	0x920c, 0xffffffff, 0x00030002,
526	0x921c, 0xffffffff, 0x00000004,
527	0x9220, 0xffffffff, 0x00070006,
528	0x9224, 0xffffffff, 0x00050001,
529	0x9228, 0xffffffff, 0x00030002,
530	0x9238, 0xffffffff, 0x00000004,
531	0x923c, 0xffffffff, 0x00070006,
532	0x9240, 0xffffffff, 0x00050001,
533	0x9244, 0xffffffff, 0x00030002,
534	0x9254, 0xffffffff, 0x00000004,
535	0x9258, 0xffffffff, 0x00070006,
536	0x9294, 0xffffffff, 0x00000001,
537	0x929c, 0xffffffff, 0x00000002,
538	0x92a0, 0xffffffff, 0x00040003,
539	0x92a4, 0xffffffff, 0x00000005
540};
541
542static const u32 rv740_golden_registers[] =
543{
544	0x88c4, 0xffffffff, 0x00000082,
545	0x28a50, 0xfffffffc, 0x00000004,
546	0x2650, 0x00040000, 0,
547	0x20bc, 0x00040000, 0,
548	0x733c, 0xffffffff, 0x00000002,
549	0x7300, 0xffffffff, 0x001000f0,
550	0x3f90, 0x00ff0000, 0,
551	0x9148, 0x00ff0000, 0,
552	0x3f94, 0x00ff0000, 0,
553	0x914c, 0x00ff0000, 0,
554	0x240c, 0xffffffff, 0x00000380,
555	0x8a14, 0x00000007, 0x00000007,
556	0x8b24, 0xffffffff, 0x00ff0fff,
557	0x28a4c, 0xffffffff, 0x00004000,
558	0xa180, 0xffffffff, 0x00003f3f,
559	0x8d00, 0xffffffff, 0x0e0e003a,
560	0x8d04, 0xffffffff, 0x013a0e2a,
561	0x8c00, 0xffffffff, 0xe400000f,
562	0x8db0, 0xffffffff, 0x98989898,
563	0x8db4, 0xffffffff, 0x98989898,
564	0x8db8, 0xffffffff, 0x98989898,
565	0x8dbc, 0xffffffff, 0x98989898,
566	0x8dc0, 0xffffffff, 0x98989898,
567	0x8dc4, 0xffffffff, 0x98989898,
568	0x8dc8, 0xffffffff, 0x98989898,
569	0x8dcc, 0xffffffff, 0x98989898,
570	0x9058, 0xffffffff, 0x0fffc40f,
571	0x900c, 0xffffffff, 0x003b033f,
572	0x28350, 0xffffffff, 0,
573	0x8cf0, 0x1fffffff, 0x08e00420,
574	0x9508, 0xffffffff, 0x00000002,
575	0x88c4, 0xffffffff, 0x000000c2,
576	0x9698, 0x18000000, 0x18000000
577};
578
579static const u32 rv740_mgcg_init[] =
580{
581	0x8bcc, 0xffffffff, 0x13030100,
582	0x5448, 0xffffffff, 0x100,
583	0x55e4, 0xffffffff, 0x100,
584	0x160c, 0xffffffff, 0x100,
585	0x5644, 0xffffffff, 0x100,
586	0xc164, 0xffffffff, 0x100,
587	0x8a18, 0xffffffff, 0x100,
588	0x897c, 0xffffffff, 0x100,
589	0x8b28, 0xffffffff, 0x100,
590	0x9144, 0xffffffff, 0x100,
591	0x9a1c, 0xffffffff, 0x10000,
592	0x9a50, 0xffffffff, 0x100,
593	0x9a1c, 0xffffffff, 0x10001,
594	0x9a50, 0xffffffff, 0x100,
595	0x9a1c, 0xffffffff, 0x10002,
596	0x9a50, 0xffffffff, 0x100,
597	0x9a1c, 0xffffffff, 0x10003,
598	0x9a50, 0xffffffff, 0x100,
599	0x9a1c, 0xffffffff, 0x0,
600	0x9870, 0xffffffff, 0x100,
601	0x8d58, 0xffffffff, 0x100,
602	0x9500, 0xffffffff, 0x0,
603	0x9510, 0xffffffff, 0x100,
604	0x9500, 0xffffffff, 0x1,
605	0x9510, 0xffffffff, 0x100,
606	0x9500, 0xffffffff, 0x2,
607	0x9510, 0xffffffff, 0x100,
608	0x9500, 0xffffffff, 0x3,
609	0x9510, 0xffffffff, 0x100,
610	0x9500, 0xffffffff, 0x4,
611	0x9510, 0xffffffff, 0x100,
612	0x9500, 0xffffffff, 0x5,
613	0x9510, 0xffffffff, 0x100,
614	0x9500, 0xffffffff, 0x6,
615	0x9510, 0xffffffff, 0x100,
616	0x9500, 0xffffffff, 0x7,
617	0x9510, 0xffffffff, 0x100,
618	0x9500, 0xffffffff, 0x8000,
619	0x9490, 0xffffffff, 0x0,
620	0x949c, 0xffffffff, 0x100,
621	0x9490, 0xffffffff, 0x1,
622	0x949c, 0xffffffff, 0x100,
623	0x9490, 0xffffffff, 0x2,
624	0x949c, 0xffffffff, 0x100,
625	0x9490, 0xffffffff, 0x3,
626	0x949c, 0xffffffff, 0x100,
627	0x9490, 0xffffffff, 0x4,
628	0x949c, 0xffffffff, 0x100,
629	0x9490, 0xffffffff, 0x5,
630	0x949c, 0xffffffff, 0x100,
631	0x9490, 0xffffffff, 0x6,
632	0x949c, 0xffffffff, 0x100,
633	0x9490, 0xffffffff, 0x7,
634	0x949c, 0xffffffff, 0x100,
635	0x9490, 0xffffffff, 0x8000,
636	0x9604, 0xffffffff, 0x0,
637	0x9654, 0xffffffff, 0x100,
638	0x9604, 0xffffffff, 0x1,
639	0x9654, 0xffffffff, 0x100,
640	0x9604, 0xffffffff, 0x2,
641	0x9654, 0xffffffff, 0x100,
642	0x9604, 0xffffffff, 0x3,
643	0x9654, 0xffffffff, 0x100,
644	0x9604, 0xffffffff, 0x4,
645	0x9654, 0xffffffff, 0x100,
646	0x9604, 0xffffffff, 0x5,
647	0x9654, 0xffffffff, 0x100,
648	0x9604, 0xffffffff, 0x6,
649	0x9654, 0xffffffff, 0x100,
650	0x9604, 0xffffffff, 0x7,
651	0x9654, 0xffffffff, 0x100,
652	0x9604, 0xffffffff, 0x80000000,
653	0x9030, 0xffffffff, 0x100,
654	0x9034, 0xffffffff, 0x100,
655	0x9038, 0xffffffff, 0x100,
656	0x903c, 0xffffffff, 0x100,
657	0x9040, 0xffffffff, 0x100,
658	0xa200, 0xffffffff, 0x100,
659	0xa204, 0xffffffff, 0x100,
660	0xa208, 0xffffffff, 0x100,
661	0xa20c, 0xffffffff, 0x100,
662	0x971c, 0xffffffff, 0x100,
663	0x915c, 0xffffffff, 0x00020001,
664	0x9160, 0xffffffff, 0x00040003,
665	0x916c, 0xffffffff, 0x00060005,
666	0x9170, 0xffffffff, 0x00080007,
667	0x9174, 0xffffffff, 0x000a0009,
668	0x9178, 0xffffffff, 0x000c000b,
669	0x917c, 0xffffffff, 0x000e000d,
670	0x9180, 0xffffffff, 0x0010000f,
671	0x918c, 0xffffffff, 0x00120011,
672	0x9190, 0xffffffff, 0x00140013,
673	0x9194, 0xffffffff, 0x00020001,
674	0x9198, 0xffffffff, 0x00040003,
675	0x919c, 0xffffffff, 0x00060005,
676	0x91a8, 0xffffffff, 0x00080007,
677	0x91ac, 0xffffffff, 0x000a0009,
678	0x91b0, 0xffffffff, 0x000c000b,
679	0x91b4, 0xffffffff, 0x000e000d,
680	0x91b8, 0xffffffff, 0x0010000f,
681	0x91c4, 0xffffffff, 0x00120011,
682	0x91c8, 0xffffffff, 0x00140013,
683	0x91cc, 0xffffffff, 0x00020001,
684	0x91d0, 0xffffffff, 0x00040003,
685	0x91d4, 0xffffffff, 0x00060005,
686	0x91e0, 0xffffffff, 0x00080007,
687	0x91e4, 0xffffffff, 0x000a0009,
688	0x91e8, 0xffffffff, 0x000c000b,
689	0x91ec, 0xffffffff, 0x00020001,
690	0x91f0, 0xffffffff, 0x00040003,
691	0x91f4, 0xffffffff, 0x00060005,
692	0x9200, 0xffffffff, 0x00080007,
693	0x9204, 0xffffffff, 0x000a0009,
694	0x9208, 0xffffffff, 0x000c000b,
695	0x920c, 0xffffffff, 0x000e000d,
696	0x9210, 0xffffffff, 0x0010000f,
697	0x921c, 0xffffffff, 0x00120011,
698	0x9220, 0xffffffff, 0x00140013,
699	0x9224, 0xffffffff, 0x00020001,
700	0x9228, 0xffffffff, 0x00040003,
701	0x922c, 0xffffffff, 0x00060005,
702	0x9238, 0xffffffff, 0x00080007,
703	0x923c, 0xffffffff, 0x000a0009,
704	0x9240, 0xffffffff, 0x000c000b,
705	0x9244, 0xffffffff, 0x000e000d,
706	0x9248, 0xffffffff, 0x0010000f,
707	0x9254, 0xffffffff, 0x00120011,
708	0x9258, 0xffffffff, 0x00140013,
709	0x9294, 0xffffffff, 0x00020001,
710	0x929c, 0xffffffff, 0x00040003,
711	0x92a0, 0xffffffff, 0x00060005,
712	0x92a4, 0xffffffff, 0x00080007
713};
714
715static void rv770_init_golden_registers(struct radeon_device *rdev)
716{
717	switch (rdev->family) {
718	case CHIP_RV770:
719		radeon_program_register_sequence(rdev,
720						 r7xx_golden_registers,
721						 (const u32)ARRAY_SIZE(r7xx_golden_registers));
722		radeon_program_register_sequence(rdev,
723						 r7xx_golden_dyn_gpr_registers,
724						 (const u32)ARRAY_SIZE(r7xx_golden_dyn_gpr_registers));
725		if (rdev->pdev->device == 0x994e)
726			radeon_program_register_sequence(rdev,
727							 rv770ce_golden_registers,
728							 (const u32)ARRAY_SIZE(rv770ce_golden_registers));
729		else
730			radeon_program_register_sequence(rdev,
731							 rv770_golden_registers,
732							 (const u32)ARRAY_SIZE(rv770_golden_registers));
733		radeon_program_register_sequence(rdev,
734						 rv770_mgcg_init,
735						 (const u32)ARRAY_SIZE(rv770_mgcg_init));
736		break;
737	case CHIP_RV730:
738		radeon_program_register_sequence(rdev,
739						 r7xx_golden_registers,
740						 (const u32)ARRAY_SIZE(r7xx_golden_registers));
741		radeon_program_register_sequence(rdev,
742						 r7xx_golden_dyn_gpr_registers,
743						 (const u32)ARRAY_SIZE(r7xx_golden_dyn_gpr_registers));
744		radeon_program_register_sequence(rdev,
745						 rv730_golden_registers,
746						 (const u32)ARRAY_SIZE(rv730_golden_registers));
747		radeon_program_register_sequence(rdev,
748						 rv730_mgcg_init,
749						 (const u32)ARRAY_SIZE(rv730_mgcg_init));
750		break;
751	case CHIP_RV710:
752		radeon_program_register_sequence(rdev,
753						 r7xx_golden_registers,
754						 (const u32)ARRAY_SIZE(r7xx_golden_registers));
755		radeon_program_register_sequence(rdev,
756						 r7xx_golden_dyn_gpr_registers,
757						 (const u32)ARRAY_SIZE(r7xx_golden_dyn_gpr_registers));
758		radeon_program_register_sequence(rdev,
759						 rv710_golden_registers,
760						 (const u32)ARRAY_SIZE(rv710_golden_registers));
761		radeon_program_register_sequence(rdev,
762						 rv710_mgcg_init,
763						 (const u32)ARRAY_SIZE(rv710_mgcg_init));
764		break;
765	case CHIP_RV740:
766		radeon_program_register_sequence(rdev,
767						 rv740_golden_registers,
768						 (const u32)ARRAY_SIZE(rv740_golden_registers));
769		radeon_program_register_sequence(rdev,
770						 rv740_mgcg_init,
771						 (const u32)ARRAY_SIZE(rv740_mgcg_init));
772		break;
773	default:
774		break;
775	}
776}
777
778#define PCIE_BUS_CLK                10000
779#define TCLK                        (PCIE_BUS_CLK / 10)
780
781/**
782 * rv770_get_xclk - get the xclk
783 *
784 * @rdev: radeon_device pointer
785 *
786 * Returns the reference clock used by the gfx engine
787 * (r7xx-cayman).
788 */
789u32 rv770_get_xclk(struct radeon_device *rdev)
790{
791	u32 reference_clock = rdev->clock.spll.reference_freq;
792	u32 tmp = RREG32(CG_CLKPIN_CNTL);
793
794	if (tmp & MUX_TCLK_TO_XCLK)
795		return TCLK;
796
797	if (tmp & XTALIN_DIVIDE)
798		return reference_clock / 4;
799
800	return reference_clock;
801}
802
803void rv770_page_flip(struct radeon_device *rdev, int crtc_id, u64 crtc_base)
804{
805	struct radeon_crtc *radeon_crtc = rdev->mode_info.crtcs[crtc_id];
806	u32 tmp = RREG32(AVIVO_D1GRPH_UPDATE + radeon_crtc->crtc_offset);
807	int i;
808
809	/* Lock the graphics update lock */
810	tmp |= AVIVO_D1GRPH_UPDATE_LOCK;
811	WREG32(AVIVO_D1GRPH_UPDATE + radeon_crtc->crtc_offset, tmp);
812
813	/* update the scanout addresses */
814	if (radeon_crtc->crtc_id) {
815		WREG32(D2GRPH_SECONDARY_SURFACE_ADDRESS_HIGH, upper_32_bits(crtc_base));
816		WREG32(D2GRPH_PRIMARY_SURFACE_ADDRESS_HIGH, upper_32_bits(crtc_base));
817	} else {
818		WREG32(D1GRPH_SECONDARY_SURFACE_ADDRESS_HIGH, upper_32_bits(crtc_base));
819		WREG32(D1GRPH_PRIMARY_SURFACE_ADDRESS_HIGH, upper_32_bits(crtc_base));
820	}
821	WREG32(D1GRPH_SECONDARY_SURFACE_ADDRESS + radeon_crtc->crtc_offset,
822	       (u32)crtc_base);
823	WREG32(D1GRPH_PRIMARY_SURFACE_ADDRESS + radeon_crtc->crtc_offset,
824	       (u32)crtc_base);
825
826	/* Wait for update_pending to go high. */
827	for (i = 0; i < rdev->usec_timeout; i++) {
828		if (RREG32(AVIVO_D1GRPH_UPDATE + radeon_crtc->crtc_offset) & AVIVO_D1GRPH_SURFACE_UPDATE_PENDING)
829			break;
830		udelay(1);
831	}
832	DRM_DEBUG("Update pending now high. Unlocking vupdate_lock.\n");
833
834	/* Unlock the lock, so double-buffering can take place inside vblank */
835	tmp &= ~AVIVO_D1GRPH_UPDATE_LOCK;
836	WREG32(AVIVO_D1GRPH_UPDATE + radeon_crtc->crtc_offset, tmp);
837}
838
839bool rv770_page_flip_pending(struct radeon_device *rdev, int crtc_id)
840{
841	struct radeon_crtc *radeon_crtc = rdev->mode_info.crtcs[crtc_id];
842
843	/* Return current update_pending status: */
844	return !!(RREG32(AVIVO_D1GRPH_UPDATE + radeon_crtc->crtc_offset) &
845		AVIVO_D1GRPH_SURFACE_UPDATE_PENDING);
846}
847
848/* get temperature in millidegrees */
849int rv770_get_temp(struct radeon_device *rdev)
850{
851	u32 temp = (RREG32(CG_MULT_THERMAL_STATUS) & ASIC_T_MASK) >>
852		ASIC_T_SHIFT;
853	int actual_temp;
854
855	if (temp & 0x400)
856		actual_temp = -256;
857	else if (temp & 0x200)
858		actual_temp = 255;
859	else if (temp & 0x100) {
860		actual_temp = temp & 0x1ff;
861		actual_temp |= ~0x1ff;
862	} else
863		actual_temp = temp & 0xff;
864
865	return (actual_temp * 1000) / 2;
866}
867
868void rv770_pm_misc(struct radeon_device *rdev)
869{
870	int req_ps_idx = rdev->pm.requested_power_state_index;
871	int req_cm_idx = rdev->pm.requested_clock_mode_index;
872	struct radeon_power_state *ps = &rdev->pm.power_state[req_ps_idx];
873	struct radeon_voltage *voltage = &ps->clock_info[req_cm_idx].voltage;
874
875	if ((voltage->type == VOLTAGE_SW) && voltage->voltage) {
876		/* 0xff01 is a flag rather then an actual voltage */
877		if (voltage->voltage == 0xff01)
878			return;
879		if (voltage->voltage != rdev->pm.current_vddc) {
880			radeon_atom_set_voltage(rdev, voltage->voltage, SET_VOLTAGE_TYPE_ASIC_VDDC);
881			rdev->pm.current_vddc = voltage->voltage;
882			DRM_DEBUG("Setting: v: %d\n", voltage->voltage);
883		}
884	}
885}
886
887/*
888 * GART
889 */
890static int rv770_pcie_gart_enable(struct radeon_device *rdev)
891{
892	u32 tmp;
893	int r, i;
894
895	if (rdev->gart.robj == NULL) {
896		dev_err(rdev->dev, "No VRAM object for PCIE GART.\n");
897		return -EINVAL;
898	}
899	r = radeon_gart_table_vram_pin(rdev);
900	if (r)
901		return r;
902	/* Setup L2 cache */
903	WREG32(VM_L2_CNTL, ENABLE_L2_CACHE | ENABLE_L2_FRAGMENT_PROCESSING |
904				ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
905				EFFECTIVE_L2_QUEUE_SIZE(7));
906	WREG32(VM_L2_CNTL2, 0);
907	WREG32(VM_L2_CNTL3, BANK_SELECT(0) | CACHE_UPDATE_MODE(2));
908	/* Setup TLB control */
909	tmp = ENABLE_L1_TLB | ENABLE_L1_FRAGMENT_PROCESSING |
910		SYSTEM_ACCESS_MODE_NOT_IN_SYS |
911		SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU |
912		EFFECTIVE_L1_TLB_SIZE(5) | EFFECTIVE_L1_QUEUE_SIZE(5);
913	WREG32(MC_VM_MD_L1_TLB0_CNTL, tmp);
914	WREG32(MC_VM_MD_L1_TLB1_CNTL, tmp);
915	WREG32(MC_VM_MD_L1_TLB2_CNTL, tmp);
916	if (rdev->family == CHIP_RV740)
917		WREG32(MC_VM_MD_L1_TLB3_CNTL, tmp);
918	WREG32(MC_VM_MB_L1_TLB0_CNTL, tmp);
919	WREG32(MC_VM_MB_L1_TLB1_CNTL, tmp);
920	WREG32(MC_VM_MB_L1_TLB2_CNTL, tmp);
921	WREG32(MC_VM_MB_L1_TLB3_CNTL, tmp);
922	WREG32(VM_CONTEXT0_PAGE_TABLE_START_ADDR, rdev->mc.gtt_start >> 12);
923	WREG32(VM_CONTEXT0_PAGE_TABLE_END_ADDR, rdev->mc.gtt_end >> 12);
924	WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR, rdev->gart.table_addr >> 12);
925	WREG32(VM_CONTEXT0_CNTL, ENABLE_CONTEXT | PAGE_TABLE_DEPTH(0) |
926				RANGE_PROTECTION_FAULT_ENABLE_DEFAULT);
927	WREG32(VM_CONTEXT0_PROTECTION_FAULT_DEFAULT_ADDR,
928			(u32)(rdev->dummy_page.addr >> 12));
929	for (i = 1; i < 7; i++)
930		WREG32(VM_CONTEXT0_CNTL + (i * 4), 0);
931
932	r600_pcie_gart_tlb_flush(rdev);
933	DRM_INFO("PCIE GART of %uM enabled (table at 0x%016llX).\n",
934		 (unsigned)(rdev->mc.gtt_size >> 20),
935		 (unsigned long long)rdev->gart.table_addr);
936	rdev->gart.ready = true;
937	return 0;
938}
939
940static void rv770_pcie_gart_disable(struct radeon_device *rdev)
941{
942	u32 tmp;
943	int i;
944
945	/* Disable all tables */
946	for (i = 0; i < 7; i++)
947		WREG32(VM_CONTEXT0_CNTL + (i * 4), 0);
948
949	/* Setup L2 cache */
950	WREG32(VM_L2_CNTL, ENABLE_L2_FRAGMENT_PROCESSING |
951				EFFECTIVE_L2_QUEUE_SIZE(7));
952	WREG32(VM_L2_CNTL2, 0);
953	WREG32(VM_L2_CNTL3, BANK_SELECT(0) | CACHE_UPDATE_MODE(2));
954	/* Setup TLB control */
955	tmp = EFFECTIVE_L1_TLB_SIZE(5) | EFFECTIVE_L1_QUEUE_SIZE(5);
956	WREG32(MC_VM_MD_L1_TLB0_CNTL, tmp);
957	WREG32(MC_VM_MD_L1_TLB1_CNTL, tmp);
958	WREG32(MC_VM_MD_L1_TLB2_CNTL, tmp);
959	WREG32(MC_VM_MB_L1_TLB0_CNTL, tmp);
960	WREG32(MC_VM_MB_L1_TLB1_CNTL, tmp);
961	WREG32(MC_VM_MB_L1_TLB2_CNTL, tmp);
962	WREG32(MC_VM_MB_L1_TLB3_CNTL, tmp);
963	radeon_gart_table_vram_unpin(rdev);
964}
965
966static void rv770_pcie_gart_fini(struct radeon_device *rdev)
967{
968	radeon_gart_fini(rdev);
969	rv770_pcie_gart_disable(rdev);
970	radeon_gart_table_vram_free(rdev);
971}
972
973
974static void rv770_agp_enable(struct radeon_device *rdev)
975{
976	u32 tmp;
977	int i;
978
979	/* Setup L2 cache */
980	WREG32(VM_L2_CNTL, ENABLE_L2_CACHE | ENABLE_L2_FRAGMENT_PROCESSING |
981				ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
982				EFFECTIVE_L2_QUEUE_SIZE(7));
983	WREG32(VM_L2_CNTL2, 0);
984	WREG32(VM_L2_CNTL3, BANK_SELECT(0) | CACHE_UPDATE_MODE(2));
985	/* Setup TLB control */
986	tmp = ENABLE_L1_TLB | ENABLE_L1_FRAGMENT_PROCESSING |
987		SYSTEM_ACCESS_MODE_NOT_IN_SYS |
988		SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU |
989		EFFECTIVE_L1_TLB_SIZE(5) | EFFECTIVE_L1_QUEUE_SIZE(5);
990	WREG32(MC_VM_MD_L1_TLB0_CNTL, tmp);
991	WREG32(MC_VM_MD_L1_TLB1_CNTL, tmp);
992	WREG32(MC_VM_MD_L1_TLB2_CNTL, tmp);
993	WREG32(MC_VM_MB_L1_TLB0_CNTL, tmp);
994	WREG32(MC_VM_MB_L1_TLB1_CNTL, tmp);
995	WREG32(MC_VM_MB_L1_TLB2_CNTL, tmp);
996	WREG32(MC_VM_MB_L1_TLB3_CNTL, tmp);
997	for (i = 0; i < 7; i++)
998		WREG32(VM_CONTEXT0_CNTL + (i * 4), 0);
999}
1000
1001static void rv770_mc_program(struct radeon_device *rdev)
1002{
1003	struct rv515_mc_save save;
1004	u32 tmp;
1005	int i, j;
1006
1007	/* Initialize HDP */
1008	for (i = 0, j = 0; i < 32; i++, j += 0x18) {
1009		WREG32((0x2c14 + j), 0x00000000);
1010		WREG32((0x2c18 + j), 0x00000000);
1011		WREG32((0x2c1c + j), 0x00000000);
1012		WREG32((0x2c20 + j), 0x00000000);
1013		WREG32((0x2c24 + j), 0x00000000);
1014	}
1015	/* r7xx hw bug.  Read from HDP_DEBUG1 rather
1016	 * than writing to HDP_REG_COHERENCY_FLUSH_CNTL
1017	 */
1018	tmp = RREG32(HDP_DEBUG1);
1019
1020	rv515_mc_stop(rdev, &save);
1021	if (r600_mc_wait_for_idle(rdev)) {
1022		dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
1023	}
1024	/* Lockout access through VGA aperture*/
1025	WREG32(VGA_HDP_CONTROL, VGA_MEMORY_DISABLE);
1026	/* Update configuration */
1027	if (rdev->flags & RADEON_IS_AGP) {
1028		if (rdev->mc.vram_start < rdev->mc.gtt_start) {
1029			/* VRAM before AGP */
1030			WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
1031				rdev->mc.vram_start >> 12);
1032			WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
1033				rdev->mc.gtt_end >> 12);
1034		} else {
1035			/* VRAM after AGP */
1036			WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
1037				rdev->mc.gtt_start >> 12);
1038			WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
1039				rdev->mc.vram_end >> 12);
1040		}
1041	} else {
1042		WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
1043			rdev->mc.vram_start >> 12);
1044		WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
1045			rdev->mc.vram_end >> 12);
1046	}
1047	WREG32(MC_VM_SYSTEM_APERTURE_DEFAULT_ADDR, rdev->vram_scratch.gpu_addr >> 12);
1048	tmp = ((rdev->mc.vram_end >> 24) & 0xFFFF) << 16;
1049	tmp |= ((rdev->mc.vram_start >> 24) & 0xFFFF);
1050	WREG32(MC_VM_FB_LOCATION, tmp);
1051	WREG32(HDP_NONSURFACE_BASE, (rdev->mc.vram_start >> 8));
1052	WREG32(HDP_NONSURFACE_INFO, (2 << 7));
1053	WREG32(HDP_NONSURFACE_SIZE, 0x3FFFFFFF);
1054	if (rdev->flags & RADEON_IS_AGP) {
1055		WREG32(MC_VM_AGP_TOP, rdev->mc.gtt_end >> 16);
1056		WREG32(MC_VM_AGP_BOT, rdev->mc.gtt_start >> 16);
1057		WREG32(MC_VM_AGP_BASE, rdev->mc.agp_base >> 22);
1058	} else {
1059		WREG32(MC_VM_AGP_BASE, 0);
1060		WREG32(MC_VM_AGP_TOP, 0x0FFFFFFF);
1061		WREG32(MC_VM_AGP_BOT, 0x0FFFFFFF);
1062	}
1063	if (r600_mc_wait_for_idle(rdev)) {
1064		dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
1065	}
1066	rv515_mc_resume(rdev, &save);
1067	/* we need to own VRAM, so turn off the VGA renderer here
1068	 * to stop it overwriting our objects */
1069	rv515_vga_render_disable(rdev);
1070}
1071
1072
1073/*
1074 * CP.
1075 */
1076void r700_cp_stop(struct radeon_device *rdev)
1077{
1078	if (rdev->asic->copy.copy_ring_index == RADEON_RING_TYPE_GFX_INDEX)
1079		radeon_ttm_set_active_vram_size(rdev, rdev->mc.visible_vram_size);
1080	WREG32(CP_ME_CNTL, (CP_ME_HALT | CP_PFP_HALT));
1081	WREG32(SCRATCH_UMSK, 0);
1082	rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = false;
1083}
1084
1085static int rv770_cp_load_microcode(struct radeon_device *rdev)
1086{
1087	const __be32 *fw_data;
1088	int i;
1089
1090	if (!rdev->me_fw || !rdev->pfp_fw)
1091		return -EINVAL;
1092
1093	r700_cp_stop(rdev);
1094	WREG32(CP_RB_CNTL,
1095#ifdef __BIG_ENDIAN
1096	       BUF_SWAP_32BIT |
1097#endif
1098	       RB_NO_UPDATE | RB_BLKSZ(15) | RB_BUFSZ(3));
1099
1100	/* Reset cp */
1101	WREG32(GRBM_SOFT_RESET, SOFT_RESET_CP);
1102	RREG32(GRBM_SOFT_RESET);
1103	mdelay(15);
1104	WREG32(GRBM_SOFT_RESET, 0);
1105
1106	fw_data = (const __be32 *)rdev->pfp_fw->data;
1107	WREG32(CP_PFP_UCODE_ADDR, 0);
1108	for (i = 0; i < R700_PFP_UCODE_SIZE; i++)
1109		WREG32(CP_PFP_UCODE_DATA, be32_to_cpup(fw_data++));
1110	WREG32(CP_PFP_UCODE_ADDR, 0);
1111
1112	fw_data = (const __be32 *)rdev->me_fw->data;
1113	WREG32(CP_ME_RAM_WADDR, 0);
1114	for (i = 0; i < R700_PM4_UCODE_SIZE; i++)
1115		WREG32(CP_ME_RAM_DATA, be32_to_cpup(fw_data++));
1116
1117	WREG32(CP_PFP_UCODE_ADDR, 0);
1118	WREG32(CP_ME_RAM_WADDR, 0);
1119	WREG32(CP_ME_RAM_RADDR, 0);
1120	return 0;
1121}
1122
1123void r700_cp_fini(struct radeon_device *rdev)
1124{
1125	struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
1126	r700_cp_stop(rdev);
1127	radeon_ring_fini(rdev, ring);
1128	radeon_scratch_free(rdev, ring->rptr_save_reg);
1129}
1130
1131void rv770_set_clk_bypass_mode(struct radeon_device *rdev)
1132{
1133	u32 tmp, i;
1134
1135	if (rdev->flags & RADEON_IS_IGP)
1136		return;
1137
1138	tmp = RREG32(CG_SPLL_FUNC_CNTL_2);
1139	tmp &= SCLK_MUX_SEL_MASK;
1140	tmp |= SCLK_MUX_SEL(1) | SCLK_MUX_UPDATE;
1141	WREG32(CG_SPLL_FUNC_CNTL_2, tmp);
1142
1143	for (i = 0; i < rdev->usec_timeout; i++) {
1144		if (RREG32(CG_SPLL_STATUS) & SPLL_CHG_STATUS)
1145			break;
1146		udelay(1);
1147	}
1148
1149	tmp &= ~SCLK_MUX_UPDATE;
1150	WREG32(CG_SPLL_FUNC_CNTL_2, tmp);
1151
1152	tmp = RREG32(MPLL_CNTL_MODE);
1153	if ((rdev->family == CHIP_RV710) || (rdev->family == CHIP_RV730))
1154		tmp &= ~RV730_MPLL_MCLK_SEL;
1155	else
1156		tmp &= ~MPLL_MCLK_SEL;
1157	WREG32(MPLL_CNTL_MODE, tmp);
1158}
1159
1160/*
1161 * Core functions
1162 */
1163static void rv770_gpu_init(struct radeon_device *rdev)
1164{
1165	int i, j, num_qd_pipes;
1166	u32 ta_aux_cntl;
1167	u32 sx_debug_1;
1168	u32 smx_dc_ctl0;
1169	u32 db_debug3;
1170	u32 num_gs_verts_per_thread;
1171	u32 vgt_gs_per_es;
1172	u32 gs_prim_buffer_depth = 0;
1173	u32 sq_ms_fifo_sizes;
1174	u32 sq_config;
1175	u32 sq_thread_resource_mgmt;
1176	u32 hdp_host_path_cntl;
1177	u32 sq_dyn_gpr_size_simd_ab_0;
1178	u32 gb_tiling_config = 0;
1179	u32 cc_gc_shader_pipe_config = 0;
1180	u32 mc_arb_ramcfg;
1181	u32 db_debug4, tmp;
1182	u32 inactive_pipes, shader_pipe_config;
1183	u32 disabled_rb_mask;
1184	unsigned active_number;
1185
1186	/* setup chip specs */
1187	rdev->config.rv770.tiling_group_size = 256;
1188	switch (rdev->family) {
1189	case CHIP_RV770:
1190		rdev->config.rv770.max_pipes = 4;
1191		rdev->config.rv770.max_tile_pipes = 8;
1192		rdev->config.rv770.max_simds = 10;
1193		rdev->config.rv770.max_backends = 4;
1194		rdev->config.rv770.max_gprs = 256;
1195		rdev->config.rv770.max_threads = 248;
1196		rdev->config.rv770.max_stack_entries = 512;
1197		rdev->config.rv770.max_hw_contexts = 8;
1198		rdev->config.rv770.max_gs_threads = 16 * 2;
1199		rdev->config.rv770.sx_max_export_size = 128;
1200		rdev->config.rv770.sx_max_export_pos_size = 16;
1201		rdev->config.rv770.sx_max_export_smx_size = 112;
1202		rdev->config.rv770.sq_num_cf_insts = 2;
1203
1204		rdev->config.rv770.sx_num_of_sets = 7;
1205		rdev->config.rv770.sc_prim_fifo_size = 0xF9;
1206		rdev->config.rv770.sc_hiz_tile_fifo_size = 0x30;
1207		rdev->config.rv770.sc_earlyz_tile_fifo_fize = 0x130;
1208		break;
1209	case CHIP_RV730:
1210		rdev->config.rv770.max_pipes = 2;
1211		rdev->config.rv770.max_tile_pipes = 4;
1212		rdev->config.rv770.max_simds = 8;
1213		rdev->config.rv770.max_backends = 2;
1214		rdev->config.rv770.max_gprs = 128;
1215		rdev->config.rv770.max_threads = 248;
1216		rdev->config.rv770.max_stack_entries = 256;
1217		rdev->config.rv770.max_hw_contexts = 8;
1218		rdev->config.rv770.max_gs_threads = 16 * 2;
1219		rdev->config.rv770.sx_max_export_size = 256;
1220		rdev->config.rv770.sx_max_export_pos_size = 32;
1221		rdev->config.rv770.sx_max_export_smx_size = 224;
1222		rdev->config.rv770.sq_num_cf_insts = 2;
1223
1224		rdev->config.rv770.sx_num_of_sets = 7;
1225		rdev->config.rv770.sc_prim_fifo_size = 0xf9;
1226		rdev->config.rv770.sc_hiz_tile_fifo_size = 0x30;
1227		rdev->config.rv770.sc_earlyz_tile_fifo_fize = 0x130;
1228		if (rdev->config.rv770.sx_max_export_pos_size > 16) {
1229			rdev->config.rv770.sx_max_export_pos_size -= 16;
1230			rdev->config.rv770.sx_max_export_smx_size += 16;
1231		}
1232		break;
1233	case CHIP_RV710:
1234		rdev->config.rv770.max_pipes = 2;
1235		rdev->config.rv770.max_tile_pipes = 2;
1236		rdev->config.rv770.max_simds = 2;
1237		rdev->config.rv770.max_backends = 1;
1238		rdev->config.rv770.max_gprs = 256;
1239		rdev->config.rv770.max_threads = 192;
1240		rdev->config.rv770.max_stack_entries = 256;
1241		rdev->config.rv770.max_hw_contexts = 4;
1242		rdev->config.rv770.max_gs_threads = 8 * 2;
1243		rdev->config.rv770.sx_max_export_size = 128;
1244		rdev->config.rv770.sx_max_export_pos_size = 16;
1245		rdev->config.rv770.sx_max_export_smx_size = 112;
1246		rdev->config.rv770.sq_num_cf_insts = 1;
1247
1248		rdev->config.rv770.sx_num_of_sets = 7;
1249		rdev->config.rv770.sc_prim_fifo_size = 0x40;
1250		rdev->config.rv770.sc_hiz_tile_fifo_size = 0x30;
1251		rdev->config.rv770.sc_earlyz_tile_fifo_fize = 0x130;
1252		break;
1253	case CHIP_RV740:
1254		rdev->config.rv770.max_pipes = 4;
1255		rdev->config.rv770.max_tile_pipes = 4;
1256		rdev->config.rv770.max_simds = 8;
1257		rdev->config.rv770.max_backends = 4;
1258		rdev->config.rv770.max_gprs = 256;
1259		rdev->config.rv770.max_threads = 248;
1260		rdev->config.rv770.max_stack_entries = 512;
1261		rdev->config.rv770.max_hw_contexts = 8;
1262		rdev->config.rv770.max_gs_threads = 16 * 2;
1263		rdev->config.rv770.sx_max_export_size = 256;
1264		rdev->config.rv770.sx_max_export_pos_size = 32;
1265		rdev->config.rv770.sx_max_export_smx_size = 224;
1266		rdev->config.rv770.sq_num_cf_insts = 2;
1267
1268		rdev->config.rv770.sx_num_of_sets = 7;
1269		rdev->config.rv770.sc_prim_fifo_size = 0x100;
1270		rdev->config.rv770.sc_hiz_tile_fifo_size = 0x30;
1271		rdev->config.rv770.sc_earlyz_tile_fifo_fize = 0x130;
1272
1273		if (rdev->config.rv770.sx_max_export_pos_size > 16) {
1274			rdev->config.rv770.sx_max_export_pos_size -= 16;
1275			rdev->config.rv770.sx_max_export_smx_size += 16;
1276		}
1277		break;
1278	default:
1279		break;
1280	}
1281
1282	/* Initialize HDP */
1283	j = 0;
1284	for (i = 0; i < 32; i++) {
1285		WREG32((0x2c14 + j), 0x00000000);
1286		WREG32((0x2c18 + j), 0x00000000);
1287		WREG32((0x2c1c + j), 0x00000000);
1288		WREG32((0x2c20 + j), 0x00000000);
1289		WREG32((0x2c24 + j), 0x00000000);
1290		j += 0x18;
1291	}
1292
1293	WREG32(GRBM_CNTL, GRBM_READ_TIMEOUT(0xff));
1294
1295	/* setup tiling, simd, pipe config */
1296	mc_arb_ramcfg = RREG32(MC_ARB_RAMCFG);
1297
1298	shader_pipe_config = RREG32(CC_GC_SHADER_PIPE_CONFIG);
1299	inactive_pipes = (shader_pipe_config & INACTIVE_QD_PIPES_MASK) >> INACTIVE_QD_PIPES_SHIFT;
1300	for (i = 0, tmp = 1, active_number = 0; i < R7XX_MAX_PIPES; i++) {
1301		if (!(inactive_pipes & tmp)) {
1302			active_number++;
1303		}
1304		tmp <<= 1;
1305	}
1306	if (active_number == 1) {
1307		WREG32(SPI_CONFIG_CNTL, DISABLE_INTERP_1);
1308	} else {
1309		WREG32(SPI_CONFIG_CNTL, 0);
1310	}
1311
1312	cc_gc_shader_pipe_config = RREG32(CC_GC_SHADER_PIPE_CONFIG) & 0xffffff00;
1313	tmp = rdev->config.rv770.max_simds -
1314		r600_count_pipe_bits((cc_gc_shader_pipe_config >> 16) & R7XX_MAX_SIMDS_MASK);
1315	rdev->config.rv770.active_simds = tmp;
1316
1317	switch (rdev->config.rv770.max_tile_pipes) {
1318	case 1:
1319	default:
1320		gb_tiling_config = PIPE_TILING(0);
1321		break;
1322	case 2:
1323		gb_tiling_config = PIPE_TILING(1);
1324		break;
1325	case 4:
1326		gb_tiling_config = PIPE_TILING(2);
1327		break;
1328	case 8:
1329		gb_tiling_config = PIPE_TILING(3);
1330		break;
1331	}
1332	rdev->config.rv770.tiling_npipes = rdev->config.rv770.max_tile_pipes;
1333
1334	disabled_rb_mask = (RREG32(CC_RB_BACKEND_DISABLE) >> 16) & R7XX_MAX_BACKENDS_MASK;
1335	tmp = 0;
1336	for (i = 0; i < rdev->config.rv770.max_backends; i++)
1337		tmp |= (1 << i);
1338	/* if all the backends are disabled, fix it up here */
1339	if ((disabled_rb_mask & tmp) == tmp) {
1340		for (i = 0; i < rdev->config.rv770.max_backends; i++)
1341			disabled_rb_mask &= ~(1 << i);
1342	}
1343	tmp = (gb_tiling_config & PIPE_TILING__MASK) >> PIPE_TILING__SHIFT;
1344	tmp = r6xx_remap_render_backend(rdev, tmp, rdev->config.rv770.max_backends,
1345					R7XX_MAX_BACKENDS, disabled_rb_mask);
1346	gb_tiling_config |= tmp << 16;
1347	rdev->config.rv770.backend_map = tmp;
1348
1349	if (rdev->family == CHIP_RV770)
1350		gb_tiling_config |= BANK_TILING(1);
1351	else {
1352		if ((mc_arb_ramcfg & NOOFBANK_MASK) >> NOOFBANK_SHIFT)
1353			gb_tiling_config |= BANK_TILING(1);
1354		else
1355			gb_tiling_config |= BANK_TILING(0);
1356	}
1357	rdev->config.rv770.tiling_nbanks = 4 << ((gb_tiling_config >> 4) & 0x3);
1358	gb_tiling_config |= GROUP_SIZE((mc_arb_ramcfg & BURSTLENGTH_MASK) >> BURSTLENGTH_SHIFT);
1359	if (((mc_arb_ramcfg & NOOFROWS_MASK) >> NOOFROWS_SHIFT) > 3) {
1360		gb_tiling_config |= ROW_TILING(3);
1361		gb_tiling_config |= SAMPLE_SPLIT(3);
1362	} else {
1363		gb_tiling_config |=
1364			ROW_TILING(((mc_arb_ramcfg & NOOFROWS_MASK) >> NOOFROWS_SHIFT));
1365		gb_tiling_config |=
1366			SAMPLE_SPLIT(((mc_arb_ramcfg & NOOFROWS_MASK) >> NOOFROWS_SHIFT));
1367	}
1368
1369	gb_tiling_config |= BANK_SWAPS(1);
1370	rdev->config.rv770.tile_config = gb_tiling_config;
1371
1372	WREG32(GB_TILING_CONFIG, gb_tiling_config);
1373	WREG32(DCP_TILING_CONFIG, (gb_tiling_config & 0xffff));
1374	WREG32(HDP_TILING_CONFIG, (gb_tiling_config & 0xffff));
1375	WREG32(DMA_TILING_CONFIG, (gb_tiling_config & 0xffff));
1376	WREG32(DMA_TILING_CONFIG2, (gb_tiling_config & 0xffff));
1377	if (rdev->family == CHIP_RV730) {
1378		WREG32(UVD_UDEC_DB_TILING_CONFIG, (gb_tiling_config & 0xffff));
1379		WREG32(UVD_UDEC_DBW_TILING_CONFIG, (gb_tiling_config & 0xffff));
1380		WREG32(UVD_UDEC_TILING_CONFIG, (gb_tiling_config & 0xffff));
1381	}
1382
1383	WREG32(CGTS_SYS_TCC_DISABLE, 0);
1384	WREG32(CGTS_TCC_DISABLE, 0);
1385	WREG32(CGTS_USER_SYS_TCC_DISABLE, 0);
1386	WREG32(CGTS_USER_TCC_DISABLE, 0);
1387
1388
1389	num_qd_pipes = R7XX_MAX_PIPES - r600_count_pipe_bits((cc_gc_shader_pipe_config & INACTIVE_QD_PIPES_MASK) >> 8);
1390	WREG32(VGT_OUT_DEALLOC_CNTL, (num_qd_pipes * 4) & DEALLOC_DIST_MASK);
1391	WREG32(VGT_VERTEX_REUSE_BLOCK_CNTL, ((num_qd_pipes * 4) - 2) & VTX_REUSE_DEPTH_MASK);
1392
1393	/* set HW defaults for 3D engine */
1394	WREG32(CP_QUEUE_THRESHOLDS, (ROQ_IB1_START(0x16) |
1395				     ROQ_IB2_START(0x2b)));
1396
1397	WREG32(CP_MEQ_THRESHOLDS, STQ_SPLIT(0x30));
1398
1399	ta_aux_cntl = RREG32(TA_CNTL_AUX);
1400	WREG32(TA_CNTL_AUX, ta_aux_cntl | DISABLE_CUBE_ANISO);
1401
1402	sx_debug_1 = RREG32(SX_DEBUG_1);
1403	sx_debug_1 |= ENABLE_NEW_SMX_ADDRESS;
1404	WREG32(SX_DEBUG_1, sx_debug_1);
1405
1406	smx_dc_ctl0 = RREG32(SMX_DC_CTL0);
1407	smx_dc_ctl0 &= ~CACHE_DEPTH(0x1ff);
1408	smx_dc_ctl0 |= CACHE_DEPTH((rdev->config.rv770.sx_num_of_sets * 64) - 1);
1409	WREG32(SMX_DC_CTL0, smx_dc_ctl0);
1410
1411	if (rdev->family != CHIP_RV740)
1412		WREG32(SMX_EVENT_CTL, (ES_FLUSH_CTL(4) |
1413				       GS_FLUSH_CTL(4) |
1414				       ACK_FLUSH_CTL(3) |
1415				       SYNC_FLUSH_CTL));
1416
1417	if (rdev->family != CHIP_RV770)
1418		WREG32(SMX_SAR_CTL0, 0x00003f3f);
1419
1420	db_debug3 = RREG32(DB_DEBUG3);
1421	db_debug3 &= ~DB_CLK_OFF_DELAY(0x1f);
1422	switch (rdev->family) {
1423	case CHIP_RV770:
1424	case CHIP_RV740:
1425		db_debug3 |= DB_CLK_OFF_DELAY(0x1f);
1426		break;
1427	case CHIP_RV710:
1428	case CHIP_RV730:
1429	default:
1430		db_debug3 |= DB_CLK_OFF_DELAY(2);
1431		break;
1432	}
1433	WREG32(DB_DEBUG3, db_debug3);
1434
1435	if (rdev->family != CHIP_RV770) {
1436		db_debug4 = RREG32(DB_DEBUG4);
1437		db_debug4 |= DISABLE_TILE_COVERED_FOR_PS_ITER;
1438		WREG32(DB_DEBUG4, db_debug4);
1439	}
1440
1441	WREG32(SX_EXPORT_BUFFER_SIZES, (COLOR_BUFFER_SIZE((rdev->config.rv770.sx_max_export_size / 4) - 1) |
1442					POSITION_BUFFER_SIZE((rdev->config.rv770.sx_max_export_pos_size / 4) - 1) |
1443					SMX_BUFFER_SIZE((rdev->config.rv770.sx_max_export_smx_size / 4) - 1)));
1444
1445	WREG32(PA_SC_FIFO_SIZE, (SC_PRIM_FIFO_SIZE(rdev->config.rv770.sc_prim_fifo_size) |
1446				 SC_HIZ_TILE_FIFO_SIZE(rdev->config.rv770.sc_hiz_tile_fifo_size) |
1447				 SC_EARLYZ_TILE_FIFO_SIZE(rdev->config.rv770.sc_earlyz_tile_fifo_fize)));
1448
1449	WREG32(PA_SC_MULTI_CHIP_CNTL, 0);
1450
1451	WREG32(VGT_NUM_INSTANCES, 1);
1452
1453	WREG32(SPI_CONFIG_CNTL_1, VTX_DONE_DELAY(4));
1454
1455	WREG32(CP_PERFMON_CNTL, 0);
1456
1457	sq_ms_fifo_sizes = (CACHE_FIFO_SIZE(16 * rdev->config.rv770.sq_num_cf_insts) |
1458			    DONE_FIFO_HIWATER(0xe0) |
1459			    ALU_UPDATE_FIFO_HIWATER(0x8));
1460	switch (rdev->family) {
1461	case CHIP_RV770:
1462	case CHIP_RV730:
1463	case CHIP_RV710:
1464		sq_ms_fifo_sizes |= FETCH_FIFO_HIWATER(0x1);
1465		break;
1466	case CHIP_RV740:
1467	default:
1468		sq_ms_fifo_sizes |= FETCH_FIFO_HIWATER(0x4);
1469		break;
1470	}
1471	WREG32(SQ_MS_FIFO_SIZES, sq_ms_fifo_sizes);
1472
1473	/* SQ_CONFIG, SQ_GPR_RESOURCE_MGMT, SQ_THREAD_RESOURCE_MGMT, SQ_STACK_RESOURCE_MGMT
1474	 * should be adjusted as needed by the 2D/3D drivers.  This just sets default values
1475	 */
1476	sq_config = RREG32(SQ_CONFIG);
1477	sq_config &= ~(PS_PRIO(3) |
1478		       VS_PRIO(3) |
1479		       GS_PRIO(3) |
1480		       ES_PRIO(3));
1481	sq_config |= (DX9_CONSTS |
1482		      VC_ENABLE |
1483		      EXPORT_SRC_C |
1484		      PS_PRIO(0) |
1485		      VS_PRIO(1) |
1486		      GS_PRIO(2) |
1487		      ES_PRIO(3));
1488	if (rdev->family == CHIP_RV710)
1489		/* no vertex cache */
1490		sq_config &= ~VC_ENABLE;
1491
1492	WREG32(SQ_CONFIG, sq_config);
1493
1494	WREG32(SQ_GPR_RESOURCE_MGMT_1,  (NUM_PS_GPRS((rdev->config.rv770.max_gprs * 24)/64) |
1495					 NUM_VS_GPRS((rdev->config.rv770.max_gprs * 24)/64) |
1496					 NUM_CLAUSE_TEMP_GPRS(((rdev->config.rv770.max_gprs * 24)/64)/2)));
1497
1498	WREG32(SQ_GPR_RESOURCE_MGMT_2,  (NUM_GS_GPRS((rdev->config.rv770.max_gprs * 7)/64) |
1499					 NUM_ES_GPRS((rdev->config.rv770.max_gprs * 7)/64)));
1500
1501	sq_thread_resource_mgmt = (NUM_PS_THREADS((rdev->config.rv770.max_threads * 4)/8) |
1502				   NUM_VS_THREADS((rdev->config.rv770.max_threads * 2)/8) |
1503				   NUM_ES_THREADS((rdev->config.rv770.max_threads * 1)/8));
1504	if (((rdev->config.rv770.max_threads * 1) / 8) > rdev->config.rv770.max_gs_threads)
1505		sq_thread_resource_mgmt |= NUM_GS_THREADS(rdev->config.rv770.max_gs_threads);
1506	else
1507		sq_thread_resource_mgmt |= NUM_GS_THREADS((rdev->config.rv770.max_gs_threads * 1)/8);
1508	WREG32(SQ_THREAD_RESOURCE_MGMT, sq_thread_resource_mgmt);
1509
1510	WREG32(SQ_STACK_RESOURCE_MGMT_1, (NUM_PS_STACK_ENTRIES((rdev->config.rv770.max_stack_entries * 1)/4) |
1511						     NUM_VS_STACK_ENTRIES((rdev->config.rv770.max_stack_entries * 1)/4)));
1512
1513	WREG32(SQ_STACK_RESOURCE_MGMT_2, (NUM_GS_STACK_ENTRIES((rdev->config.rv770.max_stack_entries * 1)/4) |
1514						     NUM_ES_STACK_ENTRIES((rdev->config.rv770.max_stack_entries * 1)/4)));
1515
1516	sq_dyn_gpr_size_simd_ab_0 = (SIMDA_RING0((rdev->config.rv770.max_gprs * 38)/64) |
1517				     SIMDA_RING1((rdev->config.rv770.max_gprs * 38)/64) |
1518				     SIMDB_RING0((rdev->config.rv770.max_gprs * 38)/64) |
1519				     SIMDB_RING1((rdev->config.rv770.max_gprs * 38)/64));
1520
1521	WREG32(SQ_DYN_GPR_SIZE_SIMD_AB_0, sq_dyn_gpr_size_simd_ab_0);
1522	WREG32(SQ_DYN_GPR_SIZE_SIMD_AB_1, sq_dyn_gpr_size_simd_ab_0);
1523	WREG32(SQ_DYN_GPR_SIZE_SIMD_AB_2, sq_dyn_gpr_size_simd_ab_0);
1524	WREG32(SQ_DYN_GPR_SIZE_SIMD_AB_3, sq_dyn_gpr_size_simd_ab_0);
1525	WREG32(SQ_DYN_GPR_SIZE_SIMD_AB_4, sq_dyn_gpr_size_simd_ab_0);
1526	WREG32(SQ_DYN_GPR_SIZE_SIMD_AB_5, sq_dyn_gpr_size_simd_ab_0);
1527	WREG32(SQ_DYN_GPR_SIZE_SIMD_AB_6, sq_dyn_gpr_size_simd_ab_0);
1528	WREG32(SQ_DYN_GPR_SIZE_SIMD_AB_7, sq_dyn_gpr_size_simd_ab_0);
1529
1530	WREG32(PA_SC_FORCE_EOV_MAX_CNTS, (FORCE_EOV_MAX_CLK_CNT(4095) |
1531					  FORCE_EOV_MAX_REZ_CNT(255)));
1532
1533	if (rdev->family == CHIP_RV710)
1534		WREG32(VGT_CACHE_INVALIDATION, (CACHE_INVALIDATION(TC_ONLY) |
1535						AUTO_INVLD_EN(ES_AND_GS_AUTO)));
1536	else
1537		WREG32(VGT_CACHE_INVALIDATION, (CACHE_INVALIDATION(VC_AND_TC) |
1538						AUTO_INVLD_EN(ES_AND_GS_AUTO)));
1539
1540	switch (rdev->family) {
1541	case CHIP_RV770:
1542	case CHIP_RV730:
1543	case CHIP_RV740:
1544		gs_prim_buffer_depth = 384;
1545		break;
1546	case CHIP_RV710:
1547		gs_prim_buffer_depth = 128;
1548		break;
1549	default:
1550		break;
1551	}
1552
1553	num_gs_verts_per_thread = rdev->config.rv770.max_pipes * 16;
1554	vgt_gs_per_es = gs_prim_buffer_depth + num_gs_verts_per_thread;
1555	/* Max value for this is 256 */
1556	if (vgt_gs_per_es > 256)
1557		vgt_gs_per_es = 256;
1558
1559	WREG32(VGT_ES_PER_GS, 128);
1560	WREG32(VGT_GS_PER_ES, vgt_gs_per_es);
1561	WREG32(VGT_GS_PER_VS, 2);
1562
1563	/* more default values. 2D/3D driver should adjust as needed */
1564	WREG32(VGT_GS_VERTEX_REUSE, 16);
1565	WREG32(PA_SC_LINE_STIPPLE_STATE, 0);
1566	WREG32(VGT_STRMOUT_EN, 0);
1567	WREG32(SX_MISC, 0);
1568	WREG32(PA_SC_MODE_CNTL, 0);
1569	WREG32(PA_SC_EDGERULE, 0xaaaaaaaa);
1570	WREG32(PA_SC_AA_CONFIG, 0);
1571	WREG32(PA_SC_CLIPRECT_RULE, 0xffff);
1572	WREG32(PA_SC_LINE_STIPPLE, 0);
1573	WREG32(SPI_INPUT_Z, 0);
1574	WREG32(SPI_PS_IN_CONTROL_0, NUM_INTERP(2));
1575	WREG32(CB_COLOR7_FRAG, 0);
1576
1577	/* clear render buffer base addresses */
1578	WREG32(CB_COLOR0_BASE, 0);
1579	WREG32(CB_COLOR1_BASE, 0);
1580	WREG32(CB_COLOR2_BASE, 0);
1581	WREG32(CB_COLOR3_BASE, 0);
1582	WREG32(CB_COLOR4_BASE, 0);
1583	WREG32(CB_COLOR5_BASE, 0);
1584	WREG32(CB_COLOR6_BASE, 0);
1585	WREG32(CB_COLOR7_BASE, 0);
1586
1587	WREG32(TCP_CNTL, 0);
1588
1589	hdp_host_path_cntl = RREG32(HDP_HOST_PATH_CNTL);
1590	WREG32(HDP_HOST_PATH_CNTL, hdp_host_path_cntl);
1591
1592	WREG32(PA_SC_MULTI_CHIP_CNTL, 0);
1593
1594	WREG32(PA_CL_ENHANCE, (CLIP_VTX_REORDER_ENA |
1595					  NUM_CLIP_SEQ(3)));
1596	WREG32(VC_ENHANCE, 0);
1597}
1598
1599void r700_vram_gtt_location(struct radeon_device *rdev, struct radeon_mc *mc)
1600{
1601	u64 size_bf, size_af;
1602
1603	if (mc->mc_vram_size > 0xE0000000) {
1604		/* leave room for at least 512M GTT */
1605		dev_warn(rdev->dev, "limiting VRAM\n");
1606		mc->real_vram_size = 0xE0000000;
1607		mc->mc_vram_size = 0xE0000000;
1608	}
1609	if (rdev->flags & RADEON_IS_AGP) {
1610		size_bf = mc->gtt_start;
1611		size_af = mc->mc_mask - mc->gtt_end;
1612		if (size_bf > size_af) {
1613			if (mc->mc_vram_size > size_bf) {
1614				dev_warn(rdev->dev, "limiting VRAM\n");
1615				mc->real_vram_size = size_bf;
1616				mc->mc_vram_size = size_bf;
1617			}
1618			mc->vram_start = mc->gtt_start - mc->mc_vram_size;
1619		} else {
1620			if (mc->mc_vram_size > size_af) {
1621				dev_warn(rdev->dev, "limiting VRAM\n");
1622				mc->real_vram_size = size_af;
1623				mc->mc_vram_size = size_af;
1624			}
1625			mc->vram_start = mc->gtt_end + 1;
1626		}
1627		mc->vram_end = mc->vram_start + mc->mc_vram_size - 1;
1628		dev_info(rdev->dev, "VRAM: %lluM 0x%08llX - 0x%08llX (%lluM used)\n",
1629				mc->mc_vram_size >> 20, mc->vram_start,
1630				mc->vram_end, mc->real_vram_size >> 20);
1631	} else {
1632		radeon_vram_location(rdev, &rdev->mc, 0);
1633		rdev->mc.gtt_base_align = 0;
1634		radeon_gtt_location(rdev, mc);
1635	}
1636}
1637
1638static int rv770_mc_init(struct radeon_device *rdev)
1639{
1640	u32 tmp;
1641	int chansize, numchan;
1642
1643	/* Get VRAM informations */
1644	rdev->mc.vram_is_ddr = true;
1645	tmp = RREG32(MC_ARB_RAMCFG);
1646	if (tmp & CHANSIZE_OVERRIDE) {
1647		chansize = 16;
1648	} else if (tmp & CHANSIZE_MASK) {
1649		chansize = 64;
1650	} else {
1651		chansize = 32;
1652	}
1653	tmp = RREG32(MC_SHARED_CHMAP);
1654	switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
1655	case 0:
1656	default:
1657		numchan = 1;
1658		break;
1659	case 1:
1660		numchan = 2;
1661		break;
1662	case 2:
1663		numchan = 4;
1664		break;
1665	case 3:
1666		numchan = 8;
1667		break;
1668	}
1669	rdev->mc.vram_width = numchan * chansize;
1670	/* Could aper size report 0 ? */
1671	rdev->mc.aper_base = pci_resource_start(rdev->pdev, 0);
1672	rdev->mc.aper_size = pci_resource_len(rdev->pdev, 0);
1673	/* Setup GPU memory space */
1674	rdev->mc.mc_vram_size = RREG32(CONFIG_MEMSIZE);
1675	rdev->mc.real_vram_size = RREG32(CONFIG_MEMSIZE);
1676	rdev->mc.visible_vram_size = rdev->mc.aper_size;
1677	r700_vram_gtt_location(rdev, &rdev->mc);
1678	radeon_update_bandwidth_info(rdev);
1679
1680	return 0;
1681}
1682
1683static int rv770_startup(struct radeon_device *rdev)
1684{
1685	struct radeon_ring *ring;
1686	int r;
1687
1688	/* enable pcie gen2 link */
1689	rv770_pcie_gen2_enable(rdev);
1690
1691	/* scratch needs to be initialized before MC */
1692	r = r600_vram_scratch_init(rdev);
1693	if (r)
1694		return r;
1695
1696	rv770_mc_program(rdev);
1697
1698	if (rdev->flags & RADEON_IS_AGP) {
1699		rv770_agp_enable(rdev);
1700	} else {
1701		r = rv770_pcie_gart_enable(rdev);
1702		if (r)
1703			return r;
1704	}
1705
1706	rv770_gpu_init(rdev);
1707
1708	/* allocate wb buffer */
1709	r = radeon_wb_init(rdev);
1710	if (r)
1711		return r;
1712
1713	r = radeon_fence_driver_start_ring(rdev, RADEON_RING_TYPE_GFX_INDEX);
1714	if (r) {
1715		dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
1716		return r;
1717	}
1718
1719	r = radeon_fence_driver_start_ring(rdev, R600_RING_TYPE_DMA_INDEX);
1720	if (r) {
1721		dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r);
1722		return r;
1723	}
1724
1725	r = uvd_v2_2_resume(rdev);
1726	if (!r) {
1727		r = radeon_fence_driver_start_ring(rdev,
1728						   R600_RING_TYPE_UVD_INDEX);
1729		if (r)
1730			dev_err(rdev->dev, "UVD fences init error (%d).\n", r);
1731	}
1732
1733	if (r)
1734		rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_size = 0;
1735
1736	/* Enable IRQ */
1737	if (!rdev->irq.installed) {
1738		r = radeon_irq_kms_init(rdev);
1739		if (r)
1740			return r;
1741	}
1742
1743	r = r600_irq_init(rdev);
1744	if (r) {
1745		DRM_ERROR("radeon: IH init failed (%d).\n", r);
1746		radeon_irq_kms_fini(rdev);
1747		return r;
1748	}
1749	r600_irq_set(rdev);
1750
1751	ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
1752	r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP_RPTR_OFFSET,
1753			     RADEON_CP_PACKET2);
1754	if (r)
1755		return r;
1756
1757	ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];
1758	r = radeon_ring_init(rdev, ring, ring->ring_size, R600_WB_DMA_RPTR_OFFSET,
1759			     DMA_PACKET(DMA_PACKET_NOP, 0, 0, 0));
1760	if (r)
1761		return r;
1762
1763	r = rv770_cp_load_microcode(rdev);
1764	if (r)
1765		return r;
1766	r = r600_cp_resume(rdev);
1767	if (r)
1768		return r;
1769
1770	r = r600_dma_resume(rdev);
1771	if (r)
1772		return r;
1773
1774	ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX];
1775	if (ring->ring_size) {
1776		r = radeon_ring_init(rdev, ring, ring->ring_size, 0,
1777				     RADEON_CP_PACKET2);
1778		if (!r)
1779			r = uvd_v1_0_init(rdev);
1780
1781		if (r)
1782			DRM_ERROR("radeon: failed initializing UVD (%d).\n", r);
1783	}
1784
1785	r = radeon_ib_pool_init(rdev);
1786	if (r) {
1787		dev_err(rdev->dev, "IB initialization failed (%d).\n", r);
1788		return r;
1789	}
1790
1791	r = r600_audio_init(rdev);
1792	if (r) {
1793		DRM_ERROR("radeon: audio init failed\n");
1794		return r;
1795	}
1796
1797	return 0;
1798}
1799
1800int rv770_resume(struct radeon_device *rdev)
1801{
1802	int r;
1803
1804	/* Do not reset GPU before posting, on rv770 hw unlike on r500 hw,
1805	 * posting will perform necessary task to bring back GPU into good
1806	 * shape.
1807	 */
1808	/* post card */
1809	atom_asic_init(rdev->mode_info.atom_context);
1810
1811	/* init golden registers */
1812	rv770_init_golden_registers(rdev);
1813
1814	if (rdev->pm.pm_method == PM_METHOD_DPM)
1815		radeon_pm_resume(rdev);
1816
1817	rdev->accel_working = true;
1818	r = rv770_startup(rdev);
1819	if (r) {
1820		DRM_ERROR("r600 startup failed on resume\n");
1821		rdev->accel_working = false;
1822		return r;
1823	}
1824
1825	return r;
1826
1827}
1828
1829int rv770_suspend(struct radeon_device *rdev)
1830{
1831	radeon_pm_suspend(rdev);
1832	r600_audio_fini(rdev);
1833	uvd_v1_0_fini(rdev);
1834	radeon_uvd_suspend(rdev);
1835	r700_cp_stop(rdev);
1836	r600_dma_stop(rdev);
1837	r600_irq_suspend(rdev);
1838	radeon_wb_disable(rdev);
1839	rv770_pcie_gart_disable(rdev);
1840
1841	return 0;
1842}
1843
1844/* Plan is to move initialization in that function and use
1845 * helper function so that radeon_device_init pretty much
1846 * do nothing more than calling asic specific function. This
1847 * should also allow to remove a bunch of callback function
1848 * like vram_info.
1849 */
1850int rv770_init(struct radeon_device *rdev)
1851{
1852	int r;
1853
1854	/* Read BIOS */
1855	if (!radeon_get_bios(rdev)) {
1856		if (ASIC_IS_AVIVO(rdev))
1857			return -EINVAL;
1858	}
1859	/* Must be an ATOMBIOS */
1860	if (!rdev->is_atom_bios) {
1861		dev_err(rdev->dev, "Expecting atombios for R600 GPU\n");
1862		return -EINVAL;
1863	}
1864	r = radeon_atombios_init(rdev);
1865	if (r)
1866		return r;
1867	/* Post card if necessary */
1868	if (!radeon_card_posted(rdev)) {
1869		if (!rdev->bios) {
1870			dev_err(rdev->dev, "Card not posted and no BIOS - ignoring\n");
1871			return -EINVAL;
1872		}
1873		DRM_INFO("GPU not posted. posting now...\n");
1874		atom_asic_init(rdev->mode_info.atom_context);
1875	}
1876	/* init golden registers */
1877	rv770_init_golden_registers(rdev);
1878	/* Initialize scratch registers */
1879	r600_scratch_init(rdev);
1880	/* Initialize surface registers */
1881	radeon_surface_init(rdev);
1882	/* Initialize clocks */
1883	radeon_get_clock_info(rdev->ddev);
1884	/* Fence driver */
1885	r = radeon_fence_driver_init(rdev);
1886	if (r)
1887		return r;
1888	/* initialize AGP */
1889	if (rdev->flags & RADEON_IS_AGP) {
1890		r = radeon_agp_init(rdev);
1891		if (r)
1892			radeon_agp_disable(rdev);
1893	}
1894	r = rv770_mc_init(rdev);
1895	if (r)
1896		return r;
1897	/* Memory manager */
1898	r = radeon_bo_init(rdev);
1899	if (r)
1900		return r;
1901
1902	if (!rdev->me_fw || !rdev->pfp_fw || !rdev->rlc_fw) {
1903		r = r600_init_microcode(rdev);
1904		if (r) {
1905			DRM_ERROR("Failed to load firmware!\n");
1906			return r;
1907		}
1908	}
1909
1910	/* Initialize power management */
1911	radeon_pm_init(rdev);
1912
1913	rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ring_obj = NULL;
1914	r600_ring_init(rdev, &rdev->ring[RADEON_RING_TYPE_GFX_INDEX], 1024 * 1024);
1915
1916	rdev->ring[R600_RING_TYPE_DMA_INDEX].ring_obj = NULL;
1917	r600_ring_init(rdev, &rdev->ring[R600_RING_TYPE_DMA_INDEX], 64 * 1024);
1918
1919	r = radeon_uvd_init(rdev);
1920	if (!r) {
1921		rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_obj = NULL;
1922		r600_ring_init(rdev, &rdev->ring[R600_RING_TYPE_UVD_INDEX],
1923			       4096);
1924	}
1925
1926	rdev->ih.ring_obj = NULL;
1927	r600_ih_ring_init(rdev, 64 * 1024);
1928
1929	r = r600_pcie_gart_init(rdev);
1930	if (r)
1931		return r;
1932
1933	rdev->accel_working = true;
1934	r = rv770_startup(rdev);
1935	if (r) {
1936		dev_err(rdev->dev, "disabling GPU acceleration\n");
1937		r700_cp_fini(rdev);
1938		r600_dma_fini(rdev);
1939		r600_irq_fini(rdev);
1940		radeon_wb_fini(rdev);
1941		radeon_ib_pool_fini(rdev);
1942		radeon_irq_kms_fini(rdev);
1943		rv770_pcie_gart_fini(rdev);
1944		rdev->accel_working = false;
1945	}
1946
1947	return 0;
1948}
1949
1950void rv770_fini(struct radeon_device *rdev)
1951{
1952	radeon_pm_fini(rdev);
1953	r700_cp_fini(rdev);
1954	r600_dma_fini(rdev);
1955	r600_irq_fini(rdev);
1956	radeon_wb_fini(rdev);
1957	radeon_ib_pool_fini(rdev);
1958	radeon_irq_kms_fini(rdev);
1959	uvd_v1_0_fini(rdev);
1960	radeon_uvd_fini(rdev);
1961	rv770_pcie_gart_fini(rdev);
1962	r600_vram_scratch_fini(rdev);
1963	radeon_gem_fini(rdev);
1964	radeon_fence_driver_fini(rdev);
1965	radeon_agp_fini(rdev);
1966	radeon_bo_fini(rdev);
1967	radeon_atombios_fini(rdev);
1968	kfree(rdev->bios);
1969	rdev->bios = NULL;
1970}
1971
1972static void rv770_pcie_gen2_enable(struct radeon_device *rdev)
1973{
1974	u32 link_width_cntl, lanes, speed_cntl, tmp;
1975	u16 link_cntl2;
1976
1977	if (radeon_pcie_gen2 == 0)
1978		return;
1979
1980	if (rdev->flags & RADEON_IS_IGP)
1981		return;
1982
1983	if (!(rdev->flags & RADEON_IS_PCIE))
1984		return;
1985
1986	/* x2 cards have a special sequence */
1987	if (ASIC_IS_X2(rdev))
1988		return;
1989
1990	if ((rdev->pdev->bus->max_bus_speed != PCIE_SPEED_5_0GT) &&
1991		(rdev->pdev->bus->max_bus_speed != PCIE_SPEED_8_0GT))
1992		return;
1993
1994	DRM_INFO("enabling PCIE gen 2 link speeds, disable with radeon.pcie_gen2=0\n");
1995
1996	/* advertise upconfig capability */
1997	link_width_cntl = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
1998	link_width_cntl &= ~LC_UPCONFIGURE_DIS;
1999	WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, link_width_cntl);
2000	link_width_cntl = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
2001	if (link_width_cntl & LC_RENEGOTIATION_SUPPORT) {
2002		lanes = (link_width_cntl & LC_LINK_WIDTH_RD_MASK) >> LC_LINK_WIDTH_RD_SHIFT;
2003		link_width_cntl &= ~(LC_LINK_WIDTH_MASK |
2004				     LC_RECONFIG_ARC_MISSING_ESCAPE);
2005		link_width_cntl |= lanes | LC_RECONFIG_NOW |
2006			LC_RENEGOTIATE_EN | LC_UPCONFIGURE_SUPPORT;
2007		WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, link_width_cntl);
2008	} else {
2009		link_width_cntl |= LC_UPCONFIGURE_DIS;
2010		WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, link_width_cntl);
2011	}
2012
2013	speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
2014	if ((speed_cntl & LC_OTHER_SIDE_EVER_SENT_GEN2) &&
2015	    (speed_cntl & LC_OTHER_SIDE_SUPPORTS_GEN2)) {
2016
2017		tmp = RREG32(0x541c);
2018		WREG32(0x541c, tmp | 0x8);
2019		WREG32(MM_CFGREGS_CNTL, MM_WR_TO_CFG_EN);
2020		link_cntl2 = RREG16(0x4088);
2021		link_cntl2 &= ~TARGET_LINK_SPEED_MASK;
2022		link_cntl2 |= 0x2;
2023		WREG16(0x4088, link_cntl2);
2024		WREG32(MM_CFGREGS_CNTL, 0);
2025
2026		speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
2027		speed_cntl &= ~LC_TARGET_LINK_SPEED_OVERRIDE_EN;
2028		WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
2029
2030		speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
2031		speed_cntl |= LC_CLR_FAILED_SPD_CHANGE_CNT;
2032		WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
2033
2034		speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
2035		speed_cntl &= ~LC_CLR_FAILED_SPD_CHANGE_CNT;
2036		WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
2037
2038		speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
2039		speed_cntl |= LC_GEN2_EN_STRAP;
2040		WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
2041
2042	} else {
2043		link_width_cntl = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
2044		/* XXX: only disable it if gen1 bridge vendor == 0x111d or 0x1106 */
2045		if (1)
2046			link_width_cntl |= LC_UPCONFIGURE_DIS;
2047		else
2048			link_width_cntl &= ~LC_UPCONFIGURE_DIS;
2049		WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, link_width_cntl);
2050	}
2051}
2052